fix: 初验针对修改

2026-09-09 16:42:21 +08:00
parent eb2722d8a0
commit b6ffe2fcea
40 changed files with 2179 additions and 279 deletions
--- a/internal/storage/local/capacity.go
+++ b/internal/storage/local/capacity.go
@@ -0,0 +1,46 @@
+package local
+
+import (
+	"fmt"
+	"math"
+
+	"git.apinb.com/ops/files/internal/storage"
+	"github.com/shirou/gopsutil/v3/disk"
+)
+
+func (c *Client) diskStatus() (*storage.DiskStatus, error) {
+	usage, err := disk.Usage(c.rootPath)
+	if err != nil {
+		return nil, fmt.Errorf("读取本地存储磁盘容量: %w", err)
+	}
+	level := "normal"
+	if usage.Free < c.minimumFreeBytes {
+		level = "critical"
+	} else if c.minimumFreeBytes <= math.MaxUint64/2 && usage.Free < c.minimumFreeBytes*2 {
+		level = "warning"
+	}
+	return &storage.DiskStatus{
+		Path:             c.rootPath,
+		TotalBytes:       usage.Total,
+		UsedBytes:        usage.Used,
+		FreeBytes:        usage.Free,
+		UsedPercent:      usage.UsedPercent,
+		MinimumFreeBytes: c.minimumFreeBytes,
+		Level:            level,
+	}, nil
+}
+
+func (c *Client) ensureCapacity(expectedSize int64) error {
+	if expectedSize < 0 {
+		return fmt.Errorf("文件大小不能为负数")
+	}
+	diskStatus, err := c.diskStatus()
+	if err != nil {
+		return err
+	}
+	requested := uint64(expectedSize)
+	if requested > math.MaxUint64-c.minimumFreeBytes || diskStatus.FreeBytes < requested+c.minimumFreeBytes {
+		return storage.ErrInsufficientSpace
+	}
+	return nil
+}
--- a/internal/storage/local/client.go
+++ b/internal/storage/local/client.go
@@ -0,0 +1,60 @@
+package local
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"git.apinb.com/ops/files/internal/config"
+)
+
+const containerName = "local"
+
+type Client struct {
+	rootPath         string
+	objectsPath      string
+	stagingPath      string
+	locksPath        string
+	externalBaseURL  string
+	signingSecret    string
+	accessTTLSeconds int64
+	minimumFreeBytes uint64
+}
+
+func New(storageConfig config.StorageConf) (*Client, error) {
+	rootPath, err := filepath.Abs(storageConfig.Local.RootPath)
+	if err != nil {
+		return nil, fmt.Errorf("解析本地存储根目录: %w", err)
+	}
+	if err := os.MkdirAll(rootPath, 0o750); err != nil {
+		return nil, fmt.Errorf("创建本地存储根目录: %w", err)
+	}
+	rootPath, err = filepath.EvalSymlinks(rootPath)
+	if err != nil {
+		return nil, fmt.Errorf("解析本地存储真实路径: %w", err)
+	}
+	client := &Client{
+		rootPath:         filepath.Clean(rootPath),
+		objectsPath:      filepath.Join(rootPath, "objects"),
+		stagingPath:      filepath.Join(rootPath, ".staging"),
+		locksPath:        filepath.Join(rootPath, ".locks"),
+		externalBaseURL:  strings.TrimRight(storageConfig.ExternalBaseURL, "/"),
+		signingSecret:    storageConfig.SigningSecret,
+		accessTTLSeconds: storageConfig.AccessTTLSeconds,
+		minimumFreeBytes: storageConfig.Local.MinimumFreeSpaceMB * 1024 * 1024,
+	}
+	for _, directory := range []string{client.objectsPath, client.stagingPath, client.locksPath} {
+		if err := client.ensureDirectory(directory); err != nil {
+			return nil, err
+		}
+	}
+	if _, err := client.Status(nil); err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+func (c *Client) Provider() string { return "local" }
+
+func (c *Client) Container() string { return containerName }
--- a/internal/storage/local/object.go
+++ b/internal/storage/local/object.go
@@ -0,0 +1,152 @@
+package local
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"git.apinb.com/ops/files/internal/storage"
+)
+
+func (c *Client) Open(ctx context.Context, objectKey string, byteRange *storage.ByteRange) (io.ReadCloser, error) {
+	path, err := c.resolveObjectPath(objectKey)
+	if err != nil {
+		return nil, err
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, storage.ErrObjectNotFound
+		}
+		return nil, fmt.Errorf("打开本地文件: %w", err)
+	}
+	if byteRange == nil {
+		return &contextReadCloser{ctx: ctx, reader: file, closer: file}, nil
+	}
+	info, err := file.Stat()
+	if err != nil {
+		_ = file.Close()
+		return nil, err
+	}
+	if byteRange.Start < 0 || byteRange.End < byteRange.Start || byteRange.End >= info.Size() {
+		_ = file.Close()
+		return nil, fmt.Errorf("文件读取范围无效")
+	}
+	if _, err := file.Seek(byteRange.Start, io.SeekStart); err != nil {
+		_ = file.Close()
+		return nil, err
+	}
+	return &contextReadCloser{ctx: ctx, reader: io.LimitReader(file, byteRange.Length()), closer: file}, nil
+}
+
+func (c *Client) Stat(ctx context.Context, objectKey string) (storage.ObjectInfo, error) {
+	path, err := c.resolveObjectPath(objectKey)
+	if err != nil {
+		return storage.ObjectInfo{}, err
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return storage.ObjectInfo{}, storage.ErrObjectNotFound
+		}
+		return storage.ObjectInfo{}, fmt.Errorf("打开本地文件: %w", err)
+	}
+	defer file.Close()
+	info, err := file.Stat()
+	if err != nil {
+		return storage.ObjectInfo{}, err
+	}
+	hasher := sha256.New()
+	if _, err := io.Copy(hasher, &contextReader{ctx: ctx, reader: file}); err != nil {
+		return storage.ObjectInfo{}, fmt.Errorf("计算本地文件摘要: %w", err)
+	}
+	return storage.ObjectInfo{
+		Size:         info.Size(),
+		StorageETag:  "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
+		LastModified: info.ModTime().UTC(),
+	}, nil
+}
+
+func (c *Client) Delete(_ context.Context, objectKey string) error {
+	path, err := c.resolveObjectPath(objectKey)
+	if err != nil {
+		return err
+	}
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("删除本地文件: %w", err)
+	}
+	c.removeEmptyParents(filepath.Dir(path))
+	return nil
+}
+
+func (c *Client) removeEmptyParents(directory string) {
+	boundary := filepath.Clean(c.objectsPath)
+	for current := filepath.Clean(directory); current != boundary; current = filepath.Dir(current) {
+		relative, err := filepath.Rel(boundary, current)
+		if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
+			return
+		}
+		if err := os.Remove(current); err != nil {
+			return
+		}
+	}
+}
+
+func (c *Client) Status(_ context.Context) (storage.RuntimeStatus, error) {
+	diskStatus, err := c.diskStatus()
+	if err != nil {
+		return storage.RuntimeStatus{Provider: c.Provider(), Container: c.Container()}, err
+	}
+	probe, err := os.CreateTemp(c.stagingPath, ".write-probe-")
+	if err != nil {
+		return storage.RuntimeStatus{Provider: c.Provider(), Container: c.Container(), Disk: diskStatus}, fmt.Errorf("本地存储目录不可写: %w", err)
+	}
+	probePath := probe.Name()
+	removeProbe := func() { _ = os.Remove(probePath) }
+	defer removeProbe()
+	if err := probe.Chmod(0o640); err != nil {
+		_ = probe.Close()
+		return storage.RuntimeStatus{Provider: c.Provider(), Container: c.Container(), Disk: diskStatus}, err
+	}
+	if _, err := probe.Write([]byte("files-storage-probe")); err != nil {
+		_ = probe.Close()
+		return storage.RuntimeStatus{Provider: c.Provider(), Container: c.Container(), Disk: diskStatus}, err
+	}
+	if err := probe.Sync(); err != nil {
+		_ = probe.Close()
+		return storage.RuntimeStatus{Provider: c.Provider(), Container: c.Container(), Disk: diskStatus}, err
+	}
+	if err := probe.Close(); err != nil {
+		return storage.RuntimeStatus{Provider: c.Provider(), Container: c.Container(), Disk: diskStatus}, err
+	}
+	if err := os.Remove(probePath); err != nil {
+		return storage.RuntimeStatus{Provider: c.Provider(), Container: c.Container(), Disk: diskStatus}, err
+	}
+	return storage.RuntimeStatus{
+		Provider:  c.Provider(),
+		Container: c.Container(),
+		Writable:  diskStatus.Level != "critical",
+		Disk:      diskStatus,
+	}, nil
+}
+
+type contextReadCloser struct {
+	ctx    context.Context
+	reader io.Reader
+	closer io.Closer
+}
+
+func (r *contextReadCloser) Read(buffer []byte) (int, error) {
+	if err := r.ctx.Err(); err != nil {
+		return 0, err
+	}
+	return r.reader.Read(buffer)
+}
+
+func (r *contextReadCloser) Close() error { return r.closer.Close() }
--- a/internal/storage/local/path.go
+++ b/internal/storage/local/path.go
@@ -0,0 +1,79 @@
+package local
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func (c *Client) resolveObjectPath(objectKey string) (string, error) {
+	if strings.TrimSpace(objectKey) == "" || filepath.IsAbs(objectKey) || strings.Contains(objectKey, "\\") {
+		return "", fmt.Errorf("对象键不是安全的相对路径")
+	}
+	for _, segment := range strings.Split(objectKey, "/") {
+		if segment == "" || segment == "." || segment == ".." {
+			return "", fmt.Errorf("对象键包含无效路径段")
+		}
+	}
+	target := filepath.Join(c.objectsPath, filepath.FromSlash(objectKey))
+	relative, err := filepath.Rel(c.objectsPath, target)
+	if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
+		return "", fmt.Errorf("对象路径超出存储目录")
+	}
+	if err := c.ensureNoSymlink(filepath.Dir(target), c.objectsPath); err != nil {
+		return "", err
+	}
+	return target, nil
+}
+
+func (c *Client) ensureDirectory(directory string) error {
+	if err := os.MkdirAll(directory, 0o750); err != nil {
+		return fmt.Errorf("创建本地存储目录 %s: %w", directory, err)
+	}
+	if err := c.ensureNoSymlink(directory, c.rootPath); err != nil {
+		return err
+	}
+	return os.Chmod(directory, 0o750)
+}
+
+func (c *Client) ensureNoSymlink(target, boundary string) error {
+	target = filepath.Clean(target)
+	boundary = filepath.Clean(boundary)
+	relative, err := filepath.Rel(boundary, target)
+	if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
+		return fmt.Errorf("路径超出本地存储边界")
+	}
+	current := boundary
+	if relative == "." {
+		return rejectSymlink(current)
+	}
+	if err := rejectSymlink(current); err != nil {
+		return err
+	}
+	for _, segment := range strings.Split(relative, string(filepath.Separator)) {
+		current = filepath.Join(current, segment)
+		info, statErr := os.Lstat(current)
+		if os.IsNotExist(statErr) {
+			continue
+		}
+		if statErr != nil {
+			return fmt.Errorf("检查路径 %s: %w", current, statErr)
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			return fmt.Errorf("本地存储路径不允许符号链接: %s", current)
+		}
+	}
+	return nil
+}
+
+func rejectSymlink(path string) error {
+	info, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+	if info.Mode()&os.ModeSymlink != 0 {
+		return fmt.Errorf("本地存储路径不允许符号链接: %s", path)
+	}
+	return nil
+}
--- a/internal/storage/local/upload.go
+++ b/internal/storage/local/upload.go
@@ -0,0 +1,190 @@
+package local
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"git.apinb.com/ops/files/internal/signing"
+	"git.apinb.com/ops/files/internal/storage"
+)
+
+func (c *Client) PrepareUpload(_ context.Context, request storage.UploadRequest) (storage.UploadInstruction, error) {
+	if err := c.ensureCapacity(request.ExpectedSize); err != nil {
+		return storage.UploadInstruction{}, err
+	}
+	signature := signing.SignUpload(c.signingSecret, request)
+	return storage.UploadInstruction{
+		Method: "PUT",
+		URL: fmt.Sprintf("%s/v1/local/uploads/%s?expires=%d&signature=%s",
+			c.externalBaseURL, request.FileID, request.ExpiresAt.Unix(), signature),
+		Headers: map[string]string{
+			"Content-Type":   request.ContentType,
+			"Content-Length": fmt.Sprintf("%d", request.ExpectedSize),
+		},
+		ExpiresAt: request.ExpiresAt,
+	}, nil
+}
+
+func (c *Client) AcquireUpload(ctx context.Context, fileID string) (func() error, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	if !validFileID(fileID) {
+		return nil, fmt.Errorf("文件标识无效")
+	}
+	lockPath := filepath.Join(c.locksPath, fileID+".lock")
+	lockFile, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
+	if err != nil {
+		if os.IsExist(err) {
+			return nil, storage.ErrUploadLocked
+		}
+		return nil, fmt.Errorf("获取上传锁: %w", err)
+	}
+	return func() error {
+		closeErr := lockFile.Close()
+		removeErr := os.Remove(lockPath)
+		if os.IsNotExist(removeErr) {
+			removeErr = nil
+		}
+		if closeErr != nil {
+			return closeErr
+		}
+		return removeErr
+	}, nil
+}
+
+func (c *Client) ReceiveUpload(ctx context.Context, request storage.UploadRequest, source io.Reader) (storage.ObjectInfo, error) {
+	if err := c.ensureCapacity(request.ExpectedSize); err != nil {
+		return storage.ObjectInfo{}, err
+	}
+	targetPath, err := c.resolveObjectPath(request.ObjectKey)
+	if err != nil {
+		return storage.ObjectInfo{}, err
+	}
+	if err := c.ensureDirectory(filepath.Dir(targetPath)); err != nil {
+		return storage.ObjectInfo{}, err
+	}
+	if existing, statErr := c.Stat(ctx, request.ObjectKey); statErr == nil {
+		if existing.Size != request.ExpectedSize {
+			return storage.ObjectInfo{}, fmt.Errorf("已存在文件的大小与上传请求不一致")
+		}
+		return existing, nil
+	} else if !errors.Is(statErr, storage.ErrObjectNotFound) {
+		return storage.ObjectInfo{}, statErr
+	}
+	nonce, err := signing.UploadNonce(c.signingSecret, request)
+	if err != nil {
+		return storage.ObjectInfo{}, err
+	}
+	stagingPath := filepath.Join(c.stagingPath, request.FileID+"."+nonce+".part")
+	staging, err := os.OpenFile(stagingPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
+	if err != nil {
+		return storage.ObjectInfo{}, fmt.Errorf("创建上传暂存文件: %w", err)
+	}
+	committed := false
+	defer func() {
+		_ = staging.Close()
+		if !committed {
+			_ = os.Remove(stagingPath)
+		}
+	}()
+
+	hasher := sha256.New()
+	written, err := io.Copy(io.MultiWriter(staging, hasher), io.LimitReader(&contextReader{ctx: ctx, reader: source}, request.ExpectedSize+1))
+	if err != nil {
+		return storage.ObjectInfo{}, fmt.Errorf("写入上传暂存文件: %w", err)
+	}
+	if written != request.ExpectedSize {
+		return storage.ObjectInfo{}, fmt.Errorf("上传文件大小不匹配: 期望 %d，实际 %d", request.ExpectedSize, written)
+	}
+	if err := staging.Sync(); err != nil {
+		return storage.ObjectInfo{}, fmt.Errorf("同步上传暂存文件: %w", err)
+	}
+	if err := staging.Close(); err != nil {
+		return storage.ObjectInfo{}, fmt.Errorf("关闭上传暂存文件: %w", err)
+	}
+	if err := os.Link(stagingPath, targetPath); err != nil {
+		return storage.ObjectInfo{}, fmt.Errorf("提交上传文件: %w", err)
+	}
+	if err := os.Chmod(targetPath, 0o640); err != nil {
+		_ = os.Remove(targetPath)
+		return storage.ObjectInfo{}, fmt.Errorf("设置上传文件权限: %w", err)
+	}
+	if err := os.Remove(stagingPath); err != nil {
+		_ = os.Remove(targetPath)
+		return storage.ObjectInfo{}, fmt.Errorf("清理上传暂存文件: %w", err)
+	}
+	if err := syncDirectory(filepath.Dir(targetPath)); err != nil {
+		_ = os.Remove(targetPath)
+		return storage.ObjectInfo{}, err
+	}
+	committed = true
+	return storage.ObjectInfo{
+		Size:         written,
+		StorageETag:  "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
+		LastModified: time.Now().UTC(),
+	}, nil
+}
+
+func (c *Client) DiscardUpload(_ context.Context, fileID string) error {
+	if !validFileID(fileID) {
+		return fmt.Errorf("文件标识无效")
+	}
+	entries, err := os.ReadDir(c.stagingPath)
+	if err != nil {
+		return err
+	}
+	prefix := fileID + "."
+	for _, entry := range entries {
+		if !entry.IsDir() && strings.HasPrefix(entry.Name(), prefix) && strings.HasSuffix(entry.Name(), ".part") {
+			if err := os.Remove(filepath.Join(c.stagingPath, entry.Name())); err != nil && !os.IsNotExist(err) {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+type contextReader struct {
+	ctx    context.Context
+	reader io.Reader
+}
+
+func (r *contextReader) Read(buffer []byte) (int, error) {
+	if err := r.ctx.Err(); err != nil {
+		return 0, err
+	}
+	return r.reader.Read(buffer)
+}
+
+func validFileID(fileID string) bool {
+	if fileID == "" {
+		return false
+	}
+	for _, character := range fileID {
+		if (character < '0' || character > '9') && (character < 'A' || character > 'Z') && (character < 'a' || character > 'z') && character != '-' {
+			return false
+		}
+	}
+	return true
+}
+
+func syncDirectory(path string) error {
+	directory, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("打开目录进行同步: %w", err)
+	}
+	defer directory.Close()
+	if err := directory.Sync(); err != nil {
+		return fmt.Errorf("同步文件目录: %w", err)
+	}
+	return nil
+}