fix: 初验针对修改
This commit is contained in:
151
internal/jobs/integrity.go
Normal file
151
internal/jobs/integrity.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/logger"
|
||||
"git.apinb.com/ops/files/internal/config"
|
||||
"git.apinb.com/ops/files/internal/impl"
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
"git.apinb.com/ops/files/internal/storage"
|
||||
)
|
||||
|
||||
type IntegrityStatus struct {
|
||||
Running bool `json:"running"`
|
||||
LastStarted time.Time `json:"last_started,omitempty"`
|
||||
LastSuccess time.Time `json:"last_success,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
Checked int64 `json:"checked"`
|
||||
Missing int64 `json:"missing"`
|
||||
SizeMismatch int64 `json:"size_mismatch"`
|
||||
ETagMismatch int64 `json:"etag_mismatch"`
|
||||
}
|
||||
|
||||
var integrityRuntime struct {
|
||||
sync.RWMutex
|
||||
IntegrityStatus
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func StartIntegrity(ctx context.Context) error {
|
||||
integrityRuntime.Lock()
|
||||
if integrityRuntime.Running {
|
||||
integrityRuntime.Unlock()
|
||||
return fmt.Errorf("文件完整性任务已启动")
|
||||
}
|
||||
integrityRuntime.Unlock()
|
||||
if err := runIntegrity(ctx); err != nil {
|
||||
return fmt.Errorf("首次核对文件完整性失败: %w", err)
|
||||
}
|
||||
integrityRuntime.Lock()
|
||||
integrityRuntime.Running = true
|
||||
integrityRuntime.Unlock()
|
||||
integrityRuntime.wg.Add(1)
|
||||
go func() {
|
||||
defer integrityRuntime.wg.Done()
|
||||
defer func() {
|
||||
integrityRuntime.Lock()
|
||||
integrityRuntime.Running = false
|
||||
integrityRuntime.Unlock()
|
||||
}()
|
||||
ticker := time.NewTicker(time.Duration(config.Spec.Cleanup.IntegrityIntervalSeconds) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := runIntegrity(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
logger.Errorf("stage=files_integrity error=%v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func IntegrityStatusSnapshot() IntegrityStatus {
|
||||
integrityRuntime.RLock()
|
||||
defer integrityRuntime.RUnlock()
|
||||
return integrityRuntime.IntegrityStatus
|
||||
}
|
||||
|
||||
func waitIntegrity(ctx context.Context) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
integrityRuntime.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func runIntegrity(ctx context.Context) error {
|
||||
startedAt := time.Now().UTC()
|
||||
status := IntegrityStatus{Running: true, LastStarted: startedAt}
|
||||
integrityRuntime.Lock()
|
||||
status.LastSuccess = integrityRuntime.LastSuccess
|
||||
integrityRuntime.IntegrityStatus = status
|
||||
integrityRuntime.Unlock()
|
||||
|
||||
var lastID uint
|
||||
for {
|
||||
var fileObjects []models.FileObject
|
||||
result := impl.DBService.WithContext(ctx).
|
||||
Where("status = ? AND id > ?", models.FileStatusReady, lastID).
|
||||
Order("id ASC").Limit(config.Spec.Cleanup.IntegrityBatchSize).Find(&fileObjects)
|
||||
if result.Error != nil {
|
||||
return finishIntegrity(status, result.Error)
|
||||
}
|
||||
if len(fileObjects) == 0 {
|
||||
break
|
||||
}
|
||||
for _, fileObject := range fileObjects {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return finishIntegrity(status, err)
|
||||
}
|
||||
status.Checked++
|
||||
objectInfo, err := impl.StorageService.Stat(ctx, fileObject.ObjectKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, storage.ErrObjectNotFound) {
|
||||
status.Missing++
|
||||
continue
|
||||
}
|
||||
return finishIntegrity(status, err)
|
||||
}
|
||||
if objectInfo.Size != fileObject.ActualSize {
|
||||
status.SizeMismatch++
|
||||
}
|
||||
if objectInfo.StorageETag != fileObject.StorageETag {
|
||||
status.ETagMismatch++
|
||||
}
|
||||
}
|
||||
lastID = fileObjects[len(fileObjects)-1].ID
|
||||
if len(fileObjects) < config.Spec.Cleanup.IntegrityBatchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
status.LastSuccess = time.Now().UTC()
|
||||
status.Running = integrityRuntime.Running
|
||||
integrityRuntime.Lock()
|
||||
integrityRuntime.IntegrityStatus = status
|
||||
integrityRuntime.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func finishIntegrity(status IntegrityStatus, err error) error {
|
||||
status.LastError = err.Error()
|
||||
status.Running = integrityRuntime.Running
|
||||
integrityRuntime.Lock()
|
||||
integrityRuntime.IntegrityStatus = status
|
||||
integrityRuntime.Unlock()
|
||||
return err
|
||||
}
|
||||
@@ -2,6 +2,9 @@ package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/logger"
|
||||
@@ -9,24 +12,108 @@ import (
|
||||
"git.apinb.com/ops/files/internal/impl"
|
||||
"git.apinb.com/ops/files/internal/lifecycle"
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
"git.apinb.com/ops/files/internal/storage"
|
||||
)
|
||||
|
||||
func StartPendingUploadCleanup() {
|
||||
// CleanupStatus 描述过期上传清理任务的运行状态。
|
||||
type CleanupStatus struct {
|
||||
Running bool `json:"running"`
|
||||
LastStarted time.Time `json:"last_started,omitempty"`
|
||||
LastSuccess time.Time `json:"last_success,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
var cleanupRuntime struct {
|
||||
sync.RWMutex
|
||||
CleanupStatus
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// StartPendingUploadCleanup 启动可取消的过期上传清理任务。
|
||||
func StartPendingUploadCleanup(ctx context.Context) error {
|
||||
cleanupRuntime.Lock()
|
||||
if cleanupRuntime.Running {
|
||||
cleanupRuntime.Unlock()
|
||||
return fmt.Errorf("过期上传清理任务已启动")
|
||||
}
|
||||
cleanupRuntime.Unlock()
|
||||
|
||||
if err := runCleanup(ctx); err != nil {
|
||||
return fmt.Errorf("首次清理过期上传失败: %w", err)
|
||||
}
|
||||
|
||||
cleanupRuntime.Lock()
|
||||
cleanupRuntime.Running = true
|
||||
cleanupRuntime.Unlock()
|
||||
cleanupRuntime.wg.Add(1)
|
||||
go func() {
|
||||
cleanupPendingUploads()
|
||||
defer cleanupRuntime.wg.Done()
|
||||
defer func() {
|
||||
cleanupRuntime.Lock()
|
||||
cleanupRuntime.Running = false
|
||||
cleanupRuntime.Unlock()
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(config.Spec.Cleanup.IntervalSeconds) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
cleanupPendingUploads()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := runCleanup(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
logger.Errorf("stage=cleanup error=%v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupPendingUploads() {
|
||||
// Status 返回清理任务状态快照。
|
||||
func Status() CleanupStatus {
|
||||
cleanupRuntime.RLock()
|
||||
defer cleanupRuntime.RUnlock()
|
||||
return cleanupRuntime.CleanupStatus
|
||||
}
|
||||
|
||||
// Wait 等待清理任务退出。
|
||||
func Wait(ctx context.Context) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
cleanupRuntime.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return waitIntegrity(ctx)
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func runCleanup(ctx context.Context) error {
|
||||
startedAt := time.Now().UTC()
|
||||
cleanupRuntime.Lock()
|
||||
cleanupRuntime.LastStarted = startedAt
|
||||
cleanupRuntime.Unlock()
|
||||
|
||||
err := cleanupPendingUploads(ctx)
|
||||
cleanupRuntime.Lock()
|
||||
defer cleanupRuntime.Unlock()
|
||||
if err != nil {
|
||||
cleanupRuntime.LastError = err.Error()
|
||||
return err
|
||||
}
|
||||
cleanupRuntime.LastSuccess = time.Now().UTC()
|
||||
cleanupRuntime.LastError = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupPendingUploads(ctx context.Context) error {
|
||||
var fileObjects []models.FileObject
|
||||
now := time.Now()
|
||||
result := impl.DBService.
|
||||
result := impl.DBService.WithContext(ctx).
|
||||
Where(
|
||||
"(status = ? AND expires_at <= ?) OR (status = ? AND (delete_lease_until IS NULL OR delete_lease_until <= ?))",
|
||||
models.FileStatusPending,
|
||||
@@ -38,10 +125,10 @@ func cleanupPendingUploads() {
|
||||
Limit(100).
|
||||
Find(&fileObjects)
|
||||
if result.Error != nil {
|
||||
logger.Error("stage=scan")
|
||||
return
|
||||
return fmt.Errorf("扫描待清理文件失败: %w", result.Error)
|
||||
}
|
||||
|
||||
var cleanupErrors []error
|
||||
for _, fileObject := range fileObjects {
|
||||
var lease lifecycle.DeletionLease
|
||||
var claimed bool
|
||||
@@ -54,26 +141,37 @@ func cleanupPendingUploads() {
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("identity=%s stage=claim", fileObject.Identity)
|
||||
cleanupErrors = append(cleanupErrors, fmt.Errorf("文件 %s 获取删除租约失败: %w", fileObject.Identity, err))
|
||||
continue
|
||||
}
|
||||
if !claimed {
|
||||
continue
|
||||
}
|
||||
|
||||
cleanupClaimedFile(fileObject, lease)
|
||||
if err := cleanupClaimedFile(ctx, fileObject, lease); err != nil {
|
||||
cleanupErrors = append(cleanupErrors, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(cleanupErrors...)
|
||||
}
|
||||
|
||||
func cleanupClaimedFile(fileObject models.FileObject, lease lifecycle.DeletionLease) {
|
||||
operationCtx, cancel := context.WithTimeout(context.Background(), lifecycle.DeleteOperationTimeout)
|
||||
func cleanupClaimedFile(ctx context.Context, fileObject models.FileObject, lease lifecycle.DeletionLease) error {
|
||||
operationCtx, cancel := context.WithTimeout(ctx, lifecycle.DeleteOperationTimeout)
|
||||
defer cancel()
|
||||
|
||||
if receiver, ok := impl.StorageService.(storage.UploadReceiver); ok {
|
||||
if err := receiver.DiscardUpload(operationCtx, fileObject.Identity); err != nil {
|
||||
return fmt.Errorf("文件 %s 清理暂存内容失败: %w", fileObject.Identity, err)
|
||||
}
|
||||
}
|
||||
if err := impl.StorageService.Delete(operationCtx, fileObject.ObjectKey); err != nil {
|
||||
logger.Errorf("identity=%s stage=oss_delete", fileObject.Identity)
|
||||
return
|
||||
logger.Errorf("identity=%s stage=storage_delete", fileObject.Identity)
|
||||
return fmt.Errorf("文件 %s 删除对象失败: %w", fileObject.Identity, err)
|
||||
}
|
||||
|
||||
if err := lifecycle.FinalizeDeletion(impl.DBService.WithContext(operationCtx), fileObject.ID, lease.Token); err != nil {
|
||||
logger.Errorf("identity=%s stage=finalize", fileObject.Identity)
|
||||
return fmt.Errorf("文件 %s 完成删除失败: %w", fileObject.Identity, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user