bsm-infra/full
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix version 1

Browse Source
This commit is contained in:
yanweidong
2026-09-22 21:15:34 +08:00
parent 9f86366638
commit d63d7e8b3a
277 changed files with 9959 additions and 1514 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
module/base/fts/internal/logic/guard_test.go Normal file
View File

@@ -0,0 +1,56 @@
package logic
import (
"path/filepath"
"strings"
"testing"
)
// 校验存储目录拼接不会越出上传根目录
func TestSafeJoin(t *testing.T) {
root := "./uploader/"
cleanRoot := filepath.Clean(root)
cases := []struct {
name string
sub []string
}{
{"正常相对路径", []string{"frc", "2026-01/ab"}},
{"bucket穿越", []string{"../../etc/cron.d", "2026-01/ab"}},
{"子目录穿越", []string{"frc", "../../../etc"}},
{"绝对路径bucket", []string{"/etc/cron.d", "2026-01/ab"}},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := safeJoin(root, c.sub...)
if err != nil {
return // 被拒绝同样是安全的
}
if got != cleanRoot && !strings.HasPrefix(got, cleanRoot+string(filepath.Separator)) {
t.Fatalf("路径越出上传根目录: %s", got)
}
})
}
if _, err := safeJoin(root, "../../etc/cron.d", "2026-01/ab"); err == nil {
t.Fatal("路径穿越未被拒绝")
}
}
// 校验 bucket 名称白名单
func TestBucketPattern(t *testing.T) {
valid := []string{"frc", "my-bucket", "my_bucket", "A1"}
for _, v := range valid {
if !bucketPattern.MatchString(v) {
t.Errorf("应允许的 bucket 被拒绝: %q", v)
}
}
invalid := []string{"", "../etc", "a/b", "a\\b", "a b", "a.b", strings.Repeat("a", 65)}
for _, v := range invalid {
if bucketPattern.MatchString(v) {
t.Errorf("应拒绝的 bucket 被放行: %q", v)
}
}
}

26
module/base/fts/internal/logic/handler.go
View File

@@ -6,7 +6,9 @@ import (
"io"
"log"
"mime/multipart"
"os"
"path/filepath"
"regexp"
"strings"
"bsm/full/module/base/fts/internal/config"
@@ -20,6 +22,9 @@ import (
"github.com/gin-gonic/gin"
)
// bucketPattern bucket名称只允许字母、数字、下划线和中划线
var bucketPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
// Handler 上传文件
func Handler(c *gin.Context) {
var (
@@ -40,6 +45,11 @@ func Handler(c *gin.Context) {
infra.Response.Error(c, errcode.NewError(400, "参数错误"))
return
}
// 校验 bucket 名称，避免非法字符导致本地路径穿越或写入任意对象存储桶
if !bucketPattern.MatchString(bucket) {
infra.Response.Error(c, errcode.NewError(400, "bucket参数错误"))
return
}
// if !strings.Contains(provider, "local") {
// log.Println("provider参数错误")
// infra.Response.Error(c, errcode.NewError(400, "provider参数错误"))
@@ -99,6 +109,8 @@ func Handler(c *gin.Context) {
return
}
if err := impl.DBService.Create(&record).Error; err != nil {
// 落库失败时清理已写入的本地文件或对象存储对象，避免残留孤儿文件
removeStoredFile(&record, bucket)
infra.Response.Error(c, err)
return
}
@@ -106,6 +118,20 @@ func Handler(c *gin.Context) {
infra.Response.Success(c, record)
}
// removeStoredFile 清理已落盘/已上传的文件。单条 INSERT 无需事务，落库失败时按记录回滚外部存储
func removeStoredFile(record *models.FtsRecord, bucket string) {
if record.LocalPath != "" {
if err := os.Remove(record.LocalPath); err != nil {
log.Printf("清理本地文件失败: %v\n", err)
}
}
if record.OssPath != "" {
if err := RemoveOssObject(bucket, record.OssPath); err != nil {
log.Printf("清理对象存储文件失败: %v\n", err)
}
}
}
func chksum(fh *multipart.FileHeader) (string, error) {
// 2. 打开文件读取内容
file, err := fh.Open()

67
module/base/fts/internal/logic/provider.go
View File

@@ -3,17 +3,16 @@ package logic
import (
"context"
"errors"
"fmt"
"io"
"log"
"mime/multipart"
"os"
"path/filepath"
"strings"
"time"
"bsm/full/module/base/fts/internal/config"
"bsm/full/module/base/fts/internal/models"
"git.apinb.com/bsm-sdk/core/infra"
"git.apinb.com/bsm-sdk/core/utils"
"github.com/gin-gonic/gin"
"github.com/minio/minio-go/v7"
@@ -23,13 +22,17 @@ import (
// LocalUpload 本地上传
func LocalUpload(fh *multipart.FileHeader, record *models.FtsRecord, ctx *gin.Context, bucket string) (err error) {
subdirpath := NewSubdir(record.OwnerIdentity)
saveDir := filepath.Join(config.Spec.Local.UploadDir, bucket, subdirpath)
// 拼接后校验路径必须位于配置的上传根目录内，防止路径穿越
saveDir, err := safeJoin(config.Spec.Local.UploadDir, bucket, subdirpath)
if err != nil {
log.Println("存储路径非法:", err)
return err
}
// 创建目录并确保权限正确
if err = os.MkdirAll(saveDir, 0755); err != nil {
log.Println("目录创建失败:", err)
infra.Response.Error(ctx, errors.New("目录创建失败"))
return
return errors.New("目录创建失败")
}
// 保存文件到指定路径
@@ -40,30 +43,26 @@ func LocalUpload(fh *multipart.FileHeader, record *models.FtsRecord, ctx *gin.Co
file, err := os.OpenFile(savePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
log.Println("文件创建失败:", err)
infra.Response.Error(ctx, err)
return
return err
}
defer file.Close()
src, err := fh.Open()
if err != nil {
log.Println("文件打开失败:", err)
infra.Response.Error(ctx, err)
return
return err
}
defer src.Close()
if _, err = io.Copy(file, src); err != nil {
log.Println("文件保存失败:", err)
infra.Response.Error(ctx, err)
return
return err
}
// 再次确认文件权限
if err = os.Chmod(savePath, 0644); err != nil {
log.Printf("警告: 设置文件权限失败: %v", err)
infra.Response.Error(ctx, err)
return
return err
}
record.LocalPath = savePath
@@ -75,11 +74,7 @@ func LocalUpload(fh *multipart.FileHeader, record *models.FtsRecord, ctx *gin.Co
// OssUpload 上传文件到指定的MinIO存储桶
func OssUpload(file *multipart.FileHeader, record *models.FtsRecord, c *gin.Context, bucket string) (err error) {
// Initialize minio client object.
minioClient, err := minio.New(config.Spec.MinioOss.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.Spec.MinioOss.AccessKeyID, config.Spec.MinioOss.AccessKeySecret, ""), // 修正字段名
Secure: config.Spec.MinioOss.UseSSL,
})
minioClient, err := newMinioClient()
if err != nil {
return err
}
@@ -97,19 +92,47 @@ func OssUpload(file *multipart.FileHeader, record *models.FtsRecord, c *gin.Cont
defer src.Close()
_, err = minioClient.PutObject(context.Background(), bucket, savePath, src, file.Size, minio.PutObjectOptions{ContentType: "application/octet-stream"})
if err != nil {
fmt.Println("err = ", err)
return err
}
record.OssPath = savePath
fmt.Println("savePath = ", savePath)
record.ResultUrl = config.Spec.MinioOss.Site + "/" + bucket + "/" + savePath
fmt.Println("record.ResultUrl = ", record.ResultUrl)
return nil
}
// RemoveOssObject 删除已上传到MinIO的对象，用于落库失败后的清理
func RemoveOssObject(bucket, objectName string) error {
if bucket == "" || objectName == "" {
return nil
}
minioClient, err := newMinioClient()
if err != nil {
return err
}
return minioClient.RemoveObject(context.Background(), bucket, objectName, minio.RemoveObjectOptions{})
}
// newMinioClient 创建MinIO客户端
func newMinioClient() (*minio.Client, error) {
return minio.New(config.Spec.MinioOss.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.Spec.MinioOss.AccessKeyID, config.Spec.MinioOss.AccessKeySecret, ""),
Secure: config.Spec.MinioOss.UseSSL,
})
}
// safeJoin 在根目录下安全拼接子路径，拒绝越出根目录的路径，防止路径穿越
func safeJoin(root string, sub ...string) (string, error) {
root = filepath.Clean(root)
target := filepath.Join(append([]string{root}, sub...)...)
if target != root && !strings.HasPrefix(target, root+string(filepath.Separator)) {
return "", errors.New("存储路径越界")
}
return target, nil
}
func NewSubdir(identity string) string {
ym := time.Now().Format("2006-01")
return ym + "/" + identity[0:2]

6
module/base/fts/internal/routers/register_test.go
View File

@@ -13,9 +13,9 @@ func TestRoutesUseRESTModulePrefix(t *testing.T) {
Register("fts", engine)
expected := map[string]bool{
"GET /rest/fts/v1/ping": false,
"GET /rest/fts/v1/config": false,
"POST /rest/fts/v1/uploader": false,
"GET /rest/fts/ping": false,
"GET /rest/fts/config": false,
"POST /rest/fts/uploader": false,
}
for _, route := range engine.Routes() {
key := route.Method + " " + route.Path