diff --git a/backend/app/services/common.go b/backend/app/services/common.go index 6810a34..c86141d 100644 --- a/backend/app/services/common.go +++ b/backend/app/services/common.go @@ -349,20 +349,20 @@ func (s *common) CompleteUpload(ctx context.Context, tenantID, userID int64, for Meta: existing.Meta, } } else { - finalPath := filepath.Join(localPath, objectKey) - if err := os.MkdirAll(filepath.Dir(finalPath), 0o755); err != nil { - return nil, errorx.ErrInternalError.WithCause(err) + contentType := strings.TrimSpace(meta.MimeType) + if contentType == "" { + contentType = "application/octet-stream" } - if err := os.Rename(mergedPath, finalPath); err != nil { - return nil, errorx.ErrInternalError.WithCause(err) + if err := s.storage.PutObject(ctx, objectKey, mergedPath, contentType); err != nil { + return nil, errorx.ErrInternalError.WithCause(err).WithMsg("上传存储失败") } asset = &models.MediaAsset{ TenantID: tid, UserID: userID, Type: consts.MediaAssetType(meta.Type), Status: consts.MediaAssetStatusUploaded, - Provider: "local", - Bucket: "default", + Provider: s.storage.Provider(), + Bucket: s.storage.Bucket(), ObjectKey: objectKey, Hash: hash, Meta: types.NewJSONType(fields.MediaAssetMeta{ @@ -513,12 +513,12 @@ func (s *common) Upload( Meta: existing.Meta, } } else { - dstPath := filepath.Join(localPath, objectKey) - if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { - return nil, errorx.ErrInternalError.WithCause(err).WithMsg("failed to create storage directory") + contentType := strings.TrimSpace(file.Header.Get("Content-Type")) + if contentType == "" { + contentType = "application/octet-stream" } - if err := os.Rename(tmpPath, dstPath); err != nil { - return nil, errorx.ErrInternalError.WithCause(err).WithMsg("failed to finalize file") + if err := s.storage.PutObject(ctx, objectKey, tmpPath, contentType); err != nil { + return nil, errorx.ErrInternalError.WithCause(err).WithMsg("上传存储失败") } os.RemoveAll(tmpDir) @@ -528,8 +528,8 @@ func (s *common) Upload( UserID: userID, Type: consts.MediaAssetType(typeArg), Status: consts.MediaAssetStatusUploaded, - Provider: "local", - Bucket: "default", + Provider: s.storage.Provider(), + Bucket: s.storage.Bucket(), ObjectKey: objectKey, Hash: hash, Meta: types.NewJSONType(fields.MediaAssetMeta{ diff --git a/backend/config.test.toml b/backend/config.test.toml index 471f725..1c7b6fa 100644 --- a/backend/config.test.toml +++ b/backend/config.test.toml @@ -60,3 +60,9 @@ Type = "local" LocalPath = "./storage" Secret = "test-storage-secret" BaseURL = "/v1/storage" +AccessKey = "" +SecretKey = "" +Region = "" +Bucket = "" +Endpoint = "" +PathStyle = true diff --git a/backend/config.toml b/backend/config.toml index de15d4b..f347740 100644 --- a/backend/config.toml +++ b/backend/config.toml @@ -137,5 +137,17 @@ Type = "local" LocalPath = "./storage" # 签名密钥 Secret = "your-storage-secret" -# 公共访问URL前缀 +# 公共访问URL前缀（本地存储使用） BaseURL = "/v1/storage" +# AccessKey（S3/MinIO） +AccessKey = "" +# SecretKey（S3/MinIO） +SecretKey = "" +# Region（S3/MinIO） +Region = "" +# Bucket（S3/MinIO） +Bucket = "" +# Endpoint（S3/MinIO，示例：http://127.0.0.1:9000） +Endpoint = "" +# PathStyle（S3/MinIO） +PathStyle = true diff --git a/backend/go.mod b/backend/go.mod index ab92dbb..a49ee42 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -39,6 +39,17 @@ require ( gorm.io/plugin/dbresolver v1.6.2 ) +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.97 // indirect + github.com/rs/xid v1.6.0 // indirect +) + replace google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 require ( diff --git a/backend/go.sum b/backend/go.sum index 8ed0c42..06c3bc4 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -47,6 +47,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -160,6 +162,11 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -177,6 +184,12 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= +github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= +github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= @@ -220,6 +233,8 @@ github.com/rogeecn/swag v1.0.1 h1:s1yxLgopqO1m8sqGjVmt6ocMBRubMPIh2JtIPG4xjQE= github.com/rogeecn/swag v1.0.1/go.mod h1:flG2NXERPxlRl2VdpU2VXTO8iBnQiERyowOXSkZVMOc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= diff --git a/backend/providers/storage/config.go b/backend/providers/storage/config.go index 7bec59d..d628578 100644 --- a/backend/providers/storage/config.go +++ b/backend/providers/storage/config.go @@ -5,4 +5,11 @@ type Config struct { LocalPath string // for local Secret string // for signing BaseURL string // public url prefix + // S3-compatible config + AccessKey string + SecretKey string + Region string + Bucket string + Endpoint string + PathStyle bool } diff --git a/backend/providers/storage/provider.go b/backend/providers/storage/provider.go index e67a3f3..abdfba5 100644 --- a/backend/providers/storage/provider.go +++ b/backend/providers/storage/provider.go @@ -1,6 +1,7 @@ package storage import ( + "context" "crypto/hmac" "crypto/sha256" "encoding/hex" @@ -9,8 +10,11 @@ import ( "os" "path/filepath" "strconv" + "strings" "time" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "go.ipao.vip/atom/container" "go.ipao.vip/atom/opt" ) @@ -33,16 +37,23 @@ func Provide(opts ...opt.Option) error { return err } return container.Container.Provide(func() (*Storage, error) { - return &Storage{Config: &config}, nil + store := &Storage{Config: &config} + if store.storageType() == "s3" { + if _, err := store.s3ClientForUse(); err != nil { + return nil, err + } + } + return store, nil }, o.DiOptions()...) } type Storage struct { - Config *Config + Config *Config + s3Client *minio.Client } func (s *Storage) Delete(key string) error { - if s.Config.Type == "local" { + if s.storageType() == "local" { localPath := s.Config.LocalPath if localPath == "" { localPath = "./storage" @@ -50,15 +61,41 @@ func (s *Storage) Delete(key string) error { path := filepath.Join(localPath, key) return os.Remove(path) } - // TODO: S3 implementation - return nil + client, err := s.s3ClientForUse() + if err != nil { + return err + } + return client.RemoveObject(context.Background(), s.Config.Bucket, key, minio.RemoveObjectOptions{}) } func (s *Storage) SignURL(method, key string, expires time.Duration) (string, error) { + if s.storageType() == "s3" { + client, err := s.s3ClientForUse() + if err != nil { + return "", err + } + switch strings.ToUpper(method) { + case "GET": + u, err := client.PresignedGetObject(context.Background(), s.Config.Bucket, key, expires, nil) + if err != nil { + return "", err + } + return u.String(), nil + case "PUT": + u, err := client.PresignedPutObject(context.Background(), s.Config.Bucket, key, expires) + if err != nil { + return "", err + } + return u.String(), nil + default: + return "", fmt.Errorf("unsupported method") + } + } + exp := time.Now().Add(expires).Unix() sign := s.signature(method, key, exp) - baseURL := s.Config.BaseURL + baseURL := strings.TrimRight(s.Config.BaseURL, "/") // Ensure BaseURL doesn't end with slash if we add one // Simplified: assume standard /v1/storage prefix in BaseURL or append it // We'll append / @@ -77,6 +114,9 @@ func (s *Storage) SignURL(method, key string, expires time.Duration) (string, er } func (s *Storage) Verify(method, key, expStr, sign string) error { + if s.storageType() == "s3" { + return fmt.Errorf("s3 storage does not use signed local urls") + } exp, err := strconv.ParseInt(expStr, 10, 64) if err != nil { return fmt.Errorf("invalid expiry") @@ -98,3 +138,105 @@ func (s *Storage) signature(method, key string, exp int64) string { h.Write([]byte(str)) return hex.EncodeToString(h.Sum(nil)) } + +func (s *Storage) PutObject(ctx context.Context, key, filePath, contentType string) error { + if s.storageType() == "local" { + localPath := s.Config.LocalPath + if localPath == "" { + localPath = "./storage" + } + dstPath := filepath.Join(localPath, key) + if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + return err + } + return os.Rename(filePath, dstPath) + } + + client, err := s.s3ClientForUse() + if err != nil { + return err + } + opts := minio.PutObjectOptions{} + if contentType != "" { + opts.ContentType = contentType + } + _, err = client.FPutObject(ctx, s.Config.Bucket, key, filePath, opts) + return err +} + +func (s *Storage) Provider() string { + if s.storageType() == "s3" { + return "s3" + } + return "local" +} + +func (s *Storage) Bucket() string { + if s.storageType() == "s3" && s.Config.Bucket != "" { + return s.Config.Bucket + } + return "default" +} + +func (s *Storage) storageType() string { + if s.Config == nil { + return "local" + } + typ := strings.TrimSpace(strings.ToLower(s.Config.Type)) + if typ == "" { + return "local" + } + return typ +} + +func (s *Storage) s3ClientForUse() (*minio.Client, error) { + if s.s3Client != nil { + return s.s3Client, nil + } + if strings.TrimSpace(s.Config.Endpoint) == "" { + return nil, fmt.Errorf("storage endpoint is required") + } + if strings.TrimSpace(s.Config.AccessKey) == "" || strings.TrimSpace(s.Config.SecretKey) == "" { + return nil, fmt.Errorf("storage access key or secret key is required") + } + if strings.TrimSpace(s.Config.Bucket) == "" { + return nil, fmt.Errorf("storage bucket is required") + } + + endpoint, secure, err := parseEndpoint(s.Config.Endpoint) + if err != nil { + return nil, err + } + + opts := &minio.Options{ + Creds: credentials.NewStaticV4(s.Config.AccessKey, s.Config.SecretKey, ""), + Secure: secure, + Region: s.Config.Region, + } + if s.Config.PathStyle { + opts.BucketLookup = minio.BucketLookupPath + } else { + opts.BucketLookup = minio.BucketLookupDNS + } + + client, err := minio.New(endpoint, opts) + if err != nil { + return nil, err + } + s.s3Client = client + return client, nil +} + +func parseEndpoint(endpoint string) (string, bool, error) { + if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") { + u, err := url.Parse(endpoint) + if err != nil { + return "", false, err + } + if u.Host == "" { + return "", false, fmt.Errorf("invalid endpoint") + } + return u.Host, u.Scheme == "https", nil + } + return endpoint, false, nil +} diff --git a/docs/storage_provider.md b/docs/storage_provider.md new file mode 100644 index 0000000..7cf36ee --- /dev/null +++ b/docs/storage_provider.md @@ -0,0 +1,32 @@ +# Storage Provider Setup + +This project supports local filesystem storage for development/tests and an S3-compatible provider (AWS S3/MinIO) for production. + +## Local (dev/test) + +Use local storage for tests and quick development. + +```toml +[Storage] +Type = "local" +LocalPath = "./storage" +Secret = "your-storage-secret" +BaseURL = "http://localhost:8080/t//v1/storage" +``` + +Note: `BaseURL` should include the tenant code because storage routes are tenant-scoped. + +## S3/MinIO (example) + +```toml +[Storage] +Type = "s3" +AccessKey = "minioadmin" +SecretKey = "minioadmin" +Region = "us-east-1" +Bucket = "quyun-assets" +Endpoint = "http://127.0.0.1:9000" +PathStyle = true +``` + +For AWS S3, set `Endpoint` with `https://` (e.g., `https://s3.amazonaws.com`) and keep `PathStyle` as `false` unless your provider requires path-style access. diff --git a/frontend/portal/src/api/common.js b/frontend/portal/src/api/common.js index 0389283..ba42538 100644 --- a/frontend/portal/src/api/common.js +++ b/frontend/portal/src/api/common.js @@ -1,4 +1,5 @@ import { request } from '../utils/request'; +import { getTenantCode } from '../utils/tenant'; export const commonApi = { getOptions: () => request('/common/options'), @@ -88,7 +89,12 @@ export const commonApi = { formData.append('type', type); xhr = new XMLHttpRequest(); - xhr.open('POST', '/v1/upload'); + const tenantCode = getTenantCode(); + if (!tenantCode) { + reject(new Error('Tenant code missing in URL')); + return; + } + xhr.open('POST', `/t/${tenantCode}/v1/upload`); const token = localStorage.getItem('token'); if (token) {