feat: add s3 storage provider integration

2026-01-13 16:46:00 +08:00
parent bd8dab5764
commit e97be37b05
9 changed files with 253 additions and 22 deletions
--- a/backend/app/services/common.go
+++ b/backend/app/services/common.go
@@ -349,20 +349,20 @@ func (s *common) CompleteUpload(ctx context.Context, tenantID, userID int64, for
 			Meta:      existing.Meta,
 		}
 	} else {
-		finalPath := filepath.Join(localPath, objectKey)
-		if err := os.MkdirAll(filepath.Dir(finalPath), 0o755); err != nil {
-			return nil, errorx.ErrInternalError.WithCause(err)
+		contentType := strings.TrimSpace(meta.MimeType)
+		if contentType == "" {
+			contentType = "application/octet-stream"
 		}
-		if err := os.Rename(mergedPath, finalPath); err != nil {
-			return nil, errorx.ErrInternalError.WithCause(err)
+		if err := s.storage.PutObject(ctx, objectKey, mergedPath, contentType); err != nil {
+			return nil, errorx.ErrInternalError.WithCause(err).WithMsg("上传存储失败")
 		}
 		asset = &models.MediaAsset{
 			TenantID:  tid,
 			UserID:    userID,
 			Type:      consts.MediaAssetType(meta.Type),
 			Status:    consts.MediaAssetStatusUploaded,
-			Provider:  "local",
-			Bucket:    "default",
+			Provider:  s.storage.Provider(),
+			Bucket:    s.storage.Bucket(),
 			ObjectKey: objectKey,
 			Hash:      hash,
 			Meta: types.NewJSONType(fields.MediaAssetMeta{
@@ -513,12 +513,12 @@ func (s *common) Upload(
 			Meta:      existing.Meta,
 		}
 	} else {
-		dstPath := filepath.Join(localPath, objectKey)
-		if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil {
-			return nil, errorx.ErrInternalError.WithCause(err).WithMsg("failed to create storage directory")
+		contentType := strings.TrimSpace(file.Header.Get("Content-Type"))
+		if contentType == "" {
+			contentType = "application/octet-stream"
 		}
-		if err := os.Rename(tmpPath, dstPath); err != nil {
-			return nil, errorx.ErrInternalError.WithCause(err).WithMsg("failed to finalize file")
+		if err := s.storage.PutObject(ctx, objectKey, tmpPath, contentType); err != nil {
+			return nil, errorx.ErrInternalError.WithCause(err).WithMsg("上传存储失败")
 		}
 		os.RemoveAll(tmpDir)

@@ -528,8 +528,8 @@ func (s *common) Upload(
 			UserID:    userID,
 			Type:      consts.MediaAssetType(typeArg),
 			Status:    consts.MediaAssetStatusUploaded,
-			Provider:  "local",
-			Bucket:    "default",
+			Provider:  s.storage.Provider(),
+			Bucket:    s.storage.Bucket(),
 			ObjectKey: objectKey,
 			Hash:      hash,
 			Meta: types.NewJSONType(fields.MediaAssetMeta{
--- a/backend/config.test.toml
+++ b/backend/config.test.toml
@@ -60,3 +60,9 @@ Type = "local"
 LocalPath = "./storage"
 Secret = "test-storage-secret"
 BaseURL = "/v1/storage"
+AccessKey = ""
+SecretKey = ""
+Region = ""
+Bucket = ""
+Endpoint = ""
+PathStyle = true
--- a/backend/config.toml
+++ b/backend/config.toml
@@ -137,5 +137,17 @@ Type = "local"
 LocalPath = "./storage"
 # 签名密钥
 Secret = "your-storage-secret"
-# 公共访问URL前缀
+# 公共访问URL前缀（本地存储使用）
 BaseURL = "/v1/storage"
+# AccessKey（S3/MinIO）
+AccessKey = ""
+# SecretKey（S3/MinIO）
+SecretKey = ""
+# Region（S3/MinIO）
+Region = ""
+# Bucket（S3/MinIO）
+Bucket = ""
+# Endpoint（S3/MinIO，示例：http://127.0.0.1:9000）
+Endpoint = ""
+# PathStyle（S3/MinIO）
+PathStyle = true
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -39,6 +39,17 @@ require (
 	gorm.io/plugin/dbresolver v1.6.2
 )

+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.11 // indirect
+	github.com/klauspost/crc32 v1.3.0 // indirect
+	github.com/minio/crc64nvme v1.1.0 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go/v7 v7.0.97 // indirect
+	github.com/rs/xid v1.6.0 // indirect
+)
+
 replace google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1

 require (
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -47,6 +47,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
 github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -160,6 +162,11 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
 github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
+github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
+github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -177,6 +184,12 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
 github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
+github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q=
+github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ=
+github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
@@ -220,6 +233,8 @@ github.com/rogeecn/swag v1.0.1 h1:s1yxLgopqO1m8sqGjVmt6ocMBRubMPIh2JtIPG4xjQE=
 github.com/rogeecn/swag v1.0.1/go.mod h1:flG2NXERPxlRl2VdpU2VXTO8iBnQiERyowOXSkZVMOc=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
 github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
--- a/backend/providers/storage/config.go
+++ b/backend/providers/storage/config.go
@@ -5,4 +5,11 @@ type Config struct {
 	LocalPath string // for local
 	Secret    string // for signing
 	BaseURL   string // public url prefix
+	// S3-compatible config
+	AccessKey string
+	SecretKey string
+	Region    string
+	Bucket    string
+	Endpoint  string
+	PathStyle bool
 }
--- a/backend/providers/storage/provider.go
+++ b/backend/providers/storage/provider.go
@@ -1,6 +1,7 @@
 package storage

 import (
+	"context"
 	"crypto/hmac"
 	"crypto/sha256"
 	"encoding/hex"
@@ -9,8 +10,11 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"

+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"go.ipao.vip/atom/container"
 	"go.ipao.vip/atom/opt"
 )
@@ -33,16 +37,23 @@ func Provide(opts ...opt.Option) error {
 		return err
 	}
 	return container.Container.Provide(func() (*Storage, error) {
-		return &Storage{Config: &config}, nil
+		store := &Storage{Config: &config}
+		if store.storageType() == "s3" {
+			if _, err := store.s3ClientForUse(); err != nil {
+				return nil, err
+			}
+		}
+		return store, nil
 	}, o.DiOptions()...)
 }

 type Storage struct {
-	Config *Config
+	Config   *Config
+	s3Client *minio.Client
 }

 func (s *Storage) Delete(key string) error {
-	if s.Config.Type == "local" {
+	if s.storageType() == "local" {
 		localPath := s.Config.LocalPath
 		if localPath == "" {
 			localPath = "./storage"
@@ -50,15 +61,41 @@ func (s *Storage) Delete(key string) error {
 		path := filepath.Join(localPath, key)
 		return os.Remove(path)
 	}
-	// TODO: S3 implementation
-	return nil
+	client, err := s.s3ClientForUse()
+	if err != nil {
+		return err
+	}
+	return client.RemoveObject(context.Background(), s.Config.Bucket, key, minio.RemoveObjectOptions{})
 }

 func (s *Storage) SignURL(method, key string, expires time.Duration) (string, error) {
+	if s.storageType() == "s3" {
+		client, err := s.s3ClientForUse()
+		if err != nil {
+			return "", err
+		}
+		switch strings.ToUpper(method) {
+		case "GET":
+			u, err := client.PresignedGetObject(context.Background(), s.Config.Bucket, key, expires, nil)
+			if err != nil {
+				return "", err
+			}
+			return u.String(), nil
+		case "PUT":
+			u, err := client.PresignedPutObject(context.Background(), s.Config.Bucket, key, expires)
+			if err != nil {
+				return "", err
+			}
+			return u.String(), nil
+		default:
+			return "", fmt.Errorf("unsupported method")
+		}
+	}
+
 	exp := time.Now().Add(expires).Unix()
 	sign := s.signature(method, key, exp)

-	baseURL := s.Config.BaseURL
+	baseURL := strings.TrimRight(s.Config.BaseURL, "/")
 	// Ensure BaseURL doesn't end with slash if we add one
 	// Simplified: assume standard /v1/storage prefix in BaseURL or append it
 	// We'll append /<key>
@@ -77,6 +114,9 @@ func (s *Storage) SignURL(method, key string, expires time.Duration) (string, er
 }

 func (s *Storage) Verify(method, key, expStr, sign string) error {
+	if s.storageType() == "s3" {
+		return fmt.Errorf("s3 storage does not use signed local urls")
+	}
 	exp, err := strconv.ParseInt(expStr, 10, 64)
 	if err != nil {
 		return fmt.Errorf("invalid expiry")
@@ -98,3 +138,105 @@ func (s *Storage) signature(method, key string, exp int64) string {
 	h.Write([]byte(str))
 	return hex.EncodeToString(h.Sum(nil))
 }
+
+func (s *Storage) PutObject(ctx context.Context, key, filePath, contentType string) error {
+	if s.storageType() == "local" {
+		localPath := s.Config.LocalPath
+		if localPath == "" {
+			localPath = "./storage"
+		}
+		dstPath := filepath.Join(localPath, key)
+		if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil {
+			return err
+		}
+		return os.Rename(filePath, dstPath)
+	}
+
+	client, err := s.s3ClientForUse()
+	if err != nil {
+		return err
+	}
+	opts := minio.PutObjectOptions{}
+	if contentType != "" {
+		opts.ContentType = contentType
+	}
+	_, err = client.FPutObject(ctx, s.Config.Bucket, key, filePath, opts)
+	return err
+}
+
+func (s *Storage) Provider() string {
+	if s.storageType() == "s3" {
+		return "s3"
+	}
+	return "local"
+}
+
+func (s *Storage) Bucket() string {
+	if s.storageType() == "s3" && s.Config.Bucket != "" {
+		return s.Config.Bucket
+	}
+	return "default"
+}
+
+func (s *Storage) storageType() string {
+	if s.Config == nil {
+		return "local"
+	}
+	typ := strings.TrimSpace(strings.ToLower(s.Config.Type))
+	if typ == "" {
+		return "local"
+	}
+	return typ
+}
+
+func (s *Storage) s3ClientForUse() (*minio.Client, error) {
+	if s.s3Client != nil {
+		return s.s3Client, nil
+	}
+	if strings.TrimSpace(s.Config.Endpoint) == "" {
+		return nil, fmt.Errorf("storage endpoint is required")
+	}
+	if strings.TrimSpace(s.Config.AccessKey) == "" || strings.TrimSpace(s.Config.SecretKey) == "" {
+		return nil, fmt.Errorf("storage access key or secret key is required")
+	}
+	if strings.TrimSpace(s.Config.Bucket) == "" {
+		return nil, fmt.Errorf("storage bucket is required")
+	}
+
+	endpoint, secure, err := parseEndpoint(s.Config.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := &minio.Options{
+		Creds:  credentials.NewStaticV4(s.Config.AccessKey, s.Config.SecretKey, ""),
+		Secure: secure,
+		Region: s.Config.Region,
+	}
+	if s.Config.PathStyle {
+		opts.BucketLookup = minio.BucketLookupPath
+	} else {
+		opts.BucketLookup = minio.BucketLookupDNS
+	}
+
+	client, err := minio.New(endpoint, opts)
+	if err != nil {
+		return nil, err
+	}
+	s.s3Client = client
+	return client, nil
+}
+
+func parseEndpoint(endpoint string) (string, bool, error) {
+	if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
+		u, err := url.Parse(endpoint)
+		if err != nil {
+			return "", false, err
+		}
+		if u.Host == "" {
+			return "", false, fmt.Errorf("invalid endpoint")
+		}
+		return u.Host, u.Scheme == "https", nil
+	}
+	return endpoint, false, nil
+}
--- a/docs/storage_provider.md
+++ b/docs/storage_provider.md
@@ -0,0 +1,32 @@
+# Storage Provider Setup
+
+This project supports local filesystem storage for development/tests and an S3-compatible provider (AWS S3/MinIO) for production.
+
+## Local (dev/test)
+
+Use local storage for tests and quick development.
+
+```toml
+[Storage]
+Type = "local"
+LocalPath = "./storage"
+Secret = "your-storage-secret"
+BaseURL = "http://localhost:8080/t/<tenantCode>/v1/storage"
+```
+
+Note: `BaseURL` should include the tenant code because storage routes are tenant-scoped.
+
+## S3/MinIO (example)
+
+```toml
+[Storage]
+Type = "s3"
+AccessKey = "minioadmin"
+SecretKey = "minioadmin"
+Region = "us-east-1"
+Bucket = "quyun-assets"
+Endpoint = "http://127.0.0.1:9000"
+PathStyle = true
+```
+
+For AWS S3, set `Endpoint` with `https://` (e.g., `https://s3.amazonaws.com`) and keep `PathStyle` as `false` unless your provider requires path-style access.
--- a/frontend/portal/src/api/common.js
+++ b/frontend/portal/src/api/common.js
@@ -1,4 +1,5 @@
 import { request } from '../utils/request';
+import { getTenantCode } from '../utils/tenant';

 export const commonApi = {
    getOptions: () => request('/common/options'),
@@ -88,7 +89,12 @@ export const commonApi = {
            formData.append('type', type);

            xhr = new XMLHttpRequest();
-            xhr.open('POST', '/v1/upload');
+            const tenantCode = getTenantCode();
+            if (!tenantCode) {
+                reject(new Error('Tenant code missing in URL'));
+                return;
+            }
+            xhr.open('POST', `/t/${tenantCode}/v1/upload`);

            const token = localStorage.getItem('token');
            if (token) {