Files
quyun-v2/backend/app/services/common.go
Rogee 221b068a84 feat: add file deduplication and hash checking for uploads
- Implemented SHA-256 hashing for uploaded files to enable deduplication.
- Added CheckHash method to verify if a file with the same hash already exists.
- Updated Upload method to reuse existing media assets if a duplicate is found.
- Introduced a new hash column in the media_assets table to store file hashes.
- Enhanced the upload process to include progress tracking and hash calculation.
- Modified frontend to check for existing files before uploading and to show upload progress.
- Added vuedraggable for drag-and-drop functionality in the content editing view.
2025-12-31 19:16:02 +08:00

212 lines
6.0 KiB
Go

package services
import (
"context"
"crypto/sha256"
"encoding/hex"
"io"
"mime/multipart"
"os"
"path/filepath"
"time"
"quyun/v2/app/errorx"
common_dto "quyun/v2/app/http/v1/dto"
"quyun/v2/app/requests"
"quyun/v2/database/fields"
"quyun/v2/database/models"
"quyun/v2/pkg/consts"
"quyun/v2/providers/storage"
"github.com/google/uuid"
"github.com/spf13/cast"
"go.ipao.vip/gen/types"
)
// @provider
type common struct {
storage *storage.Storage
}
func (s *common) Options(ctx context.Context) (*common_dto.OptionsResponse, error) {
return &common_dto.OptionsResponse{
ContentStatus: consts.ContentStatusItems(),
ContentGenre: []requests.KV{
requests.NewKV("Jingju", "京剧"),
requests.NewKV("Kunqu", "昆曲"),
requests.NewKV("Yueju", "越剧"),
requests.NewKV("Yuju", "豫剧"),
requests.NewKV("Huangmeixi", "黄梅戏"),
requests.NewKV("Pingju", "评剧"),
requests.NewKV("Qinqiang", "秦腔"),
},
}, nil
}
func (s *common) CheckHash(ctx context.Context, userID int64, hash string) (*common_dto.UploadResult, error) {
existing, err := models.MediaAssetQuery.WithContext(ctx).Where(models.MediaAssetQuery.Hash.Eq(hash)).First()
if err != nil {
return nil, nil // Not found, proceed to upload
}
// Found existing file (Global deduplication hit)
// Check if user already has it (Logic deduplication hit)
myExisting, err := models.MediaAssetQuery.WithContext(ctx).
Where(models.MediaAssetQuery.Hash.Eq(hash), models.MediaAssetQuery.UserID.Eq(userID)).
First()
if err == nil {
return s.composeUploadResult(myExisting), nil
}
// Create new record for this user reusing existing ObjectKey
t, err := models.TenantQuery.WithContext(ctx).Where(models.TenantQuery.UserID.Eq(userID)).First()
var tid int64 = 0
if err == nil {
tid = t.ID
}
asset := &models.MediaAsset{
TenantID: tid,
UserID: userID,
Type: existing.Type,
Status: consts.MediaAssetStatusUploaded,
Provider: existing.Provider,
Bucket: existing.Bucket,
ObjectKey: existing.ObjectKey,
Hash: hash,
Meta: existing.Meta,
}
if err := models.MediaAssetQuery.WithContext(ctx).Create(asset); err != nil {
return nil, errorx.ErrDatabaseError.WithCause(err)
}
return s.composeUploadResult(asset), nil
}
func (s *common) Upload(
ctx context.Context,
userID int64,
file *multipart.FileHeader,
typeArg string,
) (*common_dto.UploadResult, error) { // Mock Upload to S3/MinIO (Here we just generate key, actual upload handling via direct upload or stream is better)
// But this Upload endpoint accepts file. So we save it.
objectKey := uuid.NewString() + "_" + file.Filename
// Save file content to local storage
src, err := file.Open()
if err != nil {
return nil, errorx.ErrInternalError.WithCause(err).WithMsg("failed to open uploaded file")
}
defer src.Close()
localPath := s.storage.Config.LocalPath
if localPath == "" {
localPath = "./storage" // Fallback
}
dstPath := filepath.Join(localPath, objectKey)
if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil {
return nil, errorx.ErrInternalError.WithCause(err).WithMsg("failed to create storage directory")
}
dst, err := os.Create(dstPath)
if err != nil {
return nil, errorx.ErrInternalError.WithCause(err).WithMsg("failed to create destination file")
}
// Hash calculation while copying
hasher := sha256.New()
size, err := io.Copy(io.MultiWriter(dst, hasher), src)
dst.Close() // Close immediately to allow removal if needed
if err != nil {
return nil, errorx.ErrInternalError.WithCause(err).WithMsg("failed to save file content")
}
hash := hex.EncodeToString(hasher.Sum(nil))
t, err := models.TenantQuery.WithContext(ctx).Where(models.TenantQuery.UserID.Eq(userID)).First()
var tid int64 = 0
if err == nil {
tid = t.ID
}
var asset *models.MediaAsset
// Deduplication Check
existing, err := models.MediaAssetQuery.WithContext(ctx).Where(models.MediaAssetQuery.Hash.Eq(hash)).First()
if err == nil {
// Found existing file (Storage Deduplication)
os.Remove(dstPath) // Delete the duplicate we just wrote
// Check if user already has it (Logic Deduplication)
myExisting, err := models.MediaAssetQuery.WithContext(ctx).
Where(models.MediaAssetQuery.Hash.Eq(hash), models.MediaAssetQuery.UserID.Eq(userID)).
First()
if err == nil {
return s.composeUploadResult(myExisting), nil
}
// Create new link for user reusing existing ObjectKey
asset = &models.MediaAsset{
TenantID: tid,
UserID: userID,
Type: consts.MediaAssetType(typeArg),
Status: consts.MediaAssetStatusUploaded,
Provider: existing.Provider,
Bucket: existing.Bucket,
ObjectKey: existing.ObjectKey, // Reuse key
Hash: hash,
Meta: existing.Meta,
}
} else {
// New unique file
asset = &models.MediaAsset{
TenantID: tid,
UserID: userID,
Type: consts.MediaAssetType(typeArg),
Status: consts.MediaAssetStatusUploaded,
Provider: "local",
Bucket: "default",
ObjectKey: objectKey,
Hash: hash,
Meta: types.NewJSONType(fields.MediaAssetMeta{
Size: size,
}),
}
}
if err := models.MediaAssetQuery.WithContext(ctx).Create(asset); err != nil {
return nil, errorx.ErrDatabaseError.WithCause(err)
}
return s.composeUploadResult(asset), nil
}
func (s *common) composeUploadResult(asset *models.MediaAsset) *common_dto.UploadResult {
url := s.GetAssetURL(asset.ObjectKey)
filename := filepath.Base(asset.ObjectKey)
// Try to get original filename if stored? Currently objectKey has UUID prefix.
// We can store original filename in Meta if needed. For now, just return valid result.
// Meta is JSONType wrapper.
size := asset.Meta.Data().Size
return &common_dto.UploadResult{
ID: cast.ToString(asset.ID),
URL: url,
Filename: filename,
Size: size,
MimeType: "application/octet-stream", // TODO: Store mime type in DB
}
}
func (s *common) GetAssetURL(objectKey string) string {
if objectKey == "" {
return ""
}
url, _ := s.storage.SignURL("GET", objectKey, 1*time.Hour)
return url
}