diff --git a/backend/app/http/v1/common.go b/backend/app/http/v1/common.go index d3d3006..41f0d4b 100644 --- a/backend/app/http/v1/common.go +++ b/backend/app/http/v1/common.go @@ -66,3 +66,77 @@ func (c *Common) GetOptions(ctx fiber.Ctx) (*dto.OptionsResponse, error) { func (c *Common) CheckHash(ctx fiber.Ctx, user *models.User, hash string) (*dto.UploadResult, error) { return services.Common.CheckHash(ctx, user.ID, hash) } + +// @Router /v1/upload/init [post] +// @Summary Init multipart upload +// @Description Initialize multipart upload +// @Tags Common +// @Accept json +// @Produce json +// @Param form body dto.UploadInitForm true "Init form" +// @Success 200 {object} dto.UploadInitResponse +// @Bind user local key(__ctx_user) +// @Bind form body +func (c *Common) InitUpload(ctx fiber.Ctx, user *models.User, form *dto.UploadInitForm) (*dto.UploadInitResponse, error) { + return services.Common.InitUpload(ctx.Context(), user.ID, form) +} + +// @Router /v1/upload/part [post] +// @Summary Upload part +// @Description Upload a part +// @Tags Common +// @Accept multipart/form-data +// @Produce json +// @Param file formData file true "Part File" +// @Param form formData dto.UploadPartForm true "Part form" +// @Success 200 {string} string "OK" +// @Bind user local key(__ctx_user) +// @Bind file file +// @Bind form body +func (c *Common) UploadPart(ctx fiber.Ctx, user *models.User, file *multipart.FileHeader, form *dto.UploadPartForm) error { + return services.Common.UploadPart(ctx.Context(), user.ID, file, form) +} + +// @Router /v1/upload/complete [post] +// @Summary Complete upload +// @Description Complete multipart upload +// @Tags Common +// @Accept json +// @Produce json +// @Param form body dto.UploadCompleteForm true "Complete form" +// @Success 200 {object} dto.UploadResult +// @Bind user local key(__ctx_user) +// @Bind form body +func (c *Common) CompleteUpload(ctx fiber.Ctx, user *models.User, form *dto.UploadCompleteForm) (*dto.UploadResult, error) { + return services.Common.CompleteUpload(ctx.Context(), user.ID, form) +} + +// @Router /v1/upload/:uploadId [delete] +// @Summary Abort upload +// @Description Abort multipart upload +// @Tags Common +// @Accept json +// @Produce json +// @Param uploadId path string true "Upload ID" +// @Success 200 {string} string "OK" +// @Bind user local key(__ctx_user) +// @Bind uploadId path +func (c *Common) AbortUpload(ctx fiber.Ctx, user *models.User, uploadId string) error { + return services.Common.AbortUpload(ctx.Context(), user.ID, uploadId) +} + +// @Router /v1/media-assets/:id [delete] +// @Summary Delete media asset +// @Description Delete media asset +// @Tags Common +// @Accept json +// @Produce json +// @Param id path string true "Asset ID" +// @Success 200 {string} string "OK" +// @Bind user local key(__ctx_user) +// @Bind id path +func (c *Common) DeleteMediaAsset(ctx fiber.Ctx, user *models.User, id string) error { + return services.Common.DeleteMediaAsset(ctx.Context(), user.ID, id) +} + +// Upload file diff --git a/backend/app/http/v1/dto/upload.go b/backend/app/http/v1/dto/upload.go new file mode 100644 index 0000000..95ba0c1 --- /dev/null +++ b/backend/app/http/v1/dto/upload.go @@ -0,0 +1,24 @@ +package dto + +type UploadInitForm struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + Filename string `json:"filename"` + MimeType string `json:"mime_type"` + Type string `json:"type"` +} + +type UploadInitResponse struct { + UploadID string `json:"upload_id"` + Key string `json:"key"` // For S3 direct + ChunkSize int64 `json:"chunk_size"` +} + +type UploadPartForm struct { + UploadID string `form:"upload_id"` + PartNumber int `form:"part_number"` +} + +type UploadCompleteForm struct { + UploadID string `json:"upload_id"` +} diff --git a/backend/app/http/v1/routes.gen.go b/backend/app/http/v1/routes.gen.go index e7d67e5..5a37540 100644 --- a/backend/app/http/v1/routes.gen.go +++ b/backend/app/http/v1/routes.gen.go @@ -60,6 +60,37 @@ func (r *Routes) Register(router fiber.Router) { Local[*models.User]("__ctx_user"), QueryParam[string]("hash"), )) + r.log.Debugf("Registering route: Post /v1/upload/init -> common.InitUpload") + router.Post("/v1/upload/init"[len(r.Path()):], DataFunc2( + r.common.InitUpload, + Local[*models.User]("__ctx_user"), + Body[dto.UploadInitForm]("form"), + )) + r.log.Debugf("Registering route: Post /v1/upload/part -> common.UploadPart") + router.Post("/v1/upload/part"[len(r.Path()):], Func3( + r.common.UploadPart, + Local[*models.User]("__ctx_user"), + File[multipart.FileHeader]("file"), + Body[dto.UploadPartForm]("form"), + )) + r.log.Debugf("Registering route: Post /v1/upload/complete -> common.CompleteUpload") + router.Post("/v1/upload/complete"[len(r.Path()):], DataFunc2( + r.common.CompleteUpload, + Local[*models.User]("__ctx_user"), + Body[dto.UploadCompleteForm]("form"), + )) + r.log.Debugf("Registering route: Delete /v1/upload/:uploadId -> common.AbortUpload") + router.Delete("/v1/upload/:uploadId"[len(r.Path()):], Func2( + r.common.AbortUpload, + Local[*models.User]("__ctx_user"), + PathParam[string]("uploadId"), + )) + r.log.Debugf("Registering route: Delete /v1/media-assets/:id -> common.DeleteMediaAsset") + router.Delete("/v1/media-assets/:id"[len(r.Path()):], Func2( + r.common.DeleteMediaAsset, + Local[*models.User]("__ctx_user"), + PathParam[string]("id"), + )) r.log.Debugf("Registering route: Post /v1/upload -> common.Upload") router.Post("/v1/upload"[len(r.Path()):], DataFunc3( r.common.Upload, diff --git a/backend/app/services/common.go b/backend/app/services/common.go index ad79f62..604f428 100644 --- a/backend/app/services/common.go +++ b/backend/app/services/common.go @@ -4,10 +4,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "io" "mime/multipart" "os" "path/filepath" + "sort" + "strconv" "time" "quyun/v2/app/errorx" @@ -85,12 +88,220 @@ func (s *common) CheckHash(ctx context.Context, userID int64, hash string) (*com return s.composeUploadResult(asset), nil } +type UploadMeta struct { + Filename string + Type string + MimeType string +} + +func (s *common) InitUpload(ctx context.Context, userID int64, form *common_dto.UploadInitForm) (*common_dto.UploadInitResponse, error) { + uploadID := uuid.NewString() + localPath := s.storage.Config.LocalPath + if localPath == "" { + localPath = "./storage" + } + tempDir := filepath.Join(localPath, "temp", uploadID) + if err := os.MkdirAll(tempDir, 0755); err != nil { + return nil, errorx.ErrInternalError.WithCause(err) + } + + // Save metadata + meta := UploadMeta{ + Filename: form.Filename, + Type: form.Type, // Ensure form has Type + MimeType: form.MimeType, + } + metaFile, _ := os.Create(filepath.Join(tempDir, "meta.json")) + json.NewEncoder(metaFile).Encode(meta) + metaFile.Close() + + return &common_dto.UploadInitResponse{ + UploadID: uploadID, + ChunkSize: 5 * 1024 * 1024, + }, nil +} + +func (s *common) UploadPart(ctx context.Context, userID int64, file *multipart.FileHeader, form *common_dto.UploadPartForm) error { + localPath := s.storage.Config.LocalPath + if localPath == "" { + localPath = "./storage" + } + partPath := filepath.Join(localPath, "temp", form.UploadID, strconv.Itoa(form.PartNumber)) + + src, err := file.Open() + if err != nil { + return errorx.ErrInternalError.WithCause(err) + } + defer src.Close() + + dst, err := os.Create(partPath) + if err != nil { + return errorx.ErrInternalError.WithCause(err) + } + defer dst.Close() + + if _, err = io.Copy(dst, src); err != nil { + return errorx.ErrInternalError.WithCause(err) + } + return nil +} + +func (s *common) CompleteUpload(ctx context.Context, userID int64, form *common_dto.UploadCompleteForm) (*common_dto.UploadResult, error) { + localPath := s.storage.Config.LocalPath + if localPath == "" { + localPath = "./storage" + } + tempDir := filepath.Join(localPath, "temp", form.UploadID) + + // Read Meta + var meta UploadMeta + metaFile, err := os.Open(filepath.Join(tempDir, "meta.json")) + if err != nil { + return nil, errorx.ErrRecordNotFound.WithMsg("Upload session expired or invalid") + } + json.NewDecoder(metaFile).Decode(&meta) + metaFile.Close() + + // List parts + entries, err := os.ReadDir(tempDir) + if err != nil { + return nil, errorx.ErrInternalError.WithCause(err) + } + + var parts []int + for _, e := range entries { + if !e.IsDir() && e.Name() != "meta.json" { + if i, err := strconv.Atoi(e.Name()); err == nil { + parts = append(parts, i) + } + } + } + sort.Ints(parts) + + objectKey := uuid.NewString() + "_" + meta.Filename + dstPath := filepath.Join(localPath, objectKey) + dst, err := os.Create(dstPath) + if err != nil { + return nil, errorx.ErrInternalError.WithCause(err) + } + defer dst.Close() + + hasher := sha256.New() + var totalSize int64 + + for _, partNum := range parts { + partPath := filepath.Join(tempDir, strconv.Itoa(partNum)) + src, err := os.Open(partPath) + if err != nil { + return nil, errorx.ErrInternalError.WithCause(err) + } + + n, err := io.Copy(io.MultiWriter(dst, hasher), src) + src.Close() + if err != nil { + return nil, errorx.ErrInternalError.WithCause(err) + } + totalSize += n + } + + hash := hex.EncodeToString(hasher.Sum(nil)) + dst.Close(); // Ensure flush before potential removal + os.RemoveAll(tempDir) + + // Deduplication Logic (Similar to Upload) + t, err := models.TenantQuery.WithContext(ctx).Where(models.TenantQuery.UserID.Eq(userID)).First() + var tid int64 = 0 + if err == nil { + tid = t.ID + } + + existing, err := models.MediaAssetQuery.WithContext(ctx).Where(models.MediaAssetQuery.Hash.Eq(hash)).First() + var asset *models.MediaAsset + + if err == nil { + os.Remove(dstPath) // Delete duplicate + myExisting, err := models.MediaAssetQuery.WithContext(ctx). + Where(models.MediaAssetQuery.Hash.Eq(hash), models.MediaAssetQuery.UserID.Eq(userID)). + First() + if err == nil { + return s.composeUploadResult(myExisting), nil + } + asset = &models.MediaAsset{ + TenantID: tid, + UserID: userID, + Type: consts.MediaAssetType(meta.Type), + Status: consts.MediaAssetStatusUploaded, + Provider: existing.Provider, + Bucket: existing.Bucket, + ObjectKey: existing.ObjectKey, + Hash: hash, + Meta: existing.Meta, + } + } else { + asset = &models.MediaAsset{ + TenantID: tid, + UserID: userID, + Type: consts.MediaAssetType(meta.Type), + Status: consts.MediaAssetStatusUploaded, + Provider: "local", + Bucket: "default", + ObjectKey: objectKey, + Hash: hash, + Meta: types.NewJSONType(fields.MediaAssetMeta{ + Size: totalSize, + }), + } + } + + if err := models.MediaAssetQuery.WithContext(ctx).Create(asset); err != nil { + return nil, errorx.ErrDatabaseError.WithCause(err) + } + + return s.composeUploadResult(asset), nil +} + +func (s *common) DeleteMediaAsset(ctx context.Context, userID int64, id string) error { + aid := cast.ToInt64(id) + asset, err := models.MediaAssetQuery.WithContext(ctx). + Where(models.MediaAssetQuery.ID.Eq(aid), models.MediaAssetQuery.UserID.Eq(userID)). + First() + if err != nil { + return errorx.ErrRecordNotFound + } + + // Delete DB record + if _, err := models.MediaAssetQuery.WithContext(ctx).Where(models.MediaAssetQuery.ID.Eq(aid)).Delete(); err != nil { + return errorx.ErrDatabaseError.WithCause(err) + } + + // Check ref count + count, _ := models.MediaAssetQuery.WithContext(ctx). + Where(models.MediaAssetQuery.ObjectKey.Eq(asset.ObjectKey)). + Count() + + if count == 0 { + // Physical delete + _ = s.storage.Delete(asset.ObjectKey) + } + + return nil +} + +func (s *common) AbortUpload(ctx context.Context, userID int64, uploadId string) error { + localPath := s.storage.Config.LocalPath + if localPath == "" { + localPath = "./storage" + } + tempDir := filepath.Join(localPath, "temp", uploadId) + return os.RemoveAll(tempDir) +} + func (s *common) Upload( ctx context.Context, userID int64, file *multipart.FileHeader, typeArg string, -) (*common_dto.UploadResult, error) { // Mock Upload to S3/MinIO (Here we just generate key, actual upload handling via direct upload or stream is better) +) (*common_dto.UploadResult, error) { // But this Upload endpoint accepts file. So we save it. objectKey := uuid.NewString() + "_" + file.Filename diff --git a/backend/providers/storage/provider.go b/backend/providers/storage/provider.go index 5a89b26..e67a3f3 100644 --- a/backend/providers/storage/provider.go +++ b/backend/providers/storage/provider.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "fmt" "net/url" + "os" + "path/filepath" "strconv" "time" @@ -39,6 +41,19 @@ type Storage struct { Config *Config } +func (s *Storage) Delete(key string) error { + if s.Config.Type == "local" { + localPath := s.Config.LocalPath + if localPath == "" { + localPath = "./storage" + } + path := filepath.Join(localPath, key) + return os.Remove(path) + } + // TODO: S3 implementation + return nil +} + func (s *Storage) SignURL(method, key string, expires time.Duration) (string, error) { exp := time.Now().Add(expires).Unix() sign := s.signature(method, key, exp) diff --git a/frontend/portal/package-lock.json b/frontend/portal/package-lock.json index c8c9014..2760be3 100644 --- a/frontend/portal/package-lock.json +++ b/frontend/portal/package-lock.json @@ -11,6 +11,7 @@ "@primevue/themes": "^4.5.4", "@tailwindcss/vite": "^4.1.18", "dayjs": "^1.11.19", + "js-sha256": "^0.11.1", "pinia": "^3.0.4", "primeicons": "^7.0.0", "primevue": "^4.5.4", @@ -1890,6 +1891,12 @@ "jiti": "lib/jiti-cli.mjs" } }, + "node_modules/js-sha256": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.11.1.tgz", + "integrity": "sha512-o6WSo/LUvY2uC4j7mO50a2ms7E/EAdbP0swigLV+nzHKTTaYnaLIWJ02VdXrsJX0vGedDESQnLsOekr94ryfjg==", + "license": "MIT" + }, "node_modules/lightningcss": { "version": "1.30.2", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.30.2.tgz", diff --git a/frontend/portal/package.json b/frontend/portal/package.json index b953ab2..346b29c 100644 --- a/frontend/portal/package.json +++ b/frontend/portal/package.json @@ -12,6 +12,7 @@ "@primevue/themes": "^4.5.4", "@tailwindcss/vite": "^4.1.18", "dayjs": "^1.11.19", + "js-sha256": "^0.11.1", "pinia": "^3.0.4", "primeicons": "^7.0.0", "primevue": "^4.5.4", diff --git a/frontend/portal/src/api/common.js b/frontend/portal/src/api/common.js index 7105244..951e31a 100644 --- a/frontend/portal/src/api/common.js +++ b/frontend/portal/src/api/common.js @@ -3,12 +3,62 @@ import { request } from '../utils/request'; export const commonApi = { getOptions: () => request('/common/options'), checkHash: (hash) => request(`/upload/check?hash=${hash}`), + deleteMedia: (id) => request(`/media-assets/${id}`, { method: 'DELETE' }), upload: (file, type) => { const formData = new FormData(); formData.append('file', file); formData.append('type', type); return request('/upload', { method: 'POST', body: formData }); }, + uploadMultipart: async (file, hash, onProgress) => { + // 1. Check Hash + try { + const res = await commonApi.checkHash(hash); + if (res) { + if (onProgress) onProgress(100); + return res; + } + } catch(e) {} + + // 2. Init + const initRes = await request('/upload/init', { + method: 'POST', + body: { + filename: file.name, + size: file.size, + mime_type: file.type, + hash: hash + } + }); + + const { upload_id, chunk_size } = initRes; + const totalChunks = Math.ceil(file.size / chunk_size); + + // 3. Upload Parts + for (let i = 0; i < totalChunks; i++) { + const start = i * chunk_size; + const end = Math.min(start + chunk_size, file.size); + const chunk = file.slice(start, end); + + const formData = new FormData(); + formData.append('file', chunk); + formData.append('upload_id', upload_id); + formData.append('part_number', i + 1); + + await request('/upload/part', { method: 'POST', body: formData }); + + if (onProgress) { + const percent = Math.round(((i + 1) / totalChunks) * 100); + onProgress(percent); + } + } + + // 4. Complete + return request('/upload/complete', { + method: 'POST', + body: { upload_id } + }); + }, uploadWithProgress: (file, type, onProgress) => { return new Promise((resolve, reject) => { const formData = new FormData(); diff --git a/frontend/portal/src/views/creator/ContentsEditView.vue b/frontend/portal/src/views/creator/ContentsEditView.vue index 1df48f6..1768ec3 100644 --- a/frontend/portal/src/views/creator/ContentsEditView.vue +++ b/frontend/portal/src/views/creator/ContentsEditView.vue @@ -102,7 +102,7 @@