feat: chunk uploads

This commit is contained in:
Rogee
2025-01-15 00:26:10 +08:00
parent 591771ce77
commit 9bfdf0e0ea
16 changed files with 451 additions and 40 deletions

View File

@@ -1,10 +1,25 @@
package medias
import (
"github.com/gofiber/fiber"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"mime/multipart"
"os"
"path/filepath"
"time"
"github.com/gofiber/fiber/v3"
log "github.com/sirupsen/logrus"
)
const (
uploadTempDir = "./temp/chunks" // 临时分片目录
uploadStorageDir = "./uploads" // 最终文件存储目录
)
// @provider
type Controller struct {
svc *Service
@@ -19,6 +34,117 @@ func (ctl *Controller) Prepare() error {
// Upload
// @Router /api/v1/medias/upload [post]
// @Bind req body
func (ctl *Controller) Upload(ctx fiber.Ctx, req *UploadReq) (*UploadResp, error) {
return ctl.svc.Upload(ctx.Context(), req)
// @Bind file file
func (ctl *Controller) Upload(ctx fiber.Ctx, file *multipart.FileHeader, req *UploadReq) (*UploadResp, error) {
// 使用MD5创建唯一的临时目录
tempDir := filepath.Join(uploadTempDir, req.FileMD5)
if err := os.MkdirAll(tempDir, 0o755); err != nil {
return nil, err
}
chunkPath := filepath.Join(tempDir, fmt.Sprintf("chunk_%d", req.ChunkNumber))
if err := ctx.SaveFile(file, chunkPath); err != nil {
return nil, err
}
// 如果是最后一个分片
if req.ChunkNumber == req.TotalChunks-1 {
// 生成唯一的文件存储路径
ext := filepath.Ext(req.FileName)
storageDir := filepath.Join(uploadStorageDir, time.Now().Format("2006/01/02"))
if err := os.MkdirAll(storageDir, 0o755); err != nil {
os.RemoveAll(tempDir)
return nil, err
}
finalPath := filepath.Join(storageDir, req.FileMD5+ext)
// 计算所有分片的实际大小总和
totalSize, err := calculateTotalSize(tempDir, req.TotalChunks)
if err != nil {
os.RemoveAll(tempDir)
return nil, fmt.Errorf("计算文件大小失败: %w", err)
}
// 合并文件
if err := combineChunks(tempDir, finalPath, req.TotalChunks); err != nil {
os.RemoveAll(tempDir)
return nil, fmt.Errorf("合并文件失败: %w", err)
}
// 验证MD5
calculatedMD5, err := calculateFileMD5(finalPath)
if err != nil || calculatedMD5 != req.FileMD5 {
os.RemoveAll(tempDir)
os.Remove(finalPath)
return nil, errors.New("文件MD5验证失败")
}
// 清理临时目录
os.RemoveAll(tempDir)
return &UploadResp{
Files: []UploadFile{
{
HashID: calculatedMD5,
Name: req.FileName,
Path: finalPath,
Size: totalSize,
MimeType: file.Header.Get("Content-Type"),
},
},
}, nil
}
return &UploadResp{}, nil
}
// 计算所有分片的实际大小总和
func calculateTotalSize(tempDir string, totalChunks int) (int64, error) {
var totalSize int64
for i := 0; i < totalChunks; i++ {
chunkPath := filepath.Join(tempDir, fmt.Sprintf("chunk_%d", i))
info, err := os.Stat(chunkPath)
if err != nil {
return 0, err
}
totalSize += info.Size()
}
return totalSize, nil
}
func combineChunks(tempDir, finalPath string, totalChunks int) error {
finalFile, err := os.Create(finalPath)
if err != nil {
return err
}
defer finalFile.Close()
for i := 0; i < totalChunks; i++ {
chunkPath := fmt.Sprintf("%s/chunk_%d", tempDir, i)
chunk, err := os.ReadFile(chunkPath)
if err != nil {
return err
}
if _, err := finalFile.Write(chunk); err != nil {
return err
}
}
return nil
}
func calculateFileMD5(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()
hash := md5.New()
if _, err := io.Copy(hash, file); err != nil {
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}

View File

@@ -1,7 +1,10 @@
package medias
type UploadReq struct {
Files []string `json:"files"`
FileName string `form:"file_name"`
ChunkNumber int `form:"chunk_number"`
TotalChunks int `form:"total_chunks"`
FileMD5 string `form:"file_md5"`
}
type UploadResp struct {

View File

@@ -0,0 +1,56 @@
package medias
import (
"database/sql"
"git.ipao.vip/rogeecn/atom"
"git.ipao.vip/rogeecn/atom/container"
"git.ipao.vip/rogeecn/atom/contracts"
"git.ipao.vip/rogeecn/atom/utils/opt"
)
func Provide(opts ...opt.Option) error {
if err := container.Container.Provide(func(
svc *Service,
) (*Controller, error) {
obj := &Controller{
svc: svc,
}
if err := obj.Prepare(); err != nil {
return nil, err
}
return obj, nil
}); err != nil {
return err
}
if err := container.Container.Provide(func(
controller *Controller,
) (contracts.HttpRoute, error) {
obj := &Routes{
controller: controller,
}
if err := obj.Prepare(); err != nil {
return nil, err
}
return obj, nil
}, atom.GroupRoutes); err != nil {
return err
}
if err := container.Container.Provide(func(
db *sql.DB,
) (*Service, error) {
obj := &Service{
db: db,
}
if err := obj.Prepare(); err != nil {
return nil, err
}
return obj, nil
}); err != nil {
return err
}
return nil
}

View File

@@ -0,0 +1,37 @@
// Code generated by the atomctl ; DO NOT EDIT.
package medias
import (
. "backend/pkg/f"
_ "git.ipao.vip/rogeecn/atom"
_ "git.ipao.vip/rogeecn/atom/contracts"
"github.com/gofiber/fiber/v3"
log "github.com/sirupsen/logrus"
)
// @provider contracts.HttpRoute atom.GroupRoutes
type Routes struct {
log *log.Entry `inject:"false"`
controller *Controller
}
func (r *Routes) Prepare() error {
r.log = log.WithField("module", "routes.medias")
return nil
}
func (r *Routes) Name() string {
return "medias"
}
func (r *Routes) Register(router fiber.Router) {
// 注册路由组: Controller
router.Post("/api/v1/medias/upload", DataFunc2(
r.controller.Upload,
File("file"),
Body[UploadReq]("req"),
))
}