feat: update discover medias

backend/common/media_store/store.go

@@ -9,12 +9,12 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-type Store []UUIDMap
+type Store []VideoInfo
 
-type UUIDMap struct {
+type VideoInfo struct {
-	UUID  string
+	Hash     string
-	Name  string
+	Name     string
-	Price uint
+	Duration uint
 }
 
 func NewStore(path string) (Store, error) {
@@ -56,17 +56,17 @@ func (s Store) Save(path string) error {
 	return nil
 }
 
-func (s Store) Add(uuid, name string, price uint) {
+func (s Store) Append(info VideoInfo) Store {
-	s = append(s, UUIDMap{UUID: uuid, Name: name, Price: price})
+	return append(s, info)
 }
 
-func (s Store) UUIDs() []string {
+func (s Store) Hashes() []string {
-	var uuids []string
+	var hashes []string
 	for _, m := range s {
-		uuids = append(uuids, m.UUID)
+		hashes = append(hashes, m.Hash)
 	}
 
-	return uuids
+	return hashes
 }
 
 // Exists

backend/config.toml

@@ -29,4 +29,4 @@ Salt = "LiXi.Y@140202"
 
 [Storage]
 Type = "local"
-Path = "/projects/mp-qvyun/backend/fixtures/medias"
+Path = "/mnt/yangpingliang/publish/processed"

backend/fixtures/processed/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

backend/modules/tasks/discover/discover_medias.go

@@ -2,7 +2,9 @@ package discover
 
 import (
 	"bytes"
+	"crypto/md5"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,7 +15,6 @@ import (
 	"backend/modules/medias"
 	"backend/pkg/path"
 
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cast"
@@ -37,94 +38,116 @@ func (d *DiscoverMedias) RunE(from, to string) error {
 		return errors.New("from or to is empty")
 	}
 
+	videos, err := d.globVideos(from)
+	if err != nil {
+		return errors.Wrapf(err, "glob videos: %s", from)
+	}
+
 	store, err := media_store.NewStore(to)
 	if err != nil {
 		return errors.Wrapf(err, "new store: %s", to)
 	}
+
 	defer func() {
 		d.runCleanup(to)
 	}()
 
-	dirs, err := path.GetSubDirs(from)
+	for _, video := range videos {
-	if err != nil {
+		md5, err := d.getFileMD5(video)
-		return errors.Wrapf(err, "get sub dirs: %s", from)
+		if err != nil {
-	}
+			return errors.Wrapf(err, "get file md5: %s", video)
+		}
 
-	for _, dir := range dirs {
+		if store.Exists(md5) {
-		d.log.Infof("Discover medias in: %s", dir)
-
-		if exists := store.Exists(dir); exists {
-			d.log.Infof("Skip dir: %s", dir)
 			continue
 		}
 
-		dirPath := filepath.Join(from, dir)
+		info, err := d.processVideo(video, filepath.Join(to, md5))
-
-		uuid := uuid.New().String()
-		to := filepath.Join(to, uuid)
-
-		videos := d.globVideo(dirPath)
-		if len(videos) > 0 {
-			video := videos[0]
-			d.log.Infof("video: %s", video)
-			if err := d.ffmpegVideoToM3U8(video, to); err != nil {
-				return errors.Wrapf(err, "ffmpeg video to m3u8: %s", video)
-			}
-
-			d.log.Info("snapshot", video)
-			posterPath := filepath.Join(to, "poster.jpg")
-			if err := d.ffmpegVideoToPoster(video, posterPath, 5); err != nil {
-				return errors.Wrapf(err, "ffmpeg snapshot : %s", video)
-			}
-		}
-
-		audios := d.globAudio(dirPath)
-		if len(audios) > 0 {
-			audio := audios[0]
-			d.log.Infof("audio: %s", audio)
-			if err := d.ffmpegAudioToM3U8(audio, to); err != nil {
-				return errors.Wrapf(err, "ffmpeg audio to m3u8: %s", audio)
-			}
-		}
-
-		price, err := d.getPrice(dirPath)
 		if err != nil {
-			return errors.Wrapf(err, "get price: %s", dirPath)
+			return errors.Wrapf(err, "process video: %s", video)
 		}
+		info.Hash = md5
+		info.Name = filepath.Base(video)
 
-		store.Add(uuid, dir, price)
+		store = store.Append(info)
+		d.log.Infof("store: %+v", store)
 		if err := store.Save(to); err != nil {
-			return errors.Wrapf(err, "store save: %s", to)
+			return errors.Wrapf(err, "save store: %s", to)
 		}
 	}
 
 	return nil
 }
 
-func (d *DiscoverMedias) globVideo(root string) []string {
+func (d *DiscoverMedias) processVideo(video string, to string) (media_store.VideoInfo, error) {
-	files := []string{}
+	var info media_store.VideoInfo
 
-	exts := []string{"*.mp4", "*.MP4", "*.mov", "*.MOV"}
+	if err := d.ensureDirectory(to); err != nil {
-
+		return info, errors.Wrapf(err, "ensure directory: %s", to)
-	for _, ext := range exts {
-		fs, _ := filepath.Glob(filepath.Join(root, ext))
-		files = append(files, fs...)
 	}
 
-	return files
+	// extract audio from video
+	audioFile := filepath.Join(to, "audio.mp3")
+	if err := d.extractAudio(video, audioFile); err != nil {
+		return info, errors.Wrapf(err, "extract audio %s from %s", audioFile, video)
+	}
+	defer os.Remove(audioFile)
+
+	// ffmpeg video to m3u8
+	if err := d.ffmpegVideoToM3U8(video, to); err != nil {
+		return info, errors.Wrapf(err, "ffmpeg video to m3u8: %s", video)
+	}
+
+	// ffmpeg audio to m3u8
+	if err := d.ffmpegAudioToM3U8(audioFile, to); err != nil {
+		return info, errors.Wrapf(err, "ffmpeg audio to m3u8: %s", audioFile)
+	}
+
+	// get media duration
+	duration, err := d.getMediaDuration(video)
+	if err != nil {
+		return info, errors.Wrapf(err, "get media duration: %s", video)
+	}
+	info.Duration = uint(duration)
+
+	return info, nil
 }
 
-func (d *DiscoverMedias) globAudio(root string) []string {
+func (d *DiscoverMedias) getFileMD5(filePath string) (string, error) {
-	files := []string{}
+	file, err := os.Open(filePath)
+	if err != nil {
+		return "", errors.Wrapf(err, "open file: %s", filePath)
+	}
+	defer file.Close()
 
-	exts := []string{"*.mp3", "*.MP3", "*.wav", "*.WAV"}
+	hash := md5.New()
-
+	if _, err := io.Copy(hash, file); err != nil {
-	for _, ext := range exts {
+		return "", errors.Wrapf(err, "copy file to hash: %s", filePath)
-		fs, _ := filepath.Glob(filepath.Join(root, ext))
-		files = append(files, fs...)
 	}
 
-	return files
+	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+}
+
+// extractAudio extract audio from video
+func (d *DiscoverMedias) extractAudio(video string, output string) error {
+	args := []string{
+		"-i", video,
+		"-vn",
+		"-acodec", "libmp3lame",
+		"-ar", "44100",
+		"-b:a", "128k",
+		"-q:a", "2",
+		output,
+	}
+
+	d.log.Infof("cmd: ffmpeg %s", strings.Join(args, " "))
+	logs, err := exec.Command("ffmpeg", args...).CombinedOutput()
+	if err != nil {
+		return errors.Wrapf(err, "extract audio: %s", video)
+	}
+	d.log.Infof("extract audio: %s", logs)
+
+	return nil
 }
 
 // ensureDirectory ensure the directory exists
@@ -159,6 +182,7 @@ func (d *DiscoverMedias) ffmpegVideoToM3U8(input string, output string) error {
 		"-c:a", "aac",
 		"-strict",
 		"-2",
+		"-vf", "scale=-720:576",
 		"-f", "hls",
 		"-hls_time", "10",
 		"-hls_list_size", "0",
@@ -293,3 +317,9 @@ func (d *DiscoverMedias) getMediaDuration(file string) (float64, error) {
 
 	return duration, nil
 }
+
+// getMedias get the medias in the directory
+func (d *DiscoverMedias) globVideos(dir string) ([]string, error) {
+	// glob *.mp4 in dir
+	return filepath.Glob(filepath.Join(dir, "*.mp4"))
+}

backend/modules/tasks/discover/discover_medias_test.go

@@ -36,20 +36,6 @@ func Test_DiscoverMedias(t *testing.T) {
 	})
 }
 
-func (t *DiscoverMediasTestSuite) Test_getVideo() {
-	Convey("Test_getVideo", t.T(), func() {
-		dirs := t.Svc.globVideo("/mnt/yangpingliang/动态曲谱/河北梆子伴奏《李慧娘》依然还我生前摸样")
-		t.T().Logf("Dirs: %+v", dirs)
-	})
-}
-
-func (t *DiscoverMediasTestSuite) Test_getAudio() {
-	Convey("Test_getAudio", t.T(), func() {
-		dirs := t.Svc.globAudio("/mnt/yangpingliang/动态曲谱/河北梆子伴奏《李慧娘》依然还我生前摸样")
-		t.T().Logf("Dirs: %+v", dirs)
-	})
-}
-
 func (t *DiscoverMediasTestSuite) Test_ffmpegVideoToM3U8() {
 	Convey("Test_ffmpegVideoToM3U8", t.T(), func() {
 		output := "/projects/mp-qvyun/backend/fixtures/medias/abc"
@@ -82,3 +68,14 @@ func (t *DiscoverMediasTestSuite) TestDiscoverMedias_getMediaDuration() {
 		t.T().Logf("Duration: %0.8f", duration)
 	})
 }
+
+func (t *DiscoverMediasTestSuite) Test_globMedias() {
+	Convey("Test_globMedias", t.T(), func() {
+		f, err := t.Svc.globVideos("/mnt/yangpingliang/publish/processed")
+		So(err, ShouldBeNil)
+
+		for _, item := range f {
+			t.T().Logf("Item: %s", item)
+		}
+	})
+}

backend/modules/tasks/store/store_medias.go

@@ -1,14 +1,10 @@
 package store
 
 import (
-	"context"
-
 	"backend/common/media_store"
 	"backend/modules/medias"
 
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
-	"github.com/samber/lo"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -31,40 +27,7 @@ func (d *StoreMedias) RunE(targetPath string) error {
 	if err != nil {
 		return errors.Wrapf(err, "new store: %s", targetPath)
 	}
-
+	_ = store
-	// uuids := lo.FilterMap(store.UUIDs(), func(item string, _ int) (uuid.UUID, bool) {
-	// 	u, err := uuid.FromBytes([]byte(item))
-	// 	if err != nil {
-	// 		return uuid.Nil, false
-	// 	}
-	// 	return u, true
-	// })
-
-	if err := d.mediasSvc.UnPublishTenantWithNotInUUIDs(context.Background(), 1, store.UUIDs()); err != nil {
-		return errors.Wrapf(err, "UnPublishTenantWithNotInUUIDs: %+v", store.UUIDs())
-	}
-
-	dbUUIDs, err := d.mediasSvc.GetTenantUUIDs(context.Background(), 1)
-	if err != nil {
-		return errors.Wrap(err, "GetTenantUUIDs")
-	}
-
-	for _, item := range store {
-		if lo.Contains(dbUUIDs, item.UUID) {
-			continue
-		}
-
-		u, err := uuid.FromBytes([]byte(item.UUID))
-		if err != nil {
-			return errors.Wrap(err, "uuid from bytes")
-		}
-
-		if err := d.mediasSvc.PublishTenantMedia(context.Background(), 1, u, item.Name, item.Price); err != nil {
-			return errors.Wrapf(err, "PublishTenant: %+v", item)
-		}
-
-		d.log.Infof("PublishTenant: %+v", item)
-	}
 
 	return nil
 }