diff --git a/backend/backend b/backend/backend new file mode 100755 index 0000000..167f413 Binary files /dev/null and b/backend/backend differ diff --git a/backend/common/media_store/store.go b/backend/common/media_store/store.go index 6824e18..9cc1cce 100644 --- a/backend/common/media_store/store.go +++ b/backend/common/media_store/store.go @@ -9,12 +9,12 @@ import ( log "github.com/sirupsen/logrus" ) -type Store []UUIDMap +type Store []VideoInfo -type UUIDMap struct { - UUID string - Name string - Price uint +type VideoInfo struct { + Hash string + Name string + Duration uint } func NewStore(path string) (Store, error) { @@ -56,17 +56,17 @@ func (s Store) Save(path string) error { return nil } -func (s Store) Add(uuid, name string, price uint) { - s = append(s, UUIDMap{UUID: uuid, Name: name, Price: price}) +func (s Store) Append(info VideoInfo) Store { + return append(s, info) } -func (s Store) UUIDs() []string { - var uuids []string +func (s Store) Hashes() []string { + var hashes []string for _, m := range s { - uuids = append(uuids, m.UUID) + hashes = append(hashes, m.Hash) } - return uuids + return hashes } // Exists diff --git a/backend/config.toml b/backend/config.toml index e47890a..59b0c7f 100755 --- a/backend/config.toml +++ b/backend/config.toml @@ -29,4 +29,4 @@ Salt = "LiXi.Y@140202" [Storage] Type = "local" -Path = "/projects/mp-qvyun/backend/fixtures/medias" +Path = "/mnt/yangpingliang/publish/processed" diff --git a/backend/fixtures/processed/.gitignore b/backend/fixtures/processed/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/backend/fixtures/processed/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/backend/modules/tasks/discover/discover_medias.go b/backend/modules/tasks/discover/discover_medias.go index 092508c..2a5c8bc 100644 --- a/backend/modules/tasks/discover/discover_medias.go +++ b/backend/modules/tasks/discover/discover_medias.go @@ -2,7 +2,9 @@ package discover import ( "bytes" + "crypto/md5" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -13,7 +15,6 @@ import ( "backend/modules/medias" "backend/pkg/path" - "github.com/google/uuid" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cast" @@ -37,94 +38,116 @@ func (d *DiscoverMedias) RunE(from, to string) error { return errors.New("from or to is empty") } + videos, err := d.globVideos(from) + if err != nil { + return errors.Wrapf(err, "glob videos: %s", from) + } + store, err := media_store.NewStore(to) if err != nil { return errors.Wrapf(err, "new store: %s", to) } + defer func() { d.runCleanup(to) }() - dirs, err := path.GetSubDirs(from) - if err != nil { - return errors.Wrapf(err, "get sub dirs: %s", from) - } + for _, video := range videos { + md5, err := d.getFileMD5(video) + if err != nil { + return errors.Wrapf(err, "get file md5: %s", video) + } - for _, dir := range dirs { - d.log.Infof("Discover medias in: %s", dir) - - if exists := store.Exists(dir); exists { - d.log.Infof("Skip dir: %s", dir) + if store.Exists(md5) { continue } - dirPath := filepath.Join(from, dir) - - uuid := uuid.New().String() - to := filepath.Join(to, uuid) - - videos := d.globVideo(dirPath) - if len(videos) > 0 { - video := videos[0] - d.log.Infof("video: %s", video) - if err := d.ffmpegVideoToM3U8(video, to); err != nil { - return errors.Wrapf(err, "ffmpeg video to m3u8: %s", video) - } - - d.log.Info("snapshot", video) - posterPath := filepath.Join(to, "poster.jpg") - if err := d.ffmpegVideoToPoster(video, posterPath, 5); err != nil { - return errors.Wrapf(err, "ffmpeg snapshot : %s", video) - } - } - - audios := d.globAudio(dirPath) - if len(audios) > 0 { - audio := audios[0] - d.log.Infof("audio: %s", audio) - if err := d.ffmpegAudioToM3U8(audio, to); err != nil { - return errors.Wrapf(err, "ffmpeg audio to m3u8: %s", audio) - } - } - - price, err := d.getPrice(dirPath) + info, err := d.processVideo(video, filepath.Join(to, md5)) if err != nil { - return errors.Wrapf(err, "get price: %s", dirPath) + return errors.Wrapf(err, "process video: %s", video) } + info.Hash = md5 + info.Name = filepath.Base(video) - store.Add(uuid, dir, price) + store = store.Append(info) + d.log.Infof("store: %+v", store) if err := store.Save(to); err != nil { - return errors.Wrapf(err, "store save: %s", to) + return errors.Wrapf(err, "save store: %s", to) } } return nil } -func (d *DiscoverMedias) globVideo(root string) []string { - files := []string{} +func (d *DiscoverMedias) processVideo(video string, to string) (media_store.VideoInfo, error) { + var info media_store.VideoInfo - exts := []string{"*.mp4", "*.MP4", "*.mov", "*.MOV"} - - for _, ext := range exts { - fs, _ := filepath.Glob(filepath.Join(root, ext)) - files = append(files, fs...) + if err := d.ensureDirectory(to); err != nil { + return info, errors.Wrapf(err, "ensure directory: %s", to) } - return files + // extract audio from video + audioFile := filepath.Join(to, "audio.mp3") + if err := d.extractAudio(video, audioFile); err != nil { + return info, errors.Wrapf(err, "extract audio %s from %s", audioFile, video) + } + defer os.Remove(audioFile) + + // ffmpeg video to m3u8 + if err := d.ffmpegVideoToM3U8(video, to); err != nil { + return info, errors.Wrapf(err, "ffmpeg video to m3u8: %s", video) + } + + // ffmpeg audio to m3u8 + if err := d.ffmpegAudioToM3U8(audioFile, to); err != nil { + return info, errors.Wrapf(err, "ffmpeg audio to m3u8: %s", audioFile) + } + + // get media duration + duration, err := d.getMediaDuration(video) + if err != nil { + return info, errors.Wrapf(err, "get media duration: %s", video) + } + info.Duration = uint(duration) + + return info, nil } -func (d *DiscoverMedias) globAudio(root string) []string { - files := []string{} +func (d *DiscoverMedias) getFileMD5(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", errors.Wrapf(err, "open file: %s", filePath) + } + defer file.Close() - exts := []string{"*.mp3", "*.MP3", "*.wav", "*.WAV"} - - for _, ext := range exts { - fs, _ := filepath.Glob(filepath.Join(root, ext)) - files = append(files, fs...) + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { + return "", errors.Wrapf(err, "copy file to hash: %s", filePath) } - return files + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +// extractAudio extract audio from video +func (d *DiscoverMedias) extractAudio(video string, output string) error { + args := []string{ + "-i", video, + "-vn", + "-acodec", "libmp3lame", + "-ar", "44100", + "-b:a", "128k", + "-q:a", "2", + output, + } + + d.log.Infof("cmd: ffmpeg %s", strings.Join(args, " ")) + logs, err := exec.Command("ffmpeg", args...).CombinedOutput() + if err != nil { + return errors.Wrapf(err, "extract audio: %s", video) + } + d.log.Infof("extract audio: %s", logs) + + return nil } // ensureDirectory ensure the directory exists @@ -159,6 +182,7 @@ func (d *DiscoverMedias) ffmpegVideoToM3U8(input string, output string) error { "-c:a", "aac", "-strict", "-2", + "-vf", "scale=-720:576", "-f", "hls", "-hls_time", "10", "-hls_list_size", "0", @@ -293,3 +317,9 @@ func (d *DiscoverMedias) getMediaDuration(file string) (float64, error) { return duration, nil } + +// getMedias get the medias in the directory +func (d *DiscoverMedias) globVideos(dir string) ([]string, error) { + // glob *.mp4 in dir + return filepath.Glob(filepath.Join(dir, "*.mp4")) +} diff --git a/backend/modules/tasks/discover/discover_medias_test.go b/backend/modules/tasks/discover/discover_medias_test.go index 977c289..dc008ac 100644 --- a/backend/modules/tasks/discover/discover_medias_test.go +++ b/backend/modules/tasks/discover/discover_medias_test.go @@ -36,20 +36,6 @@ func Test_DiscoverMedias(t *testing.T) { }) } -func (t *DiscoverMediasTestSuite) Test_getVideo() { - Convey("Test_getVideo", t.T(), func() { - dirs := t.Svc.globVideo("/mnt/yangpingliang/动态曲谱/河北梆子伴奏《李慧娘》依然还我生前摸样") - t.T().Logf("Dirs: %+v", dirs) - }) -} - -func (t *DiscoverMediasTestSuite) Test_getAudio() { - Convey("Test_getAudio", t.T(), func() { - dirs := t.Svc.globAudio("/mnt/yangpingliang/动态曲谱/河北梆子伴奏《李慧娘》依然还我生前摸样") - t.T().Logf("Dirs: %+v", dirs) - }) -} - func (t *DiscoverMediasTestSuite) Test_ffmpegVideoToM3U8() { Convey("Test_ffmpegVideoToM3U8", t.T(), func() { output := "/projects/mp-qvyun/backend/fixtures/medias/abc" @@ -82,3 +68,14 @@ func (t *DiscoverMediasTestSuite) TestDiscoverMedias_getMediaDuration() { t.T().Logf("Duration: %0.8f", duration) }) } + +func (t *DiscoverMediasTestSuite) Test_globMedias() { + Convey("Test_globMedias", t.T(), func() { + f, err := t.Svc.globVideos("/mnt/yangpingliang/publish/processed") + So(err, ShouldBeNil) + + for _, item := range f { + t.T().Logf("Item: %s", item) + } + }) +} diff --git a/backend/modules/tasks/store/store_medias.go b/backend/modules/tasks/store/store_medias.go index 676df3e..83ae59f 100644 --- a/backend/modules/tasks/store/store_medias.go +++ b/backend/modules/tasks/store/store_medias.go @@ -1,14 +1,10 @@ package store import ( - "context" - "backend/common/media_store" "backend/modules/medias" - "github.com/google/uuid" "github.com/pkg/errors" - "github.com/samber/lo" log "github.com/sirupsen/logrus" ) @@ -31,40 +27,7 @@ func (d *StoreMedias) RunE(targetPath string) error { if err != nil { return errors.Wrapf(err, "new store: %s", targetPath) } - - // uuids := lo.FilterMap(store.UUIDs(), func(item string, _ int) (uuid.UUID, bool) { - // u, err := uuid.FromBytes([]byte(item)) - // if err != nil { - // return uuid.Nil, false - // } - // return u, true - // }) - - if err := d.mediasSvc.UnPublishTenantWithNotInUUIDs(context.Background(), 1, store.UUIDs()); err != nil { - return errors.Wrapf(err, "UnPublishTenantWithNotInUUIDs: %+v", store.UUIDs()) - } - - dbUUIDs, err := d.mediasSvc.GetTenantUUIDs(context.Background(), 1) - if err != nil { - return errors.Wrap(err, "GetTenantUUIDs") - } - - for _, item := range store { - if lo.Contains(dbUUIDs, item.UUID) { - continue - } - - u, err := uuid.FromBytes([]byte(item.UUID)) - if err != nil { - return errors.Wrap(err, "uuid from bytes") - } - - if err := d.mediasSvc.PublishTenantMedia(context.Background(), 1, u, item.Name, item.Price); err != nil { - return errors.Wrapf(err, "PublishTenant: %+v", item) - } - - d.log.Infof("PublishTenant: %+v", item) - } + _ = store return nil }