rogee/mp-qvyun
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update discover medias

Browse Source
This commit is contained in:
Rogee
2024-12-06 14:13:04 +08:00
parent d42911eff8
commit 2b551f1b69
7 changed files with 115 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
backend/backend Executable file
View File

Binary file not shown.

22
backend/common/media_store/store.go
View File

@@ -9,12 +9,12 @@ import (
log "github.com/sirupsen/logrus"
)
type Store []UUIDMap
type Store []VideoInfo
type UUIDMap struct {
UUID string
Name string
Price uint
type VideoInfo struct {
Hash string
Name string
Duration uint
}
func NewStore(path string) (Store, error) {
@@ -56,17 +56,17 @@ func (s Store) Save(path string) error {
return nil
}
func (s Store) Add(uuid, name string, price uint) {
s = append(s, UUIDMap{UUID: uuid, Name: name, Price: price})
func (s Store) Append(info VideoInfo) Store {
return append(s, info)
}
func (s Store) UUIDs() []string {
var uuids []string
func (s Store) Hashes() []string {
var hashes []string
for _, m := range s {
uuids = append(uuids, m.UUID)
hashes = append(hashes, m.Hash)
}
return uuids
return hashes
}
// Exists

2
backend/config.toml
View File

@@ -29,4 +29,4 @@ Salt = "LiXi.Y@140202"
[Storage]
Type = "local"
Path = "/projects/mp-qvyun/backend/fixtures/medias"
Path = "/mnt/yangpingliang/publish/processed"

2
backend/fixtures/processed/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*
!.gitignore

148
backend/modules/tasks/discover/discover_medias.go
View File

@@ -2,7 +2,9 @@ package discover
import (
"bytes"
"crypto/md5"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
@@ -13,7 +15,6 @@ import (
"backend/modules/medias"
"backend/pkg/path"
"github.com/google/uuid"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cast"
@@ -37,94 +38,116 @@ func (d *DiscoverMedias) RunE(from, to string) error {
return errors.New("from or to is empty")
}
videos, err := d.globVideos(from)
if err != nil {
return errors.Wrapf(err, "glob videos: %s", from)
}
store, err := media_store.NewStore(to)
if err != nil {
return errors.Wrapf(err, "new store: %s", to)
}
defer func() {
d.runCleanup(to)
}()
dirs, err := path.GetSubDirs(from)
if err != nil {
return errors.Wrapf(err, "get sub dirs: %s", from)
}
for _, video := range videos {
md5, err := d.getFileMD5(video)
if err != nil {
return errors.Wrapf(err, "get file md5: %s", video)
}
for _, dir := range dirs {
d.log.Infof("Discover medias in: %s", dir)
if exists := store.Exists(dir); exists {
d.log.Infof("Skip dir: %s", dir)
if store.Exists(md5) {
continue
}
dirPath := filepath.Join(from, dir)
uuid := uuid.New().String()
to := filepath.Join(to, uuid)
videos := d.globVideo(dirPath)
if len(videos) > 0 {
video := videos[0]
d.log.Infof("video: %s", video)
if err := d.ffmpegVideoToM3U8(video, to); err != nil {
return errors.Wrapf(err, "ffmpeg video to m3u8: %s", video)
}
d.log.Info("snapshot", video)
posterPath := filepath.Join(to, "poster.jpg")
if err := d.ffmpegVideoToPoster(video, posterPath, 5); err != nil {
return errors.Wrapf(err, "ffmpeg snapshot : %s", video)
}
}
audios := d.globAudio(dirPath)
if len(audios) > 0 {
audio := audios[0]
d.log.Infof("audio: %s", audio)
if err := d.ffmpegAudioToM3U8(audio, to); err != nil {
return errors.Wrapf(err, "ffmpeg audio to m3u8: %s", audio)
}
}
price, err := d.getPrice(dirPath)
info, err := d.processVideo(video, filepath.Join(to, md5))
if err != nil {
return errors.Wrapf(err, "get price: %s", dirPath)
return errors.Wrapf(err, "process video: %s", video)
}
info.Hash = md5
info.Name = filepath.Base(video)
store.Add(uuid, dir, price)
store = store.Append(info)
d.log.Infof("store: %+v", store)
if err := store.Save(to); err != nil {
return errors.Wrapf(err, "store save: %s", to)
return errors.Wrapf(err, "save store: %s", to)
}
}
return nil
}
func (d *DiscoverMedias) globVideo(root string) []string {
files := []string{}
func (d *DiscoverMedias) processVideo(video string, to string) (media_store.VideoInfo, error) {
var info media_store.VideoInfo
exts := []string{"*.mp4", "*.MP4", "*.mov", "*.MOV"}
for _, ext := range exts {
fs, _ := filepath.Glob(filepath.Join(root, ext))
files = append(files, fs...)
if err := d.ensureDirectory(to); err != nil {
return info, errors.Wrapf(err, "ensure directory: %s", to)
}
return files
// extract audio from video
audioFile := filepath.Join(to, "audio.mp3")
if err := d.extractAudio(video, audioFile); err != nil {
return info, errors.Wrapf(err, "extract audio %s from %s", audioFile, video)
}
defer os.Remove(audioFile)
// ffmpeg video to m3u8
if err := d.ffmpegVideoToM3U8(video, to); err != nil {
return info, errors.Wrapf(err, "ffmpeg video to m3u8: %s", video)
}
// ffmpeg audio to m3u8
if err := d.ffmpegAudioToM3U8(audioFile, to); err != nil {
return info, errors.Wrapf(err, "ffmpeg audio to m3u8: %s", audioFile)
}
// get media duration
duration, err := d.getMediaDuration(video)
if err != nil {
return info, errors.Wrapf(err, "get media duration: %s", video)
}
info.Duration = uint(duration)
return info, nil
}
func (d *DiscoverMedias) globAudio(root string) []string {
files := []string{}
func (d *DiscoverMedias) getFileMD5(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", errors.Wrapf(err, "open file: %s", filePath)
}
defer file.Close()
exts := []string{"*.mp3", "*.MP3", "*.wav", "*.WAV"}
for _, ext := range exts {
fs, _ := filepath.Glob(filepath.Join(root, ext))
files = append(files, fs...)
hash := md5.New()
if _, err := io.Copy(hash, file); err != nil {
return "", errors.Wrapf(err, "copy file to hash: %s", filePath)
}
return files
return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
// extractAudio extract audio from video
func (d *DiscoverMedias) extractAudio(video string, output string) error {
args := []string{
"-i", video,
"-vn",
"-acodec", "libmp3lame",
"-ar", "44100",
"-b:a", "128k",
"-q:a", "2",
output,
}
d.log.Infof("cmd: ffmpeg %s", strings.Join(args, " "))
logs, err := exec.Command("ffmpeg", args...).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "extract audio: %s", video)
}
d.log.Infof("extract audio: %s", logs)
return nil
}
// ensureDirectory ensure the directory exists
@@ -159,6 +182,7 @@ func (d *DiscoverMedias) ffmpegVideoToM3U8(input string, output string) error {
"-c:a", "aac",
"-strict",
"-2",
"-vf", "scale=-720:576",
"-f", "hls",
"-hls_time", "10",
"-hls_list_size", "0",
@@ -293,3 +317,9 @@ func (d *DiscoverMedias) getMediaDuration(file string) (float64, error) {
return duration, nil
}
// getMedias get the medias in the directory
func (d *DiscoverMedias) globVideos(dir string) ([]string, error) {
// glob *.mp4 in dir
return filepath.Glob(filepath.Join(dir, "*.mp4"))
}

25
backend/modules/tasks/discover/discover_medias_test.go
View File

@@ -36,20 +36,6 @@ func Test_DiscoverMedias(t *testing.T) {
})
}
func (t *DiscoverMediasTestSuite) Test_getVideo() {
Convey("Test_getVideo", t.T(), func() {
dirs := t.Svc.globVideo("/mnt/yangpingliang/动态曲谱/河北梆子伴奏《李慧娘》依然还我生前摸样")
t.T().Logf("Dirs: %+v", dirs)
})
}
func (t *DiscoverMediasTestSuite) Test_getAudio() {
Convey("Test_getAudio", t.T(), func() {
dirs := t.Svc.globAudio("/mnt/yangpingliang/动态曲谱/河北梆子伴奏《李慧娘》依然还我生前摸样")
t.T().Logf("Dirs: %+v", dirs)
})
}
func (t *DiscoverMediasTestSuite) Test_ffmpegVideoToM3U8() {
Convey("Test_ffmpegVideoToM3U8", t.T(), func() {
output := "/projects/mp-qvyun/backend/fixtures/medias/abc"
@@ -82,3 +68,14 @@ func (t *DiscoverMediasTestSuite) TestDiscoverMedias_getMediaDuration() {
t.T().Logf("Duration: %0.8f", duration)
})
}
func (t *DiscoverMediasTestSuite) Test_globMedias() {
Convey("Test_globMedias", t.T(), func() {
f, err := t.Svc.globVideos("/mnt/yangpingliang/publish/processed")
So(err, ShouldBeNil)
for _, item := range f {
t.T().Logf("Item: %s", item)
}
})
}

39
backend/modules/tasks/store/store_medias.go
View File

@@ -1,14 +1,10 @@
package store
import (
"context"
"backend/common/media_store"
"backend/modules/medias"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/samber/lo"
log "github.com/sirupsen/logrus"
)
@@ -31,40 +27,7 @@ func (d *StoreMedias) RunE(targetPath string) error {
if err != nil {
return errors.Wrapf(err, "new store: %s", targetPath)
}
// uuids := lo.FilterMap(store.UUIDs(), func(item string, _ int) (uuid.UUID, bool) {
// u, err := uuid.FromBytes([]byte(item))
// if err != nil {
// return uuid.Nil, false
// }
// return u, true
// })
if err := d.mediasSvc.UnPublishTenantWithNotInUUIDs(context.Background(), 1, store.UUIDs()); err != nil {
return errors.Wrapf(err, "UnPublishTenantWithNotInUUIDs: %+v", store.UUIDs())
}
dbUUIDs, err := d.mediasSvc.GetTenantUUIDs(context.Background(), 1)
if err != nil {
return errors.Wrap(err, "GetTenantUUIDs")
}
for _, item := range store {
if lo.Contains(dbUUIDs, item.UUID) {
continue
}
u, err := uuid.FromBytes([]byte(item.UUID))
if err != nil {
return errors.Wrap(err, "uuid from bytes")
}
if err := d.mediasSvc.PublishTenantMedia(context.Background(), 1, u, item.Name, item.Price); err != nil {
return errors.Wrapf(err, "PublishTenant: %+v", item)
}
d.log.Infof("PublishTenant: %+v", item)
}
_ = store
return nil
}