diff --git a/.vscode/launch.json b/.vscode/launch.json index b8b5ccd..14e043b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -25,7 +25,17 @@ // https://t.me/DNSPODT/5156 // "--alias", "DNSPODT", // https://t.me/ruyoblog/4247 - "--alias", "ruyoblog", + // "--alias", "ruyoblog", + // https://t.me/pjrjzy/11373 + // "--alias", "pjrjzy", + // https://t.me/hackerNewsSummary007/1226 + // "--alias", "hackerNewsSummary007", + // https://t.me/buliang00/190 + // "--alias", "buliang00", + // https://t.me/woshadiao/161475 + // "--alias", "woshadiao", + // https://t.me/c/2023304596/2277 + "--channel", "2023304596", "--history", ] } diff --git a/config.yml b/config.yml index c90d6a7..074d8f4 100644 --- a/config.yml +++ b/config.yml @@ -4,4 +4,5 @@ app_hash: bc7a2f7b8893889ffa6115f5f0eac278 bot_token: session_file: ./session.json log_file: ./log.json +max_size: 50M dsn: "postgresql://postgres:xixi0202@10.1.1.3:5432/telegram_resource?sslmode=disable" \ No newline at end of file diff --git a/config/config.go b/config/config.go index 664366b..7fd1712 100644 --- a/config/config.go +++ b/config/config.go @@ -14,9 +14,16 @@ type Config struct { BotToken string `mapstructure:"bot_token"` SessionFile string `mapstructure:"session_file"` LogFile string `mapstructure:"log_file"` + MaxSize string `mapstructure:"max_size"` DSN string `mapstructure:"dsn"` } +// GetMaxSize +func (c *Config) GetMaxSize() uint { + // parse 50m to 50 * 1024 * 1024 + return viper.GetSizeInBytes(c.MaxSize) +} + func Load(path string) error { // Load the config file viper.SetConfigFile(path) diff --git a/go.mod b/go.mod index 88281cb..80e011f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module exporter go 1.22.1 require ( - github.com/go-jet/jet v2.3.0+incompatible + github.com/dustin/go-humanize v1.0.1 github.com/go-jet/jet/v2 v2.11.1 github.com/gotd/td v0.107.0 github.com/lib/pq v1.10.9 diff --git a/go.sum b/go.sum index 9852e97..dd64b32 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -16,8 +18,6 @@ github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ= github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38= github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ= -github.com/go-jet/jet v2.3.0+incompatible h1:Yg7JSERDC0f9x3dHUBMA2cxe9/qC6qlozDDO/s38USU= -github.com/go-jet/jet v2.3.0+incompatible/go.mod h1:XgTt00fj8pAXMKe1ETL9R/kZWWyi2j/ymuH+gaW+EdI= github.com/go-jet/jet/v2 v2.11.1 h1:SEbh2lRUIiQweJpV0boWsQ4bV13x9p4h+RfajnL6vgM= github.com/go-jet/jet/v2 v2.11.1/go.mod h1:+DTofDkGp1c0vpooXWEZyNhyi0k0mL7N2W9tdP4YqfA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= diff --git a/internal/channel_message.go b/internal/channel_message.go index 811040d..56054a1 100644 --- a/internal/channel_message.go +++ b/internal/channel_message.go @@ -3,7 +3,10 @@ package internal import ( "encoding/json" "fmt" + "strings" "time" + + "github.com/samber/lo" ) type ChannelMessage struct { @@ -15,10 +18,23 @@ type ChannelMessage struct { } type ChannelMessageMedia struct { - Photo string - Video string - Document string - WebPage ChannelMessageMediaWebPage + Photo *string + Video *string + Document *ChannelMessageDocument + WebPage *ChannelMessageMediaWebPage +} + +type ChannelMessageDocument struct { + Ext string + Filename string + MimeType string + Size int64 + Video *ChannelMessageDocumentVideo +} +type ChannelMessageDocumentVideo struct { + Duration float64 + Width int + Height int } type ChannelMessageMediaWebPage struct { @@ -41,22 +57,28 @@ func (c *ChannelMessage) WithMessage(message string) *ChannelMessage { } func (c *ChannelMessage) WithPhoto(assetID int64, ext string) *ChannelMessage { - c.Medias = append(c.Medias, ChannelMessageMedia{Photo: fmt.Sprintf("%d.%s", assetID, ext)}) + c.Medias = append(c.Medias, ChannelMessageMedia{ + Photo: lo.ToPtr(fmt.Sprintf("%d.%s", assetID, strings.Trim(ext, "."))), + }) return c } func (c *ChannelMessage) WithVideo(video string) *ChannelMessage { - c.Medias = append(c.Medias, ChannelMessageMedia{Video: video}) + c.Medias = append(c.Medias, ChannelMessageMedia{Video: lo.ToPtr(video)}) return c } -func (c *ChannelMessage) WithDoc(docID int64, ext string) *ChannelMessage { - c.Medias = append(c.Medias, ChannelMessageMedia{Document: fmt.Sprintf("%d.%s", docID, ext)}) +func (c *ChannelMessage) WithDocument(d ChannelMessageDocument) *ChannelMessage { + c.Medias = append(c.Medias, ChannelMessageMedia{ + Document: lo.ToPtr(d), + }) return c } func (c *ChannelMessage) WithWebPage(title, url string) *ChannelMessage { - c.Medias = append(c.Medias, ChannelMessageMedia{WebPage: ChannelMessageMediaWebPage{Title: title, URL: url}}) + c.Medias = append(c.Medias, ChannelMessageMedia{ + WebPage: lo.ToPtr(ChannelMessageMediaWebPage{Title: title, URL: url}), + }) return c } diff --git a/internal/client.go b/internal/client.go index a6da0a6..51a3a9d 100644 --- a/internal/client.go +++ b/internal/client.go @@ -17,17 +17,11 @@ import ( type TClient struct { Config *config.Config Client *telegram.Client - logger *zap.Logger - api *tg.Client - - waitLogin chan error - block chan struct{} } func NewClient(logger *zap.Logger, config *config.Config) *TClient { c := &TClient{ Config: config, - logger: logger, Client: telegram.NewClient(config.AppID, config.AppHash, telegram.Options{ Logger: logger, UpdateHandler: nil, @@ -41,7 +35,7 @@ func NewClient(logger *zap.Logger, config *config.Config) *TClient { } func (t *TClient) Login(ctx context.Context) error { - t.logger.Info("login phone", zap.String("phone", t.Config.Phone)) + logger.Info("login phone", zap.String("phone", t.Config.Phone)) flow := auth.NewFlow(Terminal{PhoneNumber: client.Config.Phone}, auth.SendCodeOptions{}) if err := t.Client.Auth().IfNecessary(context.Background(), flow); err != nil { return errors.Wrap(err, "auth") diff --git a/internal/client_channel.go b/internal/client_channel.go index 194e0a8..0fcf95e 100644 --- a/internal/client_channel.go +++ b/internal/client_channel.go @@ -4,6 +4,7 @@ import ( "context" "mime" + "github.com/dustin/go-humanize" "github.com/gotd/td/telegram/downloader" "github.com/gotd/td/tg" "github.com/pkg/errors" @@ -25,24 +26,21 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann request.MinID = cfg.MinID // 提供此ID供新增加的消息 } - request.Limit = 1 - request.OffsetID = 3753 - messages, err := t.Client.API().MessagesGetHistory(ctx, request) if err != nil { return errors.Wrap(err, "messages.getHistory") } if len(messages.(*tg.MessagesChannelMessages).GetMessages()) == 0 { - t.logger.Info("no new message") + logger.Info("no new message") return errors.New("no new message") } downloader := downloader.NewDownloader() lo.ForEach(messages.(*tg.MessagesChannelMessages).GetMessages(), func(item tg.MessageClass, index int) { defer func() { - t.logger.Info("update config", zap.Int("offset", cfg.Offset)) + logger.Info("update config", zap.Int("offset", cfg.Offset)) if err := cfg.Update(ctx, item.GetID()); err != nil { - t.logger.Error("update config failed", zap.Error(err)) + logger.Error("update config failed", zap.Error(err)) } }() @@ -55,7 +53,7 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann msg, ok := item.(*tg.Message) if !ok { - t.logger.Error("convert msg to *tg.Message failed") + logger.Error("convert msg to *tg.Message failed") return } @@ -71,14 +69,18 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann switch mediaClass.(type) { case *tg.MessageMediaDocument: if docClass, ok := mediaClass.(*tg.MessageMediaDocument).GetDocument(); ok { - t.logger.Warn("document", + logger.Warn("document", zap.Int("msg_id", msg.ID), zap.String("file_name", docClass.String()), ) doc := docClass.(*tg.Document) - if doc.GetSize() > 1024*1024*10 { - t.logger.Warn("document size too large", zap.Int64("size", doc.GetSize())) + if doc.GetSize() > int64(t.Config.GetMaxSize()) { + logger.Warn( + "document size too large", + zap.Int64("size", doc.GetSize()), + zap.String("SizeHuman", humanize.Bytes(uint64(doc.GetSize()))), + ) return } @@ -95,18 +97,51 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann } exts, err := mime.ExtensionsByType(doc.GetMimeType()) if err != nil { - t.logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType())) + logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType())) return } - saveTo := cfg.Asset(doc.GetID(), exts[0]) + if len(exts) == 0 { + logger.Warn("no extension found", zap.String("mime_type", doc.GetMimeType())) + switch doc.GetMimeType() { + case "application/rar": + exts = []string{".rar"} + } + } + ext := exts[len(exts)-1] + + saveTo := cfg.Asset(doc.GetID(), ext) _, err = downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo) if err != nil { - t.logger.Error("download failed", zap.Error(err)) + logger.Error("download failed", zap.Error(err)) return } - channelMessage.WithDoc(doc.GetID(), exts[0]) - t.logger.Info("download photo success", zap.String("location", saveTo)) + docAttr := doc.GetAttributes() + + data := ChannelMessageDocument{ + Ext: ext, + MimeType: doc.GetMimeType(), + Size: doc.GetSize(), + } + if len(docAttr) > 0 { + for _, attr := range docAttr { + switch attr.(type) { + case *tg.DocumentAttributeFilename: + data.Filename = attr.(*tg.DocumentAttributeFilename).GetFileName() + case *tg.DocumentAttributeVideo: + m := attr.(*tg.DocumentAttributeVideo) + data.Video = &ChannelMessageDocumentVideo{ + Duration: m.GetDuration(), + Width: m.GetW(), + Height: m.GetH(), + } + + } + } + } + + channelMessage.WithDocument(data) + logger.Info("download document success", zap.String("location", saveTo), zap.Any("document", data)) return } case *tg.MessageMediaWebPage: @@ -114,7 +149,7 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann channelMessage.WithWebPage(page.Title, page.URL) return } - t.logger.Warn("web_page", zap.String("url", mediaClass.(*tg.MessageMediaWebPage).GetWebpage().String())) + logger.Warn("web_page", zap.String("url", mediaClass.(*tg.MessageMediaWebPage).GetWebpage().String())) case *tg.MessageMediaPhoto: if photoClass, ok := mediaClass.(*tg.MessageMediaPhoto).GetPhoto(); ok { photo := photoClass.(*tg.Photo) @@ -134,11 +169,11 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann saveTo := cfg.Asset(photo.GetID(), "jpg") _, err := downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo) if err != nil { - t.logger.Error("download failed", zap.Error(err)) + logger.Error("download failed", zap.Error(err)) return } channelMessage.WithPhoto(photo.GetID(), "jpg") - t.logger.Info("download photo success", zap.String("location", saveTo)) + logger.Info("download photo success", zap.String("location", saveTo)) } } } diff --git a/internal/db_channel.go b/internal/db_channel.go index c591f9f..aae3b77 100644 --- a/internal/db_channel.go +++ b/internal/db_channel.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "exporter/database/telegram_resource/public/model" @@ -37,7 +38,7 @@ func NewDBChannel(uuid int64, username, title string) *DBChannel { } func (c *DBChannel) Asset(assetID int64, ext string) string { - assetFile := fmt.Sprintf("outputs/%d/%d.%s", c.UUID, assetID, ext) + assetFile := fmt.Sprintf("outputs/%d/%d.%s", c.UUID, assetID, strings.Trim(ext, ".")) // if file dir not exists then create it if _, err := os.Stat(filepath.Dir(assetFile)); os.IsNotExist(err) { diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..92fbc1c --- /dev/null +++ b/main_test.go @@ -0,0 +1,14 @@ +package main + +import ( + "mime" + "testing" +) + +func Test_Mime(t *testing.T) { + // Test code here + m := "application/rar" + m = "video/mp4" + + t.Log(mime.ExtensionsByType(m)) +}