diff --git a/.vscode/launch.json b/.vscode/launch.json index 14e043b..15793d5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -35,7 +35,9 @@ // https://t.me/woshadiao/161475 // "--alias", "woshadiao", // https://t.me/c/2023304596/2277 - "--channel", "2023304596", + // "--channel", "2023304596", + // https://t.me/cgblz/25220 + "--alias", "cgblz", "--history", ] } diff --git a/config/config.go b/config/config.go index 0f5b3a5..0a0c9bd 100644 --- a/config/config.go +++ b/config/config.go @@ -16,7 +16,7 @@ type Config struct { LogFile string `mapstructure:"log_file"` MaxSize string `mapstructure:"max_size"` DSN string `mapstructure:"dsn"` - Outputs string `mapstructure:"outputs"` + Output string `mapstructure:"output"` } // GetMaxSize diff --git a/internal/client_channel.go b/internal/client_channel.go index 0fcf95e..18c8a0a 100644 --- a/internal/client_channel.go +++ b/internal/client_channel.go @@ -3,12 +3,12 @@ package internal import ( "context" "mime" + "os" "github.com/dustin/go-humanize" "github.com/gotd/td/telegram/downloader" "github.com/gotd/td/tg" "github.com/pkg/errors" - "github.com/samber/lo" "go.uber.org/zap" ) @@ -26,6 +26,9 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann request.MinID = cfg.MinID // 提供此ID供新增加的消息 } + // request.OffsetID = 1339 + // request.Limit = 1 + messages, err := t.Client.API().MessagesGetHistory(ctx, request) if err != nil { return errors.Wrap(err, "messages.getHistory") @@ -35,149 +38,173 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann return errors.New("no new message") } - downloader := downloader.NewDownloader() - lo.ForEach(messages.(*tg.MessagesChannelMessages).GetMessages(), func(item tg.MessageClass, index int) { - defer func() { - logger.Info("update config", zap.Int("offset", cfg.Offset)) - if err := cfg.Update(ctx, item.GetID()); err != nil { - logger.Error("update config failed", zap.Error(err)) - } - }() - + for _, item := range messages.(*tg.MessagesChannelMessages).GetMessages() { switch item.(type) { case *tg.MessageEmpty: - return + continue case *tg.MessageService: - return + continue } msg, ok := item.(*tg.Message) if !ok { logger.Error("convert msg to *tg.Message failed") - return + continue } channelMessage := NewChannelMessage(msg.ID, msg.GetDate()) - defer cfg.SaveMessage(ctx, channelMessage) channelMessage.WithMessage(msg.GetMessage()) if grpID, ok := msg.GetGroupedID(); ok { channelMessage.WithGroupID(grpID) } - if mediaClass, ok := msg.GetMedia(); ok { - switch mediaClass.(type) { - case *tg.MessageMediaDocument: - if docClass, ok := mediaClass.(*tg.MessageMediaDocument).GetDocument(); ok { - logger.Warn("document", - zap.Int("msg_id", msg.ID), - zap.String("file_name", docClass.String()), + mediaClass, ok := msg.GetMedia() + if !ok { + continue + } + switch mediaClass.(type) { + case *tg.MessageMediaDocument: + if docClass, ok := mediaClass.(*tg.MessageMediaDocument).GetDocument(); ok { + logger.Warn("document", + zap.Int("msg_id", msg.ID), + zap.String("file_name", docClass.String()), + ) + + doc := docClass.(*tg.Document) + + if doc.GetSize() > int64(t.Config.GetMaxSize()) { + logger.Warn( + "document size too large", + zap.Int64("size", doc.GetSize()), + zap.String("SizeHuman", humanize.Bytes(uint64(doc.GetSize()))), ) - - doc := docClass.(*tg.Document) - if doc.GetSize() > int64(t.Config.GetMaxSize()) { - logger.Warn( - "document size too large", - zap.Int64("size", doc.GetSize()), - zap.String("SizeHuman", humanize.Bytes(uint64(doc.GetSize()))), - ) - return - } - - thumbSize := "" - if len(doc.Thumbs) > 1 { - thumbSize = doc.Thumbs[len(doc.Thumbs)-1].GetType() - } - - location := &tg.InputDocumentFileLocation{ - ID: doc.GetID(), - AccessHash: doc.GetAccessHash(), - FileReference: doc.GetFileReference(), - ThumbSize: thumbSize, - } - exts, err := mime.ExtensionsByType(doc.GetMimeType()) - if err != nil { - logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType())) - return - } - - if len(exts) == 0 { - logger.Warn("no extension found", zap.String("mime_type", doc.GetMimeType())) - switch doc.GetMimeType() { - case "application/rar": - exts = []string{".rar"} - } - } - ext := exts[len(exts)-1] - - saveTo := cfg.Asset(doc.GetID(), ext) - _, err = downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo) - if err != nil { - logger.Error("download failed", zap.Error(err)) - return - } - docAttr := doc.GetAttributes() - - data := ChannelMessageDocument{ - Ext: ext, - MimeType: doc.GetMimeType(), - Size: doc.GetSize(), - } - if len(docAttr) > 0 { - for _, attr := range docAttr { - switch attr.(type) { - case *tg.DocumentAttributeFilename: - data.Filename = attr.(*tg.DocumentAttributeFilename).GetFileName() - case *tg.DocumentAttributeVideo: - m := attr.(*tg.DocumentAttributeVideo) - data.Video = &ChannelMessageDocumentVideo{ - Duration: m.GetDuration(), - Width: m.GetW(), - Height: m.GetH(), - } - - } - } - } - - channelMessage.WithDocument(data) - logger.Info("download document success", zap.String("location", saveTo), zap.Any("document", data)) - return + continue } - case *tg.MessageMediaWebPage: - if page, ok := mediaClass.(*tg.MessageMediaWebPage).GetWebpage().(*tg.WebPage); ok { - channelMessage.WithWebPage(page.Title, page.URL) - return + + data, err := t.saveDocument(ctx, cfg, doc) + if err != nil { + logger.Error("save document failed", zap.Error(err)) + return err } + channelMessage.WithDocument(data) + } + case *tg.MessageMediaWebPage: + if page, ok := mediaClass.(*tg.MessageMediaWebPage).GetWebpage().(*tg.WebPage); ok { + channelMessage.WithWebPage(page.Title, page.URL) + } else { logger.Warn("web_page", zap.String("url", mediaClass.(*tg.MessageMediaWebPage).GetWebpage().String())) - case *tg.MessageMediaPhoto: - if photoClass, ok := mediaClass.(*tg.MessageMediaPhoto).GetPhoto(); ok { - photo := photoClass.(*tg.Photo) - - thumbSize := "" - if len(photo.Sizes) > 1 { - thumbSize = photo.Sizes[len(photo.Sizes)-1].GetType() - } - - location := &tg.InputPhotoFileLocation{ - ID: photo.GetID(), - AccessHash: photo.GetAccessHash(), - FileReference: photo.GetFileReference(), - ThumbSize: thumbSize, - } - - saveTo := cfg.Asset(photo.GetID(), "jpg") - _, err := downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo) - if err != nil { - logger.Error("download failed", zap.Error(err)) - return - } - channelMessage.WithPhoto(photo.GetID(), "jpg") - logger.Info("download photo success", zap.String("location", saveTo)) + } + case *tg.MessageMediaPhoto: + if photoClass, ok := mediaClass.(*tg.MessageMediaPhoto).GetPhoto(); ok { + photo := photoClass.(*tg.Photo) + if err := t.savePhoto(ctx, cfg, photo); err != nil { + logger.Error("save photo failed", zap.Error(err)) + return err } + channelMessage.WithPhoto(photo.GetID(), "jpg") } } - }) + + logger.Info("save message", zap.Int("id", channelMessage.ID)) + if err := cfg.SaveMessage(ctx, channelMessage); err != nil { + logger.Error("save message failed", zap.Error(err)) + return err + } + + logger.Info("update config", zap.Int("offset", cfg.Offset)) + if err := cfg.Update(ctx, item.GetID()); err != nil { + logger.Error("update config failed", zap.Error(err)) + return err + } + + } return nil } + +// savePhoto +func (t *TClient) savePhoto(ctx context.Context, cfg *DBChannel, photo *tg.Photo) error { + thumbSize := "" + if len(photo.Sizes) > 1 { + thumbSize = photo.Sizes[len(photo.Sizes)-1].GetType() + } + + location := &tg.InputPhotoFileLocation{ + ID: photo.GetID(), + AccessHash: photo.GetAccessHash(), + FileReference: photo.GetFileReference(), + ThumbSize: thumbSize, + } + + saveTo := cfg.Asset(photo.GetID(), "jpg") + downloader := downloader.NewDownloader() + _, err := downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo) + if err != nil { + os.Remove(saveTo) + logger.Error("download failed", zap.Error(err)) + return err + } + + logger.Info("download photo success", zap.String("location", saveTo)) + + return nil +} + +// saveDocument +func (t *TClient) saveDocument(ctx context.Context, cfg *DBChannel, doc *tg.Document) (ChannelMessageDocument, error) { + location := &tg.InputDocumentFileLocation{ + ID: doc.GetID(), + AccessHash: doc.GetAccessHash(), + FileReference: doc.GetFileReference(), + } + exts, err := mime.ExtensionsByType(doc.GetMimeType()) + if err != nil { + logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType())) + return ChannelMessageDocument{}, err + } + + if len(exts) == 0 { + logger.Warn("no extension found", zap.String("mime_type", doc.GetMimeType())) + switch doc.GetMimeType() { + case "application/rar": + exts = []string{".rar"} + } + } + ext := exts[len(exts)-1] + + saveTo := cfg.Asset(doc.GetID(), ext) + downloader := downloader.NewDownloader() + _, err = downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo) + if err != nil { + os.Remove(saveTo) + logger.Error("download failed", zap.Error(err)) + return ChannelMessageDocument{}, err + } + docAttr := doc.GetAttributes() + + data := ChannelMessageDocument{ + Ext: ext, + MimeType: doc.GetMimeType(), + Size: doc.GetSize(), + } + if len(docAttr) > 0 { + for _, attr := range docAttr { + switch attr.(type) { + case *tg.DocumentAttributeFilename: + data.Filename = attr.(*tg.DocumentAttributeFilename).GetFileName() + case *tg.DocumentAttributeVideo: + m := attr.(*tg.DocumentAttributeVideo) + data.Video = &ChannelMessageDocumentVideo{ + Duration: m.GetDuration(), + Width: m.GetW(), + Height: m.GetH(), + } + + } + } + } + + logger.Info("download document success", zap.String("location", saveTo), zap.Any("document", data)) + return data, nil +} diff --git a/internal/db_channel.go b/internal/db_channel.go index b220e71..e66a908 100644 --- a/internal/db_channel.go +++ b/internal/db_channel.go @@ -40,7 +40,7 @@ func NewDBChannel(uuid int64, username, title string) *DBChannel { func (c *DBChannel) Asset(assetID int64, ext string) string { assetFile := fmt.Sprintf("%d/%d.%s", c.UUID, assetID, strings.Trim(ext, ".")) - assetFile = filepath.Join(config.C.Outputs, assetFile) + assetFile = filepath.Join(config.C.Output, assetFile) // if file dir not exists then create it if _, err := os.Stat(filepath.Dir(assetFile)); os.IsNotExist(err) {