rogee/tg_exporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: issues

Browse Source
This commit is contained in:
Rogee
2024-09-03 00:33:27 +08:00
parent 3f1197e5d4
commit ba16785b43
4 changed files with 152 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.vscode/launch.json vendored
View File

@@ -35,7 +35,9 @@
// https://t.me/woshadiao/161475
// "--alias", "woshadiao",
// https://t.me/c/2023304596/2277
"--channel", "2023304596",
// "--channel", "2023304596",
// https://t.me/cgblz/25220
"--alias", "cgblz",
"--history",
]
}

2
config/config.go
View File

@@ -16,7 +16,7 @@ type Config struct {
LogFile string `mapstructure:"log_file"`
MaxSize string `mapstructure:"max_size"`
DSN string `mapstructure:"dsn"`
Outputs string `mapstructure:"outputs"`
Output string `mapstructure:"output"`
}
// GetMaxSize

267
internal/client_channel.go
View File

@@ -3,12 +3,12 @@ package internal
import (
"context"
"mime"
"os"
"github.com/dustin/go-humanize"
"github.com/gotd/td/telegram/downloader"
"github.com/gotd/td/tg"
"github.com/pkg/errors"
"github.com/samber/lo"
"go.uber.org/zap"
)
@@ -26,6 +26,9 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
request.MinID = cfg.MinID // 提供此ID供新增加的消息
}
// request.OffsetID = 1339
// request.Limit = 1
messages, err := t.Client.API().MessagesGetHistory(ctx, request)
if err != nil {
return errors.Wrap(err, "messages.getHistory")
@@ -35,149 +38,173 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
return errors.New("no new message")
}
downloader := downloader.NewDownloader()
lo.ForEach(messages.(*tg.MessagesChannelMessages).GetMessages(), func(item tg.MessageClass, index int) {
defer func() {
logger.Info("update config", zap.Int("offset", cfg.Offset))
if err := cfg.Update(ctx, item.GetID()); err != nil {
logger.Error("update config failed", zap.Error(err))
}
}()
for _, item := range messages.(*tg.MessagesChannelMessages).GetMessages() {
switch item.(type) {
case *tg.MessageEmpty:
return
continue
case *tg.MessageService:
return
continue
}
msg, ok := item.(*tg.Message)
if !ok {
logger.Error("convert msg to *tg.Message failed")
return
continue
}
channelMessage := NewChannelMessage(msg.ID, msg.GetDate())
defer cfg.SaveMessage(ctx, channelMessage)
channelMessage.WithMessage(msg.GetMessage())
if grpID, ok := msg.GetGroupedID(); ok {
channelMessage.WithGroupID(grpID)
}
if mediaClass, ok := msg.GetMedia(); ok {
switch mediaClass.(type) {
case *tg.MessageMediaDocument:
if docClass, ok := mediaClass.(*tg.MessageMediaDocument).GetDocument(); ok {
logger.Warn("document",
zap.Int("msg_id", msg.ID),
zap.String("file_name", docClass.String()),
mediaClass, ok := msg.GetMedia()
if !ok {
continue
}
switch mediaClass.(type) {
case *tg.MessageMediaDocument:
if docClass, ok := mediaClass.(*tg.MessageMediaDocument).GetDocument(); ok {
logger.Warn("document",
zap.Int("msg_id", msg.ID),
zap.String("file_name", docClass.String()),
)
doc := docClass.(*tg.Document)
if doc.GetSize() > int64(t.Config.GetMaxSize()) {
logger.Warn(
"document size too large",
zap.Int64("size", doc.GetSize()),
zap.String("SizeHuman", humanize.Bytes(uint64(doc.GetSize()))),
)
doc := docClass.(*tg.Document)
if doc.GetSize() > int64(t.Config.GetMaxSize()) {
logger.Warn(
"document size too large",
zap.Int64("size", doc.GetSize()),
zap.String("SizeHuman", humanize.Bytes(uint64(doc.GetSize()))),
)
return
}
thumbSize := ""
if len(doc.Thumbs) > 1 {
thumbSize = doc.Thumbs[len(doc.Thumbs)-1].GetType()
}
location := &tg.InputDocumentFileLocation{
ID: doc.GetID(),
AccessHash: doc.GetAccessHash(),
FileReference: doc.GetFileReference(),
ThumbSize: thumbSize,
}
exts, err := mime.ExtensionsByType(doc.GetMimeType())
if err != nil {
logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType()))
return
}
if len(exts) == 0 {
logger.Warn("no extension found", zap.String("mime_type", doc.GetMimeType()))
switch doc.GetMimeType() {
case "application/rar":
exts = []string{".rar"}
}
}
ext := exts[len(exts)-1]
saveTo := cfg.Asset(doc.GetID(), ext)
_, err = downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo)
if err != nil {
logger.Error("download failed", zap.Error(err))
return
}
docAttr := doc.GetAttributes()
data := ChannelMessageDocument{
Ext: ext,
MimeType: doc.GetMimeType(),
Size: doc.GetSize(),
}
if len(docAttr) > 0 {
for _, attr := range docAttr {
switch attr.(type) {
case *tg.DocumentAttributeFilename:
data.Filename = attr.(*tg.DocumentAttributeFilename).GetFileName()
case *tg.DocumentAttributeVideo:
m := attr.(*tg.DocumentAttributeVideo)
data.Video = &ChannelMessageDocumentVideo{
Duration: m.GetDuration(),
Width: m.GetW(),
Height: m.GetH(),
}
}
}
}
channelMessage.WithDocument(data)
logger.Info("download document success", zap.String("location", saveTo), zap.Any("document", data))
return
continue
}
case *tg.MessageMediaWebPage:
if page, ok := mediaClass.(*tg.MessageMediaWebPage).GetWebpage().(*tg.WebPage); ok {
channelMessage.WithWebPage(page.Title, page.URL)
return
data, err := t.saveDocument(ctx, cfg, doc)
if err != nil {
logger.Error("save document failed", zap.Error(err))
return err
}
channelMessage.WithDocument(data)
}
case *tg.MessageMediaWebPage:
if page, ok := mediaClass.(*tg.MessageMediaWebPage).GetWebpage().(*tg.WebPage); ok {
channelMessage.WithWebPage(page.Title, page.URL)
} else {
logger.Warn("web_page", zap.String("url", mediaClass.(*tg.MessageMediaWebPage).GetWebpage().String()))
case *tg.MessageMediaPhoto:
if photoClass, ok := mediaClass.(*tg.MessageMediaPhoto).GetPhoto(); ok {
photo := photoClass.(*tg.Photo)
thumbSize := ""
if len(photo.Sizes) > 1 {
thumbSize = photo.Sizes[len(photo.Sizes)-1].GetType()
}
location := &tg.InputPhotoFileLocation{
ID: photo.GetID(),
AccessHash: photo.GetAccessHash(),
FileReference: photo.GetFileReference(),
ThumbSize: thumbSize,
}
saveTo := cfg.Asset(photo.GetID(), "jpg")
_, err := downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo)
if err != nil {
logger.Error("download failed", zap.Error(err))
return
}
channelMessage.WithPhoto(photo.GetID(), "jpg")
logger.Info("download photo success", zap.String("location", saveTo))
}
case *tg.MessageMediaPhoto:
if photoClass, ok := mediaClass.(*tg.MessageMediaPhoto).GetPhoto(); ok {
photo := photoClass.(*tg.Photo)
if err := t.savePhoto(ctx, cfg, photo); err != nil {
logger.Error("save photo failed", zap.Error(err))
return err
}
channelMessage.WithPhoto(photo.GetID(), "jpg")
}
}
})
logger.Info("save message", zap.Int("id", channelMessage.ID))
if err := cfg.SaveMessage(ctx, channelMessage); err != nil {
logger.Error("save message failed", zap.Error(err))
return err
}
logger.Info("update config", zap.Int("offset", cfg.Offset))
if err := cfg.Update(ctx, item.GetID()); err != nil {
logger.Error("update config failed", zap.Error(err))
return err
}
}
return nil
}
// savePhoto
func (t *TClient) savePhoto(ctx context.Context, cfg *DBChannel, photo *tg.Photo) error {
thumbSize := ""
if len(photo.Sizes) > 1 {
thumbSize = photo.Sizes[len(photo.Sizes)-1].GetType()
}
location := &tg.InputPhotoFileLocation{
ID: photo.GetID(),
AccessHash: photo.GetAccessHash(),
FileReference: photo.GetFileReference(),
ThumbSize: thumbSize,
}
saveTo := cfg.Asset(photo.GetID(), "jpg")
downloader := downloader.NewDownloader()
_, err := downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo)
if err != nil {
os.Remove(saveTo)
logger.Error("download failed", zap.Error(err))
return err
}
logger.Info("download photo success", zap.String("location", saveTo))
return nil
}
// saveDocument
func (t *TClient) saveDocument(ctx context.Context, cfg *DBChannel, doc *tg.Document) (ChannelMessageDocument, error) {
location := &tg.InputDocumentFileLocation{
ID: doc.GetID(),
AccessHash: doc.GetAccessHash(),
FileReference: doc.GetFileReference(),
}
exts, err := mime.ExtensionsByType(doc.GetMimeType())
if err != nil {
logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType()))
return ChannelMessageDocument{}, err
}
if len(exts) == 0 {
logger.Warn("no extension found", zap.String("mime_type", doc.GetMimeType()))
switch doc.GetMimeType() {
case "application/rar":
exts = []string{".rar"}
}
}
ext := exts[len(exts)-1]
saveTo := cfg.Asset(doc.GetID(), ext)
downloader := downloader.NewDownloader()
_, err = downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo)
if err != nil {
os.Remove(saveTo)
logger.Error("download failed", zap.Error(err))
return ChannelMessageDocument{}, err
}
docAttr := doc.GetAttributes()
data := ChannelMessageDocument{
Ext: ext,
MimeType: doc.GetMimeType(),
Size: doc.GetSize(),
}
if len(docAttr) > 0 {
for _, attr := range docAttr {
switch attr.(type) {
case *tg.DocumentAttributeFilename:
data.Filename = attr.(*tg.DocumentAttributeFilename).GetFileName()
case *tg.DocumentAttributeVideo:
m := attr.(*tg.DocumentAttributeVideo)
data.Video = &ChannelMessageDocumentVideo{
Duration: m.GetDuration(),
Width: m.GetW(),
Height: m.GetH(),
}
}
}
}
logger.Info("download document success", zap.String("location", saveTo), zap.Any("document", data))
return data, nil
}

2
internal/db_channel.go
View File

@@ -40,7 +40,7 @@ func NewDBChannel(uuid int64, username, title string) *DBChannel {
func (c *DBChannel) Asset(assetID int64, ext string) string {
assetFile := fmt.Sprintf("%d/%d.%s", c.UUID, assetID, strings.Trim(ext, "."))
assetFile = filepath.Join(config.C.Outputs, assetFile)
assetFile = filepath.Join(config.C.Output, assetFile)
// if file dir not exists then create it
if _, err := os.Stat(filepath.Dir(assetFile)); os.IsNotExist(err) {