rogee/tg_exporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: issues

Browse Source
This commit is contained in:
Rogee
2024-09-02 23:39:22 +08:00
parent d3806983a4
commit e953d41e7a
10 changed files with 123 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
internal/channel_message.go
View File

@@ -3,7 +3,10 @@ package internal
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/samber/lo"
)
type ChannelMessage struct {
@@ -15,10 +18,23 @@ type ChannelMessage struct {
}
type ChannelMessageMedia struct {
Photo string
Video string
Document string
WebPage ChannelMessageMediaWebPage
Photo *string
Video *string
Document *ChannelMessageDocument
WebPage *ChannelMessageMediaWebPage
}
type ChannelMessageDocument struct {
Ext string
Filename string
MimeType string
Size int64
Video *ChannelMessageDocumentVideo
}
type ChannelMessageDocumentVideo struct {
Duration float64
Width int
Height int
}
type ChannelMessageMediaWebPage struct {
@@ -41,22 +57,28 @@ func (c *ChannelMessage) WithMessage(message string) *ChannelMessage {
}
func (c *ChannelMessage) WithPhoto(assetID int64, ext string) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{Photo: fmt.Sprintf("%d.%s", assetID, ext)})
c.Medias = append(c.Medias, ChannelMessageMedia{
Photo: lo.ToPtr(fmt.Sprintf("%d.%s", assetID, strings.Trim(ext, "."))),
})
return c
}
func (c *ChannelMessage) WithVideo(video string) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{Video: video})
c.Medias = append(c.Medias, ChannelMessageMedia{Video: lo.ToPtr(video)})
return c
}
func (c *ChannelMessage) WithDoc(docID int64, ext string) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{Document: fmt.Sprintf("%d.%s", docID, ext)})
func (c *ChannelMessage) WithDocument(d ChannelMessageDocument) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{
Document: lo.ToPtr(d),
})
return c
}
func (c *ChannelMessage) WithWebPage(title, url string) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{WebPage: ChannelMessageMediaWebPage{Title: title, URL: url}})
c.Medias = append(c.Medias, ChannelMessageMedia{
WebPage: lo.ToPtr(ChannelMessageMediaWebPage{Title: title, URL: url}),
})
return c
}

8
internal/client.go
View File

@@ -17,17 +17,11 @@ import (
type TClient struct {
Config *config.Config
Client *telegram.Client
logger *zap.Logger
api *tg.Client
waitLogin chan error
block chan struct{}
}
func NewClient(logger *zap.Logger, config *config.Config) *TClient {
c := &TClient{
Config: config,
logger: logger,
Client: telegram.NewClient(config.AppID, config.AppHash, telegram.Options{
Logger: logger,
UpdateHandler: nil,
@@ -41,7 +35,7 @@ func NewClient(logger *zap.Logger, config *config.Config) *TClient {
}
func (t *TClient) Login(ctx context.Context) error {
t.logger.Info("login phone", zap.String("phone", t.Config.Phone))
logger.Info("login phone", zap.String("phone", t.Config.Phone))
flow := auth.NewFlow(Terminal{PhoneNumber: client.Config.Phone}, auth.SendCodeOptions{})
if err := t.Client.Auth().IfNecessary(context.Background(), flow); err != nil {
return errors.Wrap(err, "auth")

71
internal/client_channel.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"mime"
"github.com/dustin/go-humanize"
"github.com/gotd/td/telegram/downloader"
"github.com/gotd/td/tg"
"github.com/pkg/errors"
@@ -25,24 +26,21 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
request.MinID = cfg.MinID // 提供此ID供新增加的消息
}
request.Limit = 1
request.OffsetID = 3753
messages, err := t.Client.API().MessagesGetHistory(ctx, request)
if err != nil {
return errors.Wrap(err, "messages.getHistory")
}
if len(messages.(*tg.MessagesChannelMessages).GetMessages()) == 0 {
t.logger.Info("no new message")
logger.Info("no new message")
return errors.New("no new message")
}
downloader := downloader.NewDownloader()
lo.ForEach(messages.(*tg.MessagesChannelMessages).GetMessages(), func(item tg.MessageClass, index int) {
defer func() {
t.logger.Info("update config", zap.Int("offset", cfg.Offset))
logger.Info("update config", zap.Int("offset", cfg.Offset))
if err := cfg.Update(ctx, item.GetID()); err != nil {
t.logger.Error("update config failed", zap.Error(err))
logger.Error("update config failed", zap.Error(err))
}
}()
@@ -55,7 +53,7 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
msg, ok := item.(*tg.Message)
if !ok {
t.logger.Error("convert msg to *tg.Message failed")
logger.Error("convert msg to *tg.Message failed")
return
}
@@ -71,14 +69,18 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
switch mediaClass.(type) {
case *tg.MessageMediaDocument:
if docClass, ok := mediaClass.(*tg.MessageMediaDocument).GetDocument(); ok {
t.logger.Warn("document",
logger.Warn("document",
zap.Int("msg_id", msg.ID),
zap.String("file_name", docClass.String()),
)
doc := docClass.(*tg.Document)
if doc.GetSize() > 1024*1024*10 {
t.logger.Warn("document size too large", zap.Int64("size", doc.GetSize()))
if doc.GetSize() > int64(t.Config.GetMaxSize()) {
logger.Warn(
"document size too large",
zap.Int64("size", doc.GetSize()),
zap.String("SizeHuman", humanize.Bytes(uint64(doc.GetSize()))),
)
return
}
@@ -95,18 +97,51 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
}
exts, err := mime.ExtensionsByType(doc.GetMimeType())
if err != nil {
t.logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType()))
logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType()))
return
}
saveTo := cfg.Asset(doc.GetID(), exts[0])
if len(exts) == 0 {
logger.Warn("no extension found", zap.String("mime_type", doc.GetMimeType()))
switch doc.GetMimeType() {
case "application/rar":
exts = []string{".rar"}
}
}
ext := exts[len(exts)-1]
saveTo := cfg.Asset(doc.GetID(), ext)
_, err = downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo)
if err != nil {
t.logger.Error("download failed", zap.Error(err))
logger.Error("download failed", zap.Error(err))
return
}
channelMessage.WithDoc(doc.GetID(), exts[0])
t.logger.Info("download photo success", zap.String("location", saveTo))
docAttr := doc.GetAttributes()
data := ChannelMessageDocument{
Ext: ext,
MimeType: doc.GetMimeType(),
Size: doc.GetSize(),
}
if len(docAttr) > 0 {
for _, attr := range docAttr {
switch attr.(type) {
case *tg.DocumentAttributeFilename:
data.Filename = attr.(*tg.DocumentAttributeFilename).GetFileName()
case *tg.DocumentAttributeVideo:
m := attr.(*tg.DocumentAttributeVideo)
data.Video = &ChannelMessageDocumentVideo{
Duration: m.GetDuration(),
Width: m.GetW(),
Height: m.GetH(),
}
}
}
}
channelMessage.WithDocument(data)
logger.Info("download document success", zap.String("location", saveTo), zap.Any("document", data))
return
}
case *tg.MessageMediaWebPage:
@@ -114,7 +149,7 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
channelMessage.WithWebPage(page.Title, page.URL)
return
}
t.logger.Warn("web_page", zap.String("url", mediaClass.(*tg.MessageMediaWebPage).GetWebpage().String()))
logger.Warn("web_page", zap.String("url", mediaClass.(*tg.MessageMediaWebPage).GetWebpage().String()))
case *tg.MessageMediaPhoto:
if photoClass, ok := mediaClass.(*tg.MessageMediaPhoto).GetPhoto(); ok {
photo := photoClass.(*tg.Photo)
@@ -134,11 +169,11 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
saveTo := cfg.Asset(photo.GetID(), "jpg")
_, err := downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo)
if err != nil {
t.logger.Error("download failed", zap.Error(err))
logger.Error("download failed", zap.Error(err))
return
}
channelMessage.WithPhoto(photo.GetID(), "jpg")
t.logger.Info("download photo success", zap.String("location", saveTo))
logger.Info("download photo success", zap.String("location", saveTo))
}
}
}

3
internal/db_channel.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"exporter/database/telegram_resource/public/model"
@@ -37,7 +38,7 @@ func NewDBChannel(uuid int64, username, title string) *DBChannel {
}
func (c *DBChannel) Asset(assetID int64, ext string) string {
assetFile := fmt.Sprintf("outputs/%d/%d.%s", c.UUID, assetID, ext)
assetFile := fmt.Sprintf("outputs/%d/%d.%s", c.UUID, assetID, strings.Trim(ext, "."))
// if file dir not exists then create it
if _, err := os.Stat(filepath.Dir(assetFile)); os.IsNotExist(err) {