rogee/tg_exporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: issues

Browse Source
This commit is contained in:
Rogee
2024-09-02 23:39:22 +08:00
parent d3806983a4
commit e953d41e7a
10 changed files with 123 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.vscode/launch.json vendored
View File

@@ -25,7 +25,17 @@
// https://t.me/DNSPODT/5156
// "--alias", "DNSPODT",
// https://t.me/ruyoblog/4247
"--alias", "ruyoblog",
// "--alias", "ruyoblog",
// https://t.me/pjrjzy/11373
// "--alias", "pjrjzy",
// https://t.me/hackerNewsSummary007/1226
// "--alias", "hackerNewsSummary007",
// https://t.me/buliang00/190
// "--alias", "buliang00",
// https://t.me/woshadiao/161475
// "--alias", "woshadiao",
// https://t.me/c/2023304596/2277
"--channel", "2023304596",
"--history",
]
}

1
config.yml
View File

@@ -4,4 +4,5 @@ app_hash: bc7a2f7b8893889ffa6115f5f0eac278
bot_token:
session_file: ./session.json
log_file: ./log.json
max_size: 50M
dsn: "postgresql://postgres:xixi0202@10.1.1.3:5432/telegram_resource?sslmode=disable"

7
config/config.go
View File

@@ -14,9 +14,16 @@ type Config struct {
BotToken string `mapstructure:"bot_token"`
SessionFile string `mapstructure:"session_file"`
LogFile string `mapstructure:"log_file"`
MaxSize string `mapstructure:"max_size"`
DSN string `mapstructure:"dsn"`
}
// GetMaxSize
func (c *Config) GetMaxSize() uint {
// parse 50m to 50 * 1024 * 1024
return viper.GetSizeInBytes(c.MaxSize)
}
func Load(path string) error {
// Load the config file
viper.SetConfigFile(path)

2
go.mod
View File

@@ -3,7 +3,7 @@ module exporter
go 1.22.1
require (
github.com/go-jet/jet v2.3.0+incompatible
github.com/dustin/go-humanize v1.0.1
github.com/go-jet/jet/v2 v2.11.1
github.com/gotd/td v0.107.0
github.com/lib/pq v1.10.9

4
go.sum
View File

@@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -16,8 +18,6 @@ github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb
github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38=
github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
github.com/go-jet/jet v2.3.0+incompatible h1:Yg7JSERDC0f9x3dHUBMA2cxe9/qC6qlozDDO/s38USU=
github.com/go-jet/jet v2.3.0+incompatible/go.mod h1:XgTt00fj8pAXMKe1ETL9R/kZWWyi2j/ymuH+gaW+EdI=
github.com/go-jet/jet/v2 v2.11.1 h1:SEbh2lRUIiQweJpV0boWsQ4bV13x9p4h+RfajnL6vgM=
github.com/go-jet/jet/v2 v2.11.1/go.mod h1:+DTofDkGp1c0vpooXWEZyNhyi0k0mL7N2W9tdP4YqfA=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=

40
internal/channel_message.go
View File

@@ -3,7 +3,10 @@ package internal
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/samber/lo"
)
type ChannelMessage struct {
@@ -15,10 +18,23 @@ type ChannelMessage struct {
}
type ChannelMessageMedia struct {
Photo string
Video string
Document string
WebPage ChannelMessageMediaWebPage
Photo *string
Video *string
Document *ChannelMessageDocument
WebPage *ChannelMessageMediaWebPage
}
type ChannelMessageDocument struct {
Ext string
Filename string
MimeType string
Size int64
Video *ChannelMessageDocumentVideo
}
type ChannelMessageDocumentVideo struct {
Duration float64
Width int
Height int
}
type ChannelMessageMediaWebPage struct {
@@ -41,22 +57,28 @@ func (c *ChannelMessage) WithMessage(message string) *ChannelMessage {
}
func (c *ChannelMessage) WithPhoto(assetID int64, ext string) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{Photo: fmt.Sprintf("%d.%s", assetID, ext)})
c.Medias = append(c.Medias, ChannelMessageMedia{
Photo: lo.ToPtr(fmt.Sprintf("%d.%s", assetID, strings.Trim(ext, "."))),
})
return c
}
func (c *ChannelMessage) WithVideo(video string) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{Video: video})
c.Medias = append(c.Medias, ChannelMessageMedia{Video: lo.ToPtr(video)})
return c
}
func (c *ChannelMessage) WithDoc(docID int64, ext string) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{Document: fmt.Sprintf("%d.%s", docID, ext)})
func (c *ChannelMessage) WithDocument(d ChannelMessageDocument) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{
Document: lo.ToPtr(d),
})
return c
}
func (c *ChannelMessage) WithWebPage(title, url string) *ChannelMessage {
c.Medias = append(c.Medias, ChannelMessageMedia{WebPage: ChannelMessageMediaWebPage{Title: title, URL: url}})
c.Medias = append(c.Medias, ChannelMessageMedia{
WebPage: lo.ToPtr(ChannelMessageMediaWebPage{Title: title, URL: url}),
})
return c
}

8
internal/client.go
View File

@@ -17,17 +17,11 @@ import (
type TClient struct {
Config *config.Config
Client *telegram.Client
logger *zap.Logger
api *tg.Client
waitLogin chan error
block chan struct{}
}
func NewClient(logger *zap.Logger, config *config.Config) *TClient {
c := &TClient{
Config: config,
logger: logger,
Client: telegram.NewClient(config.AppID, config.AppHash, telegram.Options{
Logger: logger,
UpdateHandler: nil,
@@ -41,7 +35,7 @@ func NewClient(logger *zap.Logger, config *config.Config) *TClient {
}
func (t *TClient) Login(ctx context.Context) error {
t.logger.Info("login phone", zap.String("phone", t.Config.Phone))
logger.Info("login phone", zap.String("phone", t.Config.Phone))
flow := auth.NewFlow(Terminal{PhoneNumber: client.Config.Phone}, auth.SendCodeOptions{})
if err := t.Client.Auth().IfNecessary(context.Background(), flow); err != nil {
return errors.Wrap(err, "auth")

71
internal/client_channel.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"mime"
"github.com/dustin/go-humanize"
"github.com/gotd/td/telegram/downloader"
"github.com/gotd/td/tg"
"github.com/pkg/errors"
@@ -25,24 +26,21 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
request.MinID = cfg.MinID // 提供此ID供新增加的消息
}
request.Limit = 1
request.OffsetID = 3753
messages, err := t.Client.API().MessagesGetHistory(ctx, request)
if err != nil {
return errors.Wrap(err, "messages.getHistory")
}
if len(messages.(*tg.MessagesChannelMessages).GetMessages()) == 0 {
t.logger.Info("no new message")
logger.Info("no new message")
return errors.New("no new message")
}
downloader := downloader.NewDownloader()
lo.ForEach(messages.(*tg.MessagesChannelMessages).GetMessages(), func(item tg.MessageClass, index int) {
defer func() {
t.logger.Info("update config", zap.Int("offset", cfg.Offset))
logger.Info("update config", zap.Int("offset", cfg.Offset))
if err := cfg.Update(ctx, item.GetID()); err != nil {
t.logger.Error("update config failed", zap.Error(err))
logger.Error("update config failed", zap.Error(err))
}
}()
@@ -55,7 +53,7 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
msg, ok := item.(*tg.Message)
if !ok {
t.logger.Error("convert msg to *tg.Message failed")
logger.Error("convert msg to *tg.Message failed")
return
}
@@ -71,14 +69,18 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
switch mediaClass.(type) {
case *tg.MessageMediaDocument:
if docClass, ok := mediaClass.(*tg.MessageMediaDocument).GetDocument(); ok {
t.logger.Warn("document",
logger.Warn("document",
zap.Int("msg_id", msg.ID),
zap.String("file_name", docClass.String()),
)
doc := docClass.(*tg.Document)
if doc.GetSize() > 1024*1024*10 {
t.logger.Warn("document size too large", zap.Int64("size", doc.GetSize()))
if doc.GetSize() > int64(t.Config.GetMaxSize()) {
logger.Warn(
"document size too large",
zap.Int64("size", doc.GetSize()),
zap.String("SizeHuman", humanize.Bytes(uint64(doc.GetSize()))),
)
return
}
@@ -95,18 +97,51 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
}
exts, err := mime.ExtensionsByType(doc.GetMimeType())
if err != nil {
t.logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType()))
logger.Error("get extension failed", zap.Error(err), zap.String("mime_type", doc.GetMimeType()))
return
}
saveTo := cfg.Asset(doc.GetID(), exts[0])
if len(exts) == 0 {
logger.Warn("no extension found", zap.String("mime_type", doc.GetMimeType()))
switch doc.GetMimeType() {
case "application/rar":
exts = []string{".rar"}
}
}
ext := exts[len(exts)-1]
saveTo := cfg.Asset(doc.GetID(), ext)
_, err = downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo)
if err != nil {
t.logger.Error("download failed", zap.Error(err))
logger.Error("download failed", zap.Error(err))
return
}
channelMessage.WithDoc(doc.GetID(), exts[0])
t.logger.Info("download photo success", zap.String("location", saveTo))
docAttr := doc.GetAttributes()
data := ChannelMessageDocument{
Ext: ext,
MimeType: doc.GetMimeType(),
Size: doc.GetSize(),
}
if len(docAttr) > 0 {
for _, attr := range docAttr {
switch attr.(type) {
case *tg.DocumentAttributeFilename:
data.Filename = attr.(*tg.DocumentAttributeFilename).GetFileName()
case *tg.DocumentAttributeVideo:
m := attr.(*tg.DocumentAttributeVideo)
data.Video = &ChannelMessageDocumentVideo{
Duration: m.GetDuration(),
Width: m.GetW(),
Height: m.GetH(),
}
}
}
}
channelMessage.WithDocument(data)
logger.Info("download document success", zap.String("location", saveTo), zap.Any("document", data))
return
}
case *tg.MessageMediaWebPage:
@@ -114,7 +149,7 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
channelMessage.WithWebPage(page.Title, page.URL)
return
}
t.logger.Warn("web_page", zap.String("url", mediaClass.(*tg.MessageMediaWebPage).GetWebpage().String()))
logger.Warn("web_page", zap.String("url", mediaClass.(*tg.MessageMediaWebPage).GetWebpage().String()))
case *tg.MessageMediaPhoto:
if photoClass, ok := mediaClass.(*tg.MessageMediaPhoto).GetPhoto(); ok {
photo := photoClass.(*tg.Photo)
@@ -134,11 +169,11 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChann
saveTo := cfg.Asset(photo.GetID(), "jpg")
_, err := downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo)
if err != nil {
t.logger.Error("download failed", zap.Error(err))
logger.Error("download failed", zap.Error(err))
return
}
channelMessage.WithPhoto(photo.GetID(), "jpg")
t.logger.Info("download photo success", zap.String("location", saveTo))
logger.Info("download photo success", zap.String("location", saveTo))
}
}
}

3
internal/db_channel.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"exporter/database/telegram_resource/public/model"
@@ -37,7 +38,7 @@ func NewDBChannel(uuid int64, username, title string) *DBChannel {
}
func (c *DBChannel) Asset(assetID int64, ext string) string {
assetFile := fmt.Sprintf("outputs/%d/%d.%s", c.UUID, assetID, ext)
assetFile := fmt.Sprintf("outputs/%d/%d.%s", c.UUID, assetID, strings.Trim(ext, "."))
// if file dir not exists then create it
if _, err := os.Stat(filepath.Dir(assetFile)); os.IsNotExist(err) {

14
main_test.go Normal file
View File

@@ -0,0 +1,14 @@
package main
import (
"mime"
"testing"
)
func Test_Mime(t *testing.T) {
// Test code here
m := "application/rar"
m = "video/mp4"
t.Log(mime.ExtensionsByType(m))
}