From 73ce8585bb08cead4d6794697eabeb5e0a7d1e3b Mon Sep 17 00:00:00 2001 From: Rogee Date: Mon, 2 Sep 2024 16:07:14 +0800 Subject: [PATCH] fix: issues --- .vscode/launch.json | 4 +- internal/channel_message.go | 33 +++++++++ internal/client.go | 11 --- internal/client_channel.go | 81 ++++++++------------- internal/client_channel_config.go | 116 ++++++++++++++++++++++++++++++ internal/cmd_export.go | 22 +++--- main.go | 8 +++ outputs/.gitignore | 2 + 8 files changed, 201 insertions(+), 76 deletions(-) create mode 100644 internal/channel_message.go create mode 100644 internal/client_channel_config.go create mode 100644 outputs/.gitignore diff --git a/.vscode/launch.json b/.vscode/launch.json index 625fb81..994be44 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,9 @@ "mode": "auto", "program": "${workspaceFolder}/main.go", "args": [ - "login" + "export", + "--alias", + "yunpanshare" ] } ] diff --git a/internal/channel_message.go b/internal/channel_message.go new file mode 100644 index 0000000..0c0019e --- /dev/null +++ b/internal/channel_message.go @@ -0,0 +1,33 @@ +package internal + +import "fmt" + +type ChannelMessage struct { + ID int + Message string + Medias []ChannelMessageMedia +} + +type ChannelMessageMedia struct { + Photo string + Video string +} + +func NewChannelMessage(id int) *ChannelMessage { + return &ChannelMessage{ID: id} +} + +func (c *ChannelMessage) WithMessage(message string) *ChannelMessage { + c.Message = message + return c +} + +func (c *ChannelMessage) WithPhoto(assetID int64, ext string) *ChannelMessage { + c.Medias = append(c.Medias, ChannelMessageMedia{Photo: fmt.Sprintf("%d.%s", assetID, ext)}) + return c +} + +func (c *ChannelMessage) WithVideo(video string) *ChannelMessage { + c.Medias = append(c.Medias, ChannelMessageMedia{Video: video}) + return c +} diff --git a/internal/client.go b/internal/client.go index 93763bb..a6da0a6 100644 --- a/internal/client.go +++ b/internal/client.go @@ -20,16 +20,10 @@ type TClient struct { logger *zap.Logger api *tg.Client - downloadMedia bool - waitLogin chan error block chan struct{} } -// func (t *TClient) Close() { -// t.block <- struct{}{} -// } - func NewClient(logger *zap.Logger, config *config.Config) *TClient { c := &TClient{ Config: config, @@ -46,11 +40,6 @@ func NewClient(logger *zap.Logger, config *config.Config) *TClient { return c } -func (t *TClient) WithMedia() *TClient { - t.downloadMedia = true - return t -} - func (t *TClient) Login(ctx context.Context) error { t.logger.Info("login phone", zap.String("phone", t.Config.Phone)) flow := auth.NewFlow(Terminal{PhoneNumber: client.Config.Phone}, auth.SendCodeOptions{}) diff --git a/internal/client_channel.go b/internal/client_channel.go index 7ba9db1..bd11424 100644 --- a/internal/client_channel.go +++ b/internal/client_channel.go @@ -2,24 +2,29 @@ package internal import ( "context" - "encoding/json" - "fmt" - "os" - "path/filepath" "github.com/gotd/td/telegram/downloader" "github.com/gotd/td/tg" "github.com/pkg/errors" "github.com/samber/lo" + "go.uber.org/zap" ) -func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, offset int) error { +func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *ChannelConfig, modeHistory bool) error { inputPeer := &tg.InputPeerChannel{ChannelID: channel.ID, AccessHash: channel.AccessHash} - messages, err := t.api.MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{ - Peer: inputPeer, - Limit: 10, - OffsetID: offset, - }) + + request := &tg.MessagesGetHistoryRequest{ + Peer: inputPeer, + Limit: 1, + } + + if modeHistory { // 提供此ID供遍历历史消息 + request.OffsetID = cfg.Offset + } else { + request.MinID = cfg.MinID // 提供此ID供新增加的消息 + } + + messages, err := t.Client.API().MessagesGetHistory(ctx, request) if err != nil { return errors.Wrap(err, "messages.getHistory") } @@ -28,13 +33,19 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, offset int) lo.ForEach(messages.(*tg.MessagesChannelMessages).GetMessages(), func(item tg.MessageClass, index int) { msg, ok := item.(*tg.Message) if !ok { - fmt.Println("ID: get failed") + t.logger.Error("convert msg to *tg.Message failed") return } + defer func() { + if err := cfg.Update(ctx, msg.ID); err != nil { + t.logger.Error("update config failed", zap.Error(err)) + } + }() - if !t.downloadMedia { - return - } + channelMessage := NewChannelMessage(msg.ID) + defer cfg.SaveMessage(channelMessage) + + channelMessage.WithMessage(msg.GetMessage()) if mediaClass, ok := msg.GetMedia(); ok { if photoClass, ok := mediaClass.(*tg.MessageMediaPhoto).GetPhoto(); ok { @@ -52,47 +63,17 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, offset int) ThumbSize: thumbSize, } - saveTo := lo.Must(filepath.Abs(fmt.Sprintf("./photos/%d.jpg", photo.GetID()))) - storage, err := downloader.Download(t.api, location).ToPath(ctx, saveTo) - + saveTo := cfg.Asset(photo.GetID(), "jpg") + _, err := downloader.Download(t.Client.API(), location).ToPath(ctx, saveTo) if err != nil { - fmt.Println(err) - } else { - fmt.Println("Downloaded : ", storage) + t.logger.Error("download failed", zap.Error(err)) + return } + channelMessage.WithPhoto(photo.GetID(), "jpg") + t.logger.Info("download failed", zap.String("asset", saveTo)) } } }) return nil } - -type ChannelConfig struct { - ID int64 `json:"id"` - Offset int `json:"offset"` -} - -func (t *TClient) SaveChannelConfig(ctx context.Context, channelID int64, offset int) (*ChannelConfig, error) { - channelConfigFile := fmt.Sprintf("outputs/%d/config.json", channelID) - // if file not exists then create it - if _, err := os.Stat(channelConfigFile); os.IsNotExist(err) { - // create config file - data, _ := json.Marshal(&ChannelConfig{ID: channelID}) - if err := os.WriteFile(channelConfigFile, data, 0o644); err != nil { - return nil, errors.Wrap(err, "write channel config") - } - } - - // read config file - data, err := os.ReadFile(channelConfigFile) - if err != nil { - return nil, errors.Wrap(err, "read channel config") - } - - var config *ChannelConfig - if err := json.Unmarshal(data, config); err != nil { - return nil, errors.Wrap(err, "unmarshal channel config") - } - - return config, nil -} diff --git a/internal/client_channel_config.go b/internal/client_channel_config.go new file mode 100644 index 0000000..3536881 --- /dev/null +++ b/internal/client_channel_config.go @@ -0,0 +1,116 @@ +package internal + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +type ChannelConfig struct { + ID int64 `json:"id"` + Offset int `json:"offset"` + MinID int `json:"min_id"` +} + +func NewChannelConfig(channelID int64) *ChannelConfig { + return &ChannelConfig{ID: channelID} +} + +func (c *ChannelConfig) Asset(assetID int64, ext string) string { + assetFile := fmt.Sprintf("outputs/%d/assets/%d.%s", c.ID, assetID, ext) + + // if file dir not exists then create it + if _, err := os.Stat(filepath.Dir(assetFile)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(assetFile), 0o755); err != nil { + panic(err) + } + } + + // if file exists then delete it + if _, err := os.Stat(assetFile); err == nil { + if err := os.Remove(assetFile); err != nil { + panic(err) + } + } + + return assetFile +} + +func (c *ChannelConfig) file(_ context.Context) (string, error) { + channelConfigFile := fmt.Sprintf("outputs/%d/config.json", c.ID) + + // if file dir not exists then create it + if _, err := os.Stat(filepath.Dir(channelConfigFile)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(channelConfigFile), 0o755); err != nil { + return "", errors.Wrap(err, "create channel config dir") + } + } + + // if file not exists then create it + if _, err := os.Stat(channelConfigFile); os.IsNotExist(err) { + // create config file + data, _ := json.Marshal(&ChannelConfig{ID: channelID}) + if err := os.WriteFile(channelConfigFile, data, 0o644); err != nil { + return "", errors.Wrap(err, "write channel config") + } + } + return channelConfigFile, nil +} + +func (c *ChannelConfig) Read(ctx context.Context) error { + channelConfigFile, err := c.file(ctx) + if err != nil { + return err + } + // read config file + data, err := os.ReadFile(channelConfigFile) + if err != nil { + return errors.Wrap(err, "read channel config") + } + + if err := json.Unmarshal(data, &c); err != nil { + return errors.Wrap(err, "unmarshal channel config") + } + + return nil +} + +func (c *ChannelConfig) Update(ctx context.Context, offsetID int) error { + channelConfigFile, err := c.file(ctx) + if err != nil { + return err + } + + c.Offset = offsetID + + if c.MinID < offsetID { + c.MinID = offsetID + } + + b, err := json.Marshal(c) + if err != nil { + return errors.Wrap(err, "marshal channel config") + } + + if err := os.WriteFile(channelConfigFile, b, 0o644); err != nil { + return errors.Wrap(err, "write channel config") + } + + return nil +} + +func (c *ChannelConfig) SaveMessage(msg *ChannelMessage) { + // save message + saveTo := fmt.Sprintf("outputs/%d/%d.json", c.ID, msg.ID) + b, err := json.Marshal(msg) + if err != nil { + panic(err) + } + if err := os.WriteFile(saveTo, b, 0o644); err != nil { + panic(err) + } +} diff --git a/internal/cmd_export.go b/internal/cmd_export.go index c8479a3..760f343 100644 --- a/internal/cmd_export.go +++ b/internal/cmd_export.go @@ -9,10 +9,9 @@ import ( ) var ( - channelID int64 - offsetID int - channelAlias string - downloadMedia bool + channelID int64 + modeHistory bool + channelAlias string ) func ExportCmd() *cobra.Command { @@ -23,9 +22,8 @@ func ExportCmd() *cobra.Command { } cmd.Flags().Int64Var(&channelID, "channel", 0, "channel id") - cmd.Flags().IntVar(&offsetID, "offset", 0, "offset id") + cmd.Flags().BoolVar(&modeHistory, "history", false, "history mode") cmd.Flags().StringVar(&channelAlias, "alias", "", "channel alias") - cmd.Flags().BoolVar(&downloadMedia, "media", false, "download media") return cmd } @@ -49,15 +47,11 @@ func exportCmd(ctx context.Context) error { } } - if downloadMedia { - client.WithMedia() - } - - cfg, err := client.SaveChannelConfig(ctx, channel.ID, offsetID) - if err != nil { + cfg := NewChannelConfig(channel.ID) + if err := cfg.Read(ctx); err != nil { return err } - return nil - return client.Channel(ctx, channel, cfg.Offset) + // https://t.me/yunpanshare/37426 + return client.Channel(ctx, channel, cfg, modeHistory) } diff --git a/main.go b/main.go index c8d4b18..51f0aee 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,14 @@ func main() { internal.ExportCmd(), ) + // rootCmd.SilenceErrors = true + rootCmd.SilenceUsage = true + rootCmd.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error { + cmd.Println(err) + cmd.Println(cmd.UsageString()) + return err + }) + err := rootCmd.Execute() if err != nil { os.Exit(1) diff --git a/outputs/.gitignore b/outputs/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/outputs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file