diff --git a/.gitignore b/.gitignore index cac4bee..8ecb156 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,6 @@ go.work log.json +log-*.json session.json tdl-export.json \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..819582f --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +.PHONY: model +model: + rm -rf ./database + jet -dsn=postgresql://postgres:xixi0202@10.1.1.3:5432/telegram_resource?sslmode=disable -path=./database + + # gofumpt -w -l -extra ./database \ No newline at end of file diff --git a/config.yml b/config.yml index f84ad35..c90d6a7 100644 --- a/config.yml +++ b/config.yml @@ -3,4 +3,5 @@ app_id: 29433191 app_hash: bc7a2f7b8893889ffa6115f5f0eac278 bot_token: session_file: ./session.json -log_file: ./log.json \ No newline at end of file +log_file: ./log.json +dsn: "postgresql://postgres:xixi0202@10.1.1.3:5432/telegram_resource?sslmode=disable" \ No newline at end of file diff --git a/database/telegram_resource/public/model/channel_messages.go b/database/telegram_resource/public/model/channel_messages.go new file mode 100644 index 0000000..3748699 --- /dev/null +++ b/database/telegram_resource/public/model/channel_messages.go @@ -0,0 +1,22 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "time" +) + +type ChannelMessages struct { + ID int64 `sql:"primary_key"` + ChannelID int64 + UUID int64 + Content *string + Media string + PublishedAt time.Time + CreatedAt time.Time +} diff --git a/database/telegram_resource/public/model/channels.go b/database/telegram_resource/public/model/channels.go new file mode 100644 index 0000000..08c2c38 --- /dev/null +++ b/database/telegram_resource/public/model/channels.go @@ -0,0 +1,23 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "time" +) + +type Channels struct { + ID int64 `sql:"primary_key"` + UUID int64 + Username string + Title string + CreatedAt *time.Time + UpdatedAt *time.Time + Offset int64 + MinID int64 +} diff --git a/database/telegram_resource/public/table/channel_messages.go b/database/telegram_resource/public/table/channel_messages.go new file mode 100644 index 0000000..69f6466 --- /dev/null +++ b/database/telegram_resource/public/table/channel_messages.go @@ -0,0 +1,93 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var ChannelMessages = newChannelMessagesTable("public", "channel_messages", "") + +type channelMessagesTable struct { + postgres.Table + + // Columns + ID postgres.ColumnInteger + ChannelID postgres.ColumnInteger + UUID postgres.ColumnInteger + Content postgres.ColumnString + Media postgres.ColumnString + PublishedAt postgres.ColumnTimestampz + CreatedAt postgres.ColumnTimestampz + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type ChannelMessagesTable struct { + channelMessagesTable + + EXCLUDED channelMessagesTable +} + +// AS creates new ChannelMessagesTable with assigned alias +func (a ChannelMessagesTable) AS(alias string) *ChannelMessagesTable { + return newChannelMessagesTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new ChannelMessagesTable with assigned schema name +func (a ChannelMessagesTable) FromSchema(schemaName string) *ChannelMessagesTable { + return newChannelMessagesTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new ChannelMessagesTable with assigned table prefix +func (a ChannelMessagesTable) WithPrefix(prefix string) *ChannelMessagesTable { + return newChannelMessagesTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new ChannelMessagesTable with assigned table suffix +func (a ChannelMessagesTable) WithSuffix(suffix string) *ChannelMessagesTable { + return newChannelMessagesTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newChannelMessagesTable(schemaName, tableName, alias string) *ChannelMessagesTable { + return &ChannelMessagesTable{ + channelMessagesTable: newChannelMessagesTableImpl(schemaName, tableName, alias), + EXCLUDED: newChannelMessagesTableImpl("", "excluded", ""), + } +} + +func newChannelMessagesTableImpl(schemaName, tableName, alias string) channelMessagesTable { + var ( + IDColumn = postgres.IntegerColumn("id") + ChannelIDColumn = postgres.IntegerColumn("channel_id") + UUIDColumn = postgres.IntegerColumn("uuid") + ContentColumn = postgres.StringColumn("content") + MediaColumn = postgres.StringColumn("media") + PublishedAtColumn = postgres.TimestampzColumn("published_at") + CreatedAtColumn = postgres.TimestampzColumn("created_at") + allColumns = postgres.ColumnList{IDColumn, ChannelIDColumn, UUIDColumn, ContentColumn, MediaColumn, PublishedAtColumn, CreatedAtColumn} + mutableColumns = postgres.ColumnList{ChannelIDColumn, UUIDColumn, ContentColumn, MediaColumn, PublishedAtColumn, CreatedAtColumn} + ) + + return channelMessagesTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + ChannelID: ChannelIDColumn, + UUID: UUIDColumn, + Content: ContentColumn, + Media: MediaColumn, + PublishedAt: PublishedAtColumn, + CreatedAt: CreatedAtColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/database/telegram_resource/public/table/channels.go b/database/telegram_resource/public/table/channels.go new file mode 100644 index 0000000..1c94557 --- /dev/null +++ b/database/telegram_resource/public/table/channels.go @@ -0,0 +1,96 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var Channels = newChannelsTable("public", "channels", "") + +type channelsTable struct { + postgres.Table + + // Columns + ID postgres.ColumnInteger + UUID postgres.ColumnInteger + Username postgres.ColumnString + Title postgres.ColumnString + CreatedAt postgres.ColumnTimestampz + UpdatedAt postgres.ColumnTimestampz + Offset postgres.ColumnInteger + MinID postgres.ColumnInteger + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type ChannelsTable struct { + channelsTable + + EXCLUDED channelsTable +} + +// AS creates new ChannelsTable with assigned alias +func (a ChannelsTable) AS(alias string) *ChannelsTable { + return newChannelsTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new ChannelsTable with assigned schema name +func (a ChannelsTable) FromSchema(schemaName string) *ChannelsTable { + return newChannelsTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new ChannelsTable with assigned table prefix +func (a ChannelsTable) WithPrefix(prefix string) *ChannelsTable { + return newChannelsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new ChannelsTable with assigned table suffix +func (a ChannelsTable) WithSuffix(suffix string) *ChannelsTable { + return newChannelsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newChannelsTable(schemaName, tableName, alias string) *ChannelsTable { + return &ChannelsTable{ + channelsTable: newChannelsTableImpl(schemaName, tableName, alias), + EXCLUDED: newChannelsTableImpl("", "excluded", ""), + } +} + +func newChannelsTableImpl(schemaName, tableName, alias string) channelsTable { + var ( + IDColumn = postgres.IntegerColumn("id") + UUIDColumn = postgres.IntegerColumn("uuid") + UsernameColumn = postgres.StringColumn("username") + TitleColumn = postgres.StringColumn("title") + CreatedAtColumn = postgres.TimestampzColumn("created_at") + UpdatedAtColumn = postgres.TimestampzColumn("updated_at") + OffsetColumn = postgres.IntegerColumn("offset") + MinIDColumn = postgres.IntegerColumn("min_id") + allColumns = postgres.ColumnList{IDColumn, UUIDColumn, UsernameColumn, TitleColumn, CreatedAtColumn, UpdatedAtColumn, OffsetColumn, MinIDColumn} + mutableColumns = postgres.ColumnList{UUIDColumn, UsernameColumn, TitleColumn, CreatedAtColumn, UpdatedAtColumn, OffsetColumn, MinIDColumn} + ) + + return channelsTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + UUID: UUIDColumn, + Username: UsernameColumn, + Title: TitleColumn, + CreatedAt: CreatedAtColumn, + UpdatedAt: UpdatedAtColumn, + Offset: OffsetColumn, + MinID: MinIDColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/database/telegram_resource/public/table/table_use_schema.go b/database/telegram_resource/public/table/table_use_schema.go new file mode 100644 index 0000000..b254913 --- /dev/null +++ b/database/telegram_resource/public/table/table_use_schema.go @@ -0,0 +1,15 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +// UseSchema sets a new schema name for all generated table SQL builder types. It is recommended to invoke +// this method only once at the beginning of the program. +func UseSchema(schema string) { + ChannelMessages = ChannelMessages.FromSchema(schema) + Channels = Channels.FromSchema(schema) +} diff --git a/go.mod b/go.mod index deae250..88281cb 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module exporter go 1.22.1 require ( + github.com/go-jet/jet v2.3.0+incompatible + github.com/go-jet/jet/v2 v2.11.1 github.com/gotd/td v0.107.0 + github.com/lib/pq v1.10.9 github.com/pkg/errors v0.9.1 github.com/samber/lo v1.47.0 - github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 go.uber.org/zap v1.27.0 @@ -16,10 +18,12 @@ require ( require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-faster/jx v1.1.0 // indirect github.com/go-faster/xor v1.0.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gotd/ige v0.2.2 // indirect github.com/gotd/neo v0.1.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -28,6 +32,7 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect @@ -35,6 +40,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect diff --git a/go.sum b/go.sum index a315dce..9852e97 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,14 @@ github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ= github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38= github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ= +github.com/go-jet/jet v2.3.0+incompatible h1:Yg7JSERDC0f9x3dHUBMA2cxe9/qC6qlozDDO/s38USU= +github.com/go-jet/jet v2.3.0+incompatible/go.mod h1:XgTt00fj8pAXMKe1ETL9R/kZWWyi2j/ymuH+gaW+EdI= +github.com/go-jet/jet/v2 v2.11.1 h1:SEbh2lRUIiQweJpV0boWsQ4bV13x9p4h+RfajnL6vgM= +github.com/go-jet/jet/v2 v2.11.1/go.mod h1:+DTofDkGp1c0vpooXWEZyNhyi0k0mL7N2W9tdP4YqfA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk= github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0= github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ= @@ -34,6 +40,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -56,8 +64,6 @@ github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -74,7 +80,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -103,7 +108,6 @@ golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= diff --git a/internal/channel_message.go b/internal/channel_message.go index 0c0019e..7efce43 100644 --- a/internal/channel_message.go +++ b/internal/channel_message.go @@ -1,11 +1,16 @@ package internal -import "fmt" +import ( + "encoding/json" + "fmt" + "time" +) type ChannelMessage struct { - ID int - Message string - Medias []ChannelMessageMedia + ID int + Message string + Medias []ChannelMessageMedia + PublishAt time.Time } type ChannelMessageMedia struct { @@ -13,8 +18,8 @@ type ChannelMessageMedia struct { Video string } -func NewChannelMessage(id int) *ChannelMessage { - return &ChannelMessage{ID: id} +func NewChannelMessage(id, ts int) *ChannelMessage { + return &ChannelMessage{ID: id, PublishAt: time.Unix(int64(ts), 0)} } func (c *ChannelMessage) WithMessage(message string) *ChannelMessage { @@ -31,3 +36,8 @@ func (c *ChannelMessage) WithVideo(video string) *ChannelMessage { c.Medias = append(c.Medias, ChannelMessageMedia{Video: video}) return c } + +func (c *ChannelMessage) GetMedia() string { + b, _ := json.Marshal(c.Medias) + return string(b) +} diff --git a/internal/client_channel.go b/internal/client_channel.go index de1f3c8..cbed244 100644 --- a/internal/client_channel.go +++ b/internal/client_channel.go @@ -10,7 +10,7 @@ import ( "go.uber.org/zap" ) -func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *ChannelConfig, modeHistory bool) error { +func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *DBChannel, modeHistory bool) error { inputPeer := &tg.InputPeerChannel{ChannelID: channel.ID, AccessHash: channel.AccessHash} request := &tg.MessagesGetHistoryRequest{ @@ -55,8 +55,8 @@ func (t *TClient) Channel(ctx context.Context, channel *tg.Channel, cfg *Channel return } - channelMessage := NewChannelMessage(msg.ID) - defer cfg.SaveMessage(channelMessage) + channelMessage := NewChannelMessage(msg.ID, msg.GetDate()) + defer cfg.SaveMessage(ctx, channelMessage) channelMessage.WithMessage(msg.GetMessage()) diff --git a/internal/client_channel_config.go b/internal/client_channel_config.go deleted file mode 100644 index fe239ed..0000000 --- a/internal/client_channel_config.go +++ /dev/null @@ -1,119 +0,0 @@ -package internal - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -type ChannelConfig struct { - ID int64 `json:"id"` - Offset int `json:"offset"` - MinID int `json:"min_id"` -} - -func NewChannelConfig(channelID int64) *ChannelConfig { - if channelID == 0 { - panic("channel id is required") - } - return &ChannelConfig{ID: channelID} -} - -func (c *ChannelConfig) Asset(assetID int64, ext string) string { - assetFile := fmt.Sprintf("outputs/%d/assets/%d.%s", c.ID, assetID, ext) - - // if file dir not exists then create it - if _, err := os.Stat(filepath.Dir(assetFile)); os.IsNotExist(err) { - if err := os.MkdirAll(filepath.Dir(assetFile), 0o755); err != nil { - panic(err) - } - } - - // if file exists then delete it - if _, err := os.Stat(assetFile); err == nil { - if err := os.Remove(assetFile); err != nil { - panic(err) - } - } - - return assetFile -} - -func (c *ChannelConfig) file(_ context.Context) (string, error) { - channelConfigFile := fmt.Sprintf("outputs/%d/config.json", c.ID) - - // if file dir not exists then create it - if _, err := os.Stat(filepath.Dir(channelConfigFile)); os.IsNotExist(err) { - if err := os.MkdirAll(filepath.Dir(channelConfigFile), 0o755); err != nil { - return "", errors.Wrap(err, "create channel config dir") - } - } - - // if file not exists then create it - if _, err := os.Stat(channelConfigFile); os.IsNotExist(err) { - // create config file - data, _ := json.Marshal(c) - if err := os.WriteFile(channelConfigFile, data, 0o644); err != nil { - return "", errors.Wrap(err, "write channel config") - } - } - return channelConfigFile, nil -} - -func (c *ChannelConfig) Read(ctx context.Context) error { - channelConfigFile, err := c.file(ctx) - if err != nil { - return err - } - // read config file - data, err := os.ReadFile(channelConfigFile) - if err != nil { - return errors.Wrap(err, "read channel config") - } - - if err := json.Unmarshal(data, &c); err != nil { - return errors.Wrap(err, "unmarshal channel config") - } - - return nil -} - -func (c *ChannelConfig) Update(ctx context.Context, offsetID int) error { - channelConfigFile, err := c.file(ctx) - if err != nil { - return err - } - - c.Offset = offsetID - - if c.MinID < offsetID { - c.MinID = offsetID - } - - b, err := json.Marshal(c) - if err != nil { - return errors.Wrap(err, "marshal channel config") - } - - if err := os.WriteFile(channelConfigFile, b, 0o644); err != nil { - return errors.Wrap(err, "write channel config") - } - - return nil -} - -func (c *ChannelConfig) SaveMessage(msg *ChannelMessage) { - // save message - saveTo := fmt.Sprintf("outputs/%d/%d.json", c.ID, msg.ID) - b, err := json.Marshal(msg) - if err != nil { - panic(err) - } - if err := os.WriteFile(saveTo, b, 0o644); err != nil { - panic(err) - } -} diff --git a/internal/cmd_export.go b/internal/cmd_export.go index 27d3f3a..6918218 100644 --- a/internal/cmd_export.go +++ b/internal/cmd_export.go @@ -51,8 +51,8 @@ func exportCmd(ctx context.Context) error { return errors.New("channel not found") } - cfg := NewChannelConfig(channel.GetID()) - if err := cfg.Read(ctx); err != nil { + cfg := NewDBChannel(channel.GetID(), channel.Username, channel.Title) + if err := cfg.Get(ctx); err != nil { return err } diff --git a/internal/common.go b/internal/common.go index af99dbc..3fd8a40 100644 --- a/internal/common.go +++ b/internal/common.go @@ -37,6 +37,9 @@ func InitClient(cfg *config.Config) error { func Close() error { _ = logger.Sync() + if db != nil { + db.Close() + } return nil } diff --git a/internal/db.go b/internal/db.go new file mode 100644 index 0000000..55897a1 --- /dev/null +++ b/internal/db.go @@ -0,0 +1,18 @@ +package internal + +import ( + "database/sql" + + _ "github.com/lib/pq" +) + +var db *sql.DB + +func InitDB(dsn string) error { + var err error + db, err = sql.Open("postgres", dsn) + if err != nil { + return err + } + return db.Ping() +} diff --git a/internal/db_channel.go b/internal/db_channel.go new file mode 100644 index 0000000..495d462 --- /dev/null +++ b/internal/db_channel.go @@ -0,0 +1,126 @@ +package internal + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "time" + + "exporter/database/telegram_resource/public/model" + "exporter/database/telegram_resource/public/table" + + . "github.com/go-jet/jet/v2/postgres" + "github.com/pkg/errors" + "github.com/samber/lo" +) + +type DBChannel struct { + UUID int64 + Username string + Title string + Offset int + MinID int +} + +func NewDBChannel(uuid int64, username, title string) *DBChannel { + if uuid == 0 { + panic("channel id is required") + } + return &DBChannel{ + UUID: uuid, + Username: username, + Title: title, + } +} + +func (c *DBChannel) Asset(assetID int64, ext string) string { + assetFile := fmt.Sprintf("outputs/%d/%d.%s", c.UUID, assetID, ext) + + // if file dir not exists then create it + if _, err := os.Stat(filepath.Dir(assetFile)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(assetFile), 0o755); err != nil { + panic(err) + } + } + + // if file exists then delete it + if _, err := os.Stat(assetFile); err == nil { + if err := os.Remove(assetFile); err != nil { + panic(err) + } + } + + return assetFile +} + +func (c *DBChannel) Get(ctx context.Context) error { + tbl := table.Channels + + var m *model.Channels + + err := tbl.SELECT(tbl.AllColumns).QueryContext(ctx, db, &m) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + // create new channel with default value + m = &model.Channels{ + UUID: c.UUID, + Username: c.Username, + Title: c.Title, + MinID: 0, + Offset: 0, + CreatedAt: lo.ToPtr(time.Now()), + UpdatedAt: lo.ToPtr(time.Now()), + } + + if _, err := tbl.INSERT(tbl.AllColumns).MODEL(m).ExecContext(ctx, db); err != nil { + return errors.Wrap(err, "insert channel") + } + } else { + return errors.Wrap(err, "select channel") + } + } + c.MinID = int(m.MinID) + c.Offset = int(m.Offset) + + return nil +} + +func (c *DBChannel) Update(ctx context.Context, offsetID int) error { + c.Offset = offsetID + + if c.MinID < offsetID { + c.MinID = offsetID + } + + tbl := table.Channels + _, err := tbl.UPDATE().SET( + tbl.Offset.SET(Int(int64(c.Offset))), + tbl.MinID.SET(Int(int64(c.MinID))), + ).ExecContext(ctx, db) + if err != nil { + return errors.Wrap(err, "update channel") + } + + return nil +} + +func (c *DBChannel) SaveMessage(ctx context.Context, msg *ChannelMessage) error { + message := &model.ChannelMessages{ + ChannelID: c.UUID, + UUID: int64(msg.ID), + Content: lo.ToPtr(msg.Message), + Media: msg.GetMedia(), + PublishedAt: msg.PublishAt, + CreatedAt: time.Now(), + } + + tbl := table.ChannelMessages + + _, err := tbl.INSERT(tbl.AllColumns).MODEL(message).ExecContext(ctx, db) + if err != nil { + return errors.Wrap(err, "insert message") + } + return nil +} diff --git a/internal/db_test.go b/internal/db_test.go new file mode 100644 index 0000000..0ce93d2 --- /dev/null +++ b/internal/db_test.go @@ -0,0 +1,36 @@ +package internal + +import ( + "context" + "testing" + "time" + + "exporter/database/telegram_resource/public/model" + "exporter/database/telegram_resource/public/table" + + "github.com/samber/lo" +) + +func TestInitDB(t *testing.T) { + dsn := "postgresql://postgres:xixi0202@10.1.1.3:5432/telegram_resource?sslmode=disable" + // dns1 := "host=%s user=%s password=%s dbname=%s port=%s sslmode=%s TimeZone=%s" + if err := InitDB(dsn); err != nil { + t.Fatalf("InitDB() error = %v", err) + } + t.Logf("InitDB() success") + + stmt := table.Channels.INSERT(table.Channels.AllColumns).MODEL(model.Channels{ + ID: 1, + Username: "test", + Title: "helo", + CreatedAt: lo.ToPtr(time.Now()), + UpdatedAt: lo.ToPtr(time.Now()), + Offset: 10, + MinID: 11, + }) + t.Logf("sql: %v", stmt.DebugSql()) + + if _, err := stmt.ExecContext(context.Background(), db); err != nil { + t.Fatalf("stmt.ExecContext() error = %v", err) + } +}