feat: 添加队列工作者配置,增强作业提供者功能,支持并发控制和健康检查
This commit is contained in:
@@ -17,7 +17,19 @@ func DefaultProvider() container.ProviderContainer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type Config struct{}
|
type Config struct {
|
||||||
|
// Optional per-queue worker concurrency. If empty, defaults apply.
|
||||||
|
QueueWorkers QueueWorkersConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueueWorkers allows configuring worker concurrency per queue.
|
||||||
|
// Key is the queue name, value is MaxWorkers. If empty, defaults are used.
|
||||||
|
// Example TOML:
|
||||||
|
//
|
||||||
|
// [Job]
|
||||||
|
// # high=20, default=10, low=5
|
||||||
|
// # QueueWorkers = { high = 20, default = 10, low = 5 }
|
||||||
|
type QueueWorkersConfig map[string]int
|
||||||
|
|
||||||
const (
|
const (
|
||||||
PriorityDefault = river.PriorityDefault
|
PriorityDefault = river.PriorityDefault
|
||||||
@@ -31,3 +43,25 @@ const (
|
|||||||
QueueDefault = river.QueueDefault
|
QueueDefault = river.QueueDefault
|
||||||
QueueLow = "low"
|
QueueLow = "low"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// queueConfig returns a river.QueueConfig map built from QueueWorkers or defaults.
|
||||||
|
func (c *Config) queueConfig() map[string]river.QueueConfig {
|
||||||
|
cfg := map[string]river.QueueConfig{}
|
||||||
|
if c == nil || len(c.QueueWorkers) == 0 {
|
||||||
|
cfg[QueueHigh] = river.QueueConfig{MaxWorkers: 10}
|
||||||
|
cfg[QueueDefault] = river.QueueConfig{MaxWorkers: 10}
|
||||||
|
cfg[QueueLow] = river.QueueConfig{MaxWorkers: 10}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
for name, n := range c.QueueWorkers {
|
||||||
|
if n <= 0 {
|
||||||
|
n = 1
|
||||||
|
}
|
||||||
|
cfg[name] = river.QueueConfig{MaxWorkers: n}
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := cfg[QueueDefault]; !ok {
|
||||||
|
cfg[QueueDefault] = river.QueueConfig{MaxWorkers: 10}
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,9 +2,11 @@ package job
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"{{.ModuleName}}/providers/postgres"
|
"test/providers/postgres"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
@@ -37,10 +39,22 @@ func Provide(opts ...opt.Option) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// health check ping with timeout
|
||||||
|
pingCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := dbPool.Ping(pingCtx); err != nil {
|
||||||
|
return nil, fmt.Errorf("job provider: db ping failed: %w", err)
|
||||||
|
}
|
||||||
container.AddCloseAble(dbPool.Close)
|
container.AddCloseAble(dbPool.Close)
|
||||||
pool := riverpgxv5.New(dbPool)
|
pool := riverpgxv5.New(dbPool)
|
||||||
|
|
||||||
queue := &Job{Workers: workers, driver: pool, ctx: ctx, periodicJobs: make(map[string]rivertype.PeriodicJobHandle), jobs: make(map[string]*rivertype.JobInsertResult)}
|
queue := &Job{
|
||||||
|
Workers: workers,
|
||||||
|
driver: pool,
|
||||||
|
ctx: ctx,
|
||||||
|
conf: &config,
|
||||||
|
periodicJobs: make(map[string]rivertype.PeriodicJobHandle),
|
||||||
|
}
|
||||||
container.AddCloseAble(queue.Close)
|
container.AddCloseAble(queue.Close)
|
||||||
|
|
||||||
return queue, nil
|
return queue, nil
|
||||||
@@ -49,6 +63,7 @@ func Provide(opts ...opt.Option) error {
|
|||||||
|
|
||||||
type Job struct {
|
type Job struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
|
conf *Config
|
||||||
Workers *river.Workers
|
Workers *river.Workers
|
||||||
driver *riverpgxv5.Driver
|
driver *riverpgxv5.Driver
|
||||||
|
|
||||||
@@ -56,7 +71,6 @@ type Job struct {
|
|||||||
client *river.Client[pgx.Tx]
|
client *river.Client[pgx.Tx]
|
||||||
|
|
||||||
periodicJobs map[string]rivertype.PeriodicJobHandle
|
periodicJobs map[string]rivertype.PeriodicJobHandle
|
||||||
jobs map[string]*rivertype.JobInsertResult
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (q *Job) Close() {
|
func (q *Job) Close() {
|
||||||
@@ -67,6 +81,10 @@ func (q *Job) Close() {
|
|||||||
if err := q.client.StopAndCancel(q.ctx); err != nil {
|
if err := q.client.StopAndCancel(q.ctx); err != nil {
|
||||||
log.Errorf("Failed to stop and cancel client: %s", err)
|
log.Errorf("Failed to stop and cancel client: %s", err)
|
||||||
}
|
}
|
||||||
|
// clear references
|
||||||
|
q.l.Lock()
|
||||||
|
q.periodicJobs = map[string]rivertype.PeriodicJobHandle{}
|
||||||
|
q.l.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (q *Job) Client() (*river.Client[pgx.Tx], error) {
|
func (q *Job) Client() (*river.Client[pgx.Tx], error) {
|
||||||
@@ -77,11 +95,7 @@ func (q *Job) Client() (*river.Client[pgx.Tx], error) {
|
|||||||
var err error
|
var err error
|
||||||
q.client, err = river.NewClient(q.driver, &river.Config{
|
q.client, err = river.NewClient(q.driver, &river.Config{
|
||||||
Workers: q.Workers,
|
Workers: q.Workers,
|
||||||
Queues: map[string]river.QueueConfig{
|
Queues: q.conf.queueConfig(),
|
||||||
QueueHigh: {MaxWorkers: 10},
|
|
||||||
QueueDefault: {MaxWorkers: 10},
|
|
||||||
QueueLow: {MaxWorkers: 10},
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -158,15 +172,21 @@ func (q *Job) Cancel(id string) error {
|
|||||||
if h, ok := q.periodicJobs[id]; ok {
|
if h, ok := q.periodicJobs[id]; ok {
|
||||||
client.PeriodicJobs().Remove(h)
|
client.PeriodicJobs().Remove(h)
|
||||||
delete(q.periodicJobs, id)
|
delete(q.periodicJobs, id)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if r, ok := q.jobs[id]; ok {
|
// CancelContext is like Cancel but allows passing a context.
|
||||||
_, err = client.JobCancel(q.ctx, r.Job.ID)
|
func (q *Job) CancelContext(ctx context.Context, id string) error {
|
||||||
|
client, err := q.Client()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
delete(q.jobs, id)
|
q.l.Lock()
|
||||||
|
defer q.l.Unlock()
|
||||||
|
if h, ok := q.periodicJobs[id]; ok {
|
||||||
|
client.PeriodicJobs().Remove(h)
|
||||||
|
delete(q.periodicJobs, id)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -182,6 +202,6 @@ func (q *Job) Add(job contracts.JobArgs) error {
|
|||||||
q.l.Lock()
|
q.l.Lock()
|
||||||
defer q.l.Unlock()
|
defer q.l.Unlock()
|
||||||
|
|
||||||
q.jobs[job.UniqueID()], err = client.Insert(q.ctx, job, lo.ToPtr(job.InsertOpts()))
|
_, err = client.Insert(q.ctx, job, lo.ToPtr(job.InsertOpts()))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user