feat(worker): add durable background jobs
This commit is contained in:
@@ -242,6 +242,7 @@ func autoMigrate(db *gorm.DB) error {
|
||||
&model.EmailTemplate{},
|
||||
&model.MessageReaction{},
|
||||
&model.Report{},
|
||||
&model.BackgroundJob{},
|
||||
// S6: WorkingHour — out-of-office / business hours per inbox
|
||||
&model.WorkingHour{},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
BackgroundJobStatusQueued = "queued"
|
||||
BackgroundJobStatusRunning = "running"
|
||||
BackgroundJobStatusRetrying = "retrying"
|
||||
BackgroundJobStatusCompleted = "completed"
|
||||
BackgroundJobStatusDead = "dead"
|
||||
)
|
||||
|
||||
const DefaultBackgroundJobQueue = "default"
|
||||
|
||||
// BackgroundJob stores durable background work in the database.
|
||||
// Reference: Chatwoot ActiveJob/Sidekiq jobs with retry, delayed scheduling,
|
||||
// mutex/idempotency, and observable failure state.
|
||||
type BackgroundJob struct {
|
||||
ID uint `gorm:"primaryKey" json:"id"`
|
||||
Queue string `gorm:"size:100;not null;default:'default';index:idx_background_jobs_ready,priority:2" json:"queue"`
|
||||
JobType string `gorm:"size:150;not null;index" json:"job_type"`
|
||||
Payload json.RawMessage `gorm:"type:jsonb;not null;default:'{}'" json:"payload"`
|
||||
Status string `gorm:"size:50;not null;default:'queued';index:idx_background_jobs_ready,priority:1" json:"status"`
|
||||
Priority int `gorm:"not null;default:0" json:"priority"`
|
||||
Attempts int `gorm:"not null;default:0" json:"attempts"`
|
||||
MaxAttempts int `gorm:"not null;default:3" json:"max_attempts"`
|
||||
ScheduledAt time.Time `gorm:"not null;index:idx_background_jobs_ready,priority:3" json:"scheduled_at"`
|
||||
LockedAt *time.Time `gorm:"index" json:"locked_at,omitempty"`
|
||||
LockedBy string `gorm:"size:150" json:"locked_by,omitempty"`
|
||||
IdempotencyKey string `gorm:"size:255;index" json:"idempotency_key,omitempty"`
|
||||
LastError string `gorm:"type:text" json:"last_error,omitempty"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
FailedAt *time.Time `json:"failed_at,omitempty"`
|
||||
CreatedAt time.Time `gorm:"autoCreateTime" json:"created_at"`
|
||||
UpdatedAt time.Time `gorm:"autoUpdateTime" json:"updated_at"`
|
||||
}
|
||||
|
||||
func (BackgroundJob) TableName() string { return "background_jobs" }
|
||||
@@ -168,6 +168,7 @@ func defaultTestModels() []interface{} {
|
||||
&model.Mention{},
|
||||
&model.IntegrationHook{},
|
||||
&model.IntegrationApp{},
|
||||
&model.BackgroundJob{},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+407
-17
@@ -1,31 +1,421 @@
|
||||
package worker
|
||||
|
||||
// Worker processes background jobs such as:
|
||||
// - Auto-assignment of conversations to agents
|
||||
// - Sending outgoing messages through channels
|
||||
// - Processing webhook events asynchronously
|
||||
// - Notification delivery
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
// WorkerPool manages a set of background job processors.
|
||||
// This is a placeholder — will be implemented with proper job queue
|
||||
// (e.g., Redis-based or database-backed queue).
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
var ErrWorkerDatabaseRequired = errors.New("worker database is required")
|
||||
|
||||
// JobHandler performs one durable background job.
|
||||
type JobHandler func(context.Context, *model.BackgroundJob) error
|
||||
|
||||
type BackoffFunc func(attempt int) time.Duration
|
||||
|
||||
// WorkerPool manages durable background job processors.
|
||||
// Reference: Chatwoot ActiveJob/Sidekiq workers use persistent jobs, scheduled
|
||||
// execution, retry, and dead-letter visibility for async side effects.
|
||||
type WorkerPool struct {
|
||||
// TODO: implement with job queue
|
||||
db *gorm.DB
|
||||
handlers map[string]JobHandler
|
||||
queues []string
|
||||
workerID string
|
||||
workerCount int
|
||||
pollInterval time.Duration
|
||||
staleLockTimeout time.Duration
|
||||
backoff BackoffFunc
|
||||
now func() time.Time
|
||||
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewWorkerPool creates a new WorkerPool instance.
|
||||
func NewWorkerPool() *WorkerPool {
|
||||
return &WorkerPool{}
|
||||
type Option func(*WorkerPool)
|
||||
|
||||
// NewWorkerPool creates a WorkerPool. The variadic DB preserves older no-arg
|
||||
// construction while allowing Phase 5 durable workers to be wired incrementally.
|
||||
func NewWorkerPool(db ...*gorm.DB) *WorkerPool {
|
||||
wp := &WorkerPool{
|
||||
handlers: make(map[string]JobHandler),
|
||||
queues: []string{model.DefaultBackgroundJobQueue},
|
||||
workerID: fmt.Sprintf("worker-%d", time.Now().UnixNano()),
|
||||
workerCount: 1,
|
||||
pollInterval: 500 * time.Millisecond,
|
||||
staleLockTimeout: 15 * time.Minute,
|
||||
backoff: defaultBackoff,
|
||||
now: time.Now,
|
||||
}
|
||||
if len(db) > 0 {
|
||||
wp.db = db[0]
|
||||
}
|
||||
return wp
|
||||
}
|
||||
|
||||
func NewWorkerPoolWithOptions(db *gorm.DB, opts ...Option) *WorkerPool {
|
||||
wp := NewWorkerPool(db)
|
||||
for _, opt := range opts {
|
||||
opt(wp)
|
||||
}
|
||||
return wp
|
||||
}
|
||||
|
||||
func WithWorkerCount(count int) Option {
|
||||
return func(wp *WorkerPool) {
|
||||
if count > 0 {
|
||||
wp.workerCount = count
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithQueues(queues ...string) Option {
|
||||
return func(wp *WorkerPool) {
|
||||
if len(queues) > 0 {
|
||||
wp.queues = append([]string(nil), queues...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithPollInterval(interval time.Duration) Option {
|
||||
return func(wp *WorkerPool) {
|
||||
if interval > 0 {
|
||||
wp.pollInterval = interval
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithWorkerID(id string) Option {
|
||||
return func(wp *WorkerPool) {
|
||||
if id != "" {
|
||||
wp.workerID = id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithStaleLockTimeout(timeout time.Duration) Option {
|
||||
return func(wp *WorkerPool) {
|
||||
if timeout > 0 {
|
||||
wp.staleLockTimeout = timeout
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithBackoff(backoff BackoffFunc) Option {
|
||||
return func(wp *WorkerPool) {
|
||||
if backoff != nil {
|
||||
wp.backoff = backoff
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithNow(now func() time.Time) Option {
|
||||
return func(wp *WorkerPool) {
|
||||
if now != nil {
|
||||
wp.now = now
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) Register(jobType string, handler JobHandler) {
|
||||
wp.mu.Lock()
|
||||
defer wp.mu.Unlock()
|
||||
wp.handlers[jobType] = handler
|
||||
}
|
||||
|
||||
type EnqueueOption func(*model.BackgroundJob)
|
||||
|
||||
func WithQueue(queue string) EnqueueOption {
|
||||
return func(job *model.BackgroundJob) {
|
||||
if queue != "" {
|
||||
job.Queue = queue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithScheduledAt(at time.Time) EnqueueOption {
|
||||
return func(job *model.BackgroundJob) {
|
||||
if !at.IsZero() {
|
||||
job.ScheduledAt = at
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithMaxAttempts(max int) EnqueueOption {
|
||||
return func(job *model.BackgroundJob) {
|
||||
if max > 0 {
|
||||
job.MaxAttempts = max
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithPriority(priority int) EnqueueOption {
|
||||
return func(job *model.BackgroundJob) {
|
||||
job.Priority = priority
|
||||
}
|
||||
}
|
||||
|
||||
func WithIdempotencyKey(key string) EnqueueOption {
|
||||
return func(job *model.BackgroundJob) {
|
||||
job.IdempotencyKey = key
|
||||
}
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) Enqueue(ctx context.Context, jobType string, payload any, opts ...EnqueueOption) (*model.BackgroundJob, error) {
|
||||
if wp.db == nil {
|
||||
return nil, ErrWorkerDatabaseRequired
|
||||
}
|
||||
if jobType == "" {
|
||||
return nil, errors.New("job type is required")
|
||||
}
|
||||
|
||||
payloadBytes, err := marshalPayload(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
job := &model.BackgroundJob{
|
||||
Queue: model.DefaultBackgroundJobQueue,
|
||||
JobType: jobType,
|
||||
Payload: payloadBytes,
|
||||
Status: model.BackgroundJobStatusQueued,
|
||||
MaxAttempts: 3,
|
||||
ScheduledAt: wp.now(),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(job)
|
||||
}
|
||||
if job.ScheduledAt.IsZero() {
|
||||
job.ScheduledAt = wp.now()
|
||||
}
|
||||
if job.MaxAttempts <= 0 {
|
||||
job.MaxAttempts = 3
|
||||
}
|
||||
|
||||
if job.IdempotencyKey != "" {
|
||||
var existing model.BackgroundJob
|
||||
err := wp.db.WithContext(ctx).Where("idempotency_key = ?", job.IdempotencyKey).First(&existing).Error
|
||||
if err == nil {
|
||||
return &existing, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := wp.db.WithContext(ctx).Create(job).Error; err != nil {
|
||||
if job.IdempotencyKey != "" {
|
||||
var existing model.BackgroundJob
|
||||
if findErr := wp.db.WithContext(ctx).Where("idempotency_key = ?", job.IdempotencyKey).First(&existing).Error; findErr == nil {
|
||||
return &existing, nil
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// Start initializes and begins processing background jobs.
|
||||
func (wp *WorkerPool) Start() error {
|
||||
// TODO: implement
|
||||
if wp.db == nil {
|
||||
return nil
|
||||
}
|
||||
wp.mu.Lock()
|
||||
if wp.cancel != nil {
|
||||
wp.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
wp.ctx, wp.cancel = context.WithCancel(context.Background())
|
||||
ctx := wp.ctx
|
||||
workerCount := wp.workerCount
|
||||
wp.mu.Unlock()
|
||||
|
||||
if _, err := wp.RequeueStaleJobs(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i < workerCount; i++ {
|
||||
wp.wg.Add(1)
|
||||
go wp.run(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the worker pool.
|
||||
func (wp *WorkerPool) Stop() error {
|
||||
// TODO: implement
|
||||
wp.mu.Lock()
|
||||
cancel := wp.cancel
|
||||
wp.cancel = nil
|
||||
wp.ctx = nil
|
||||
wp.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
wp.wg.Wait()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) ProcessOne(ctx context.Context) (bool, error) {
|
||||
if wp.db == nil {
|
||||
return false, ErrWorkerDatabaseRequired
|
||||
}
|
||||
job, err := wp.claimNext(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, wp.perform(ctx, job)
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) RequeueStaleJobs(ctx context.Context) (int64, error) {
|
||||
if wp.db == nil {
|
||||
return 0, ErrWorkerDatabaseRequired
|
||||
}
|
||||
cutoff := wp.now().Add(-wp.staleLockTimeout)
|
||||
updates := map[string]any{
|
||||
"status": model.BackgroundJobStatusRetrying,
|
||||
"locked_at": nil,
|
||||
"locked_by": "",
|
||||
}
|
||||
result := wp.db.WithContext(ctx).Model(&model.BackgroundJob{}).
|
||||
Where("status = ? AND locked_at IS NOT NULL AND locked_at < ?", model.BackgroundJobStatusRunning, cutoff).
|
||||
Updates(updates)
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) run(ctx context.Context) {
|
||||
defer wp.wg.Done()
|
||||
ticker := time.NewTicker(wp.pollInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
processed, _ := wp.ProcessOne(ctx)
|
||||
if processed {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) claimNext(ctx context.Context) (*model.BackgroundJob, error) {
|
||||
var job model.BackgroundJob
|
||||
err := wp.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
query := tx.Where("status IN ? AND scheduled_at <= ?", []string{model.BackgroundJobStatusQueued, model.BackgroundJobStatusRetrying}, wp.now())
|
||||
if len(wp.queues) > 0 {
|
||||
query = query.Where("queue IN ?", wp.queues)
|
||||
}
|
||||
if tx.Dialector != nil && tx.Dialector.Name() == "postgres" {
|
||||
query = query.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"})
|
||||
}
|
||||
if err := query.Order("priority DESC, scheduled_at ASC, id ASC").First(&job).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lockedAt := wp.now()
|
||||
updates := map[string]any{
|
||||
"status": model.BackgroundJobStatusRunning,
|
||||
"locked_at": &lockedAt,
|
||||
"locked_by": wp.workerID,
|
||||
"attempts": gorm.Expr("attempts + 1"),
|
||||
}
|
||||
if err := tx.Model(&model.BackgroundJob{}).Where("id = ?", job.ID).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.First(&job, job.ID).Error
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &job, nil
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) perform(ctx context.Context, job *model.BackgroundJob) error {
|
||||
handler := wp.handlerFor(job.JobType)
|
||||
if handler == nil {
|
||||
return wp.fail(ctx, job, fmt.Errorf("no handler registered for job type %q", job.JobType))
|
||||
}
|
||||
if err := handler(ctx, job); err != nil {
|
||||
return wp.fail(ctx, job, err)
|
||||
}
|
||||
finishedAt := wp.now()
|
||||
updates := map[string]any{
|
||||
"status": model.BackgroundJobStatusCompleted,
|
||||
"locked_at": nil,
|
||||
"locked_by": "",
|
||||
"last_error": "",
|
||||
"finished_at": &finishedAt,
|
||||
}
|
||||
return wp.db.WithContext(ctx).Model(&model.BackgroundJob{}).Where("id = ?", job.ID).Updates(updates).Error
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) fail(ctx context.Context, job *model.BackgroundJob, err error) error {
|
||||
now := wp.now()
|
||||
updates := map[string]any{
|
||||
"locked_at": nil,
|
||||
"locked_by": "",
|
||||
"last_error": err.Error(),
|
||||
}
|
||||
if job.Attempts >= job.MaxAttempts {
|
||||
updates["status"] = model.BackgroundJobStatusDead
|
||||
updates["failed_at"] = &now
|
||||
} else {
|
||||
updates["status"] = model.BackgroundJobStatusRetrying
|
||||
updates["scheduled_at"] = now.Add(wp.backoff(job.Attempts))
|
||||
}
|
||||
if updateErr := wp.db.WithContext(ctx).Model(&model.BackgroundJob{}).Where("id = ?", job.ID).Updates(updates).Error; updateErr != nil {
|
||||
return updateErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (wp *WorkerPool) handlerFor(jobType string) JobHandler {
|
||||
wp.mu.RLock()
|
||||
defer wp.mu.RUnlock()
|
||||
return wp.handlers[jobType]
|
||||
}
|
||||
|
||||
func marshalPayload(payload any) (json.RawMessage, error) {
|
||||
if payload == nil {
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
switch value := payload.(type) {
|
||||
case json.RawMessage:
|
||||
if !json.Valid(value) {
|
||||
return nil, errors.New("job payload must be valid JSON")
|
||||
}
|
||||
return value, nil
|
||||
case []byte:
|
||||
if !json.Valid(value) {
|
||||
return nil, errors.New("job payload must be valid JSON")
|
||||
}
|
||||
return json.RawMessage(value), nil
|
||||
default:
|
||||
payloadBytes, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return payloadBytes, nil
|
||||
}
|
||||
}
|
||||
|
||||
func defaultBackoff(attempt int) time.Duration {
|
||||
if attempt <= 0 {
|
||||
attempt = 1
|
||||
}
|
||||
seconds := math.Pow(2, float64(attempt-1))
|
||||
if seconds > 300 {
|
||||
seconds = 300
|
||||
}
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func newWorkerTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open("file:worker-test?mode=memory&cache=shared"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlite db handle: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
if err := db.AutoMigrate(&model.BackgroundJob{}); err != nil {
|
||||
t.Fatalf("migrate background jobs: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
db.Exec("DELETE FROM background_jobs")
|
||||
sqlDB.Close()
|
||||
})
|
||||
return db
|
||||
}
|
||||
|
||||
func loadJob(t *testing.T, db *gorm.DB, id uint) model.BackgroundJob {
|
||||
t.Helper()
|
||||
var job model.BackgroundJob
|
||||
if err := db.First(&job, id).Error; err != nil {
|
||||
t.Fatalf("load job: %v", err)
|
||||
}
|
||||
return job
|
||||
}
|
||||
|
||||
func TestWorkerPoolEnqueueStoresPayloadAndIdempotency(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }))
|
||||
|
||||
job, err := wp.Enqueue(context.Background(), "contact_export", map[string]any{"account_id": 1}, WithIdempotencyKey("contact-export:1"), WithQueue("exports"), WithMaxAttempts(5), WithPriority(10))
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
duplicate, err := wp.Enqueue(context.Background(), "contact_export", map[string]any{"account_id": 2}, WithIdempotencyKey("contact-export:1"))
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue duplicate: %v", err)
|
||||
}
|
||||
if duplicate.ID != job.ID {
|
||||
t.Fatalf("expected duplicate enqueue to return existing job %d, got %d", job.ID, duplicate.ID)
|
||||
}
|
||||
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Queue != "exports" || reloaded.JobType != "contact_export" || reloaded.Status != model.BackgroundJobStatusQueued || reloaded.MaxAttempts != 5 || reloaded.Priority != 10 {
|
||||
t.Fatalf("unexpected job fields: %+v", reloaded)
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(reloaded.Payload, &payload); err != nil {
|
||||
t.Fatalf("unmarshal payload: %v", err)
|
||||
}
|
||||
if payload["account_id"].(float64) != 1 {
|
||||
t.Fatalf("unexpected payload: %s", string(reloaded.Payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolProcessOneCompletesDueJob(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithWorkerID("test-worker"))
|
||||
var handled atomic.Int32
|
||||
wp.Register("send_reply", func(ctx context.Context, job *model.BackgroundJob) error {
|
||||
handled.Add(1)
|
||||
if job.Attempts != 1 || job.LockedBy != "test-worker" || job.LockedAt == nil {
|
||||
t.Fatalf("job was not claimed before handler: %+v", job)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
job, err := wp.Enqueue(context.Background(), "send_reply", map[string]any{"message_id": 7})
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
|
||||
processed, err := wp.ProcessOne(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("process one: %v", err)
|
||||
}
|
||||
if !processed || handled.Load() != 1 {
|
||||
t.Fatalf("expected one handled job, processed=%v handled=%d", processed, handled.Load())
|
||||
}
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusCompleted || reloaded.FinishedAt == nil || reloaded.LockedAt != nil || reloaded.LockedBy != "" {
|
||||
t.Fatalf("expected completed unlocked job: %+v", reloaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolRetriesThenDeadLettersFailures(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithBackoff(func(attempt int) time.Duration { return 0 }))
|
||||
boom := errors.New("provider timeout")
|
||||
wp.Register("webhook_delivery", func(ctx context.Context, job *model.BackgroundJob) error { return boom })
|
||||
job, err := wp.Enqueue(context.Background(), "webhook_delivery", nil, WithMaxAttempts(2))
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
|
||||
processed, err := wp.ProcessOne(context.Background())
|
||||
if !processed || !errors.Is(err, boom) {
|
||||
t.Fatalf("expected first failure, processed=%v err=%v", processed, err)
|
||||
}
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusRetrying || reloaded.Attempts != 1 || reloaded.LastError != boom.Error() || reloaded.FailedAt != nil {
|
||||
t.Fatalf("expected retrying job after first failure: %+v", reloaded)
|
||||
}
|
||||
|
||||
processed, err = wp.ProcessOne(context.Background())
|
||||
if !processed || !errors.Is(err, boom) {
|
||||
t.Fatalf("expected second failure, processed=%v err=%v", processed, err)
|
||||
}
|
||||
reloaded = loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusDead || reloaded.Attempts != 2 || reloaded.FailedAt == nil || reloaded.LockedAt != nil {
|
||||
t.Fatalf("expected dead-lettered job: %+v", reloaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolRespectsScheduleAndQueues(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithQueues("critical"))
|
||||
wp.Register("sla_scan", func(ctx context.Context, job *model.BackgroundJob) error { return nil })
|
||||
if _, err := wp.Enqueue(context.Background(), "sla_scan", nil, WithQueue("default")); err != nil {
|
||||
t.Fatalf("enqueue default: %v", err)
|
||||
}
|
||||
if _, err := wp.Enqueue(context.Background(), "sla_scan", nil, WithQueue("critical"), WithScheduledAt(now.Add(time.Hour))); err != nil {
|
||||
t.Fatalf("enqueue future: %v", err)
|
||||
}
|
||||
processed, err := wp.ProcessOne(context.Background())
|
||||
if err != nil || processed {
|
||||
t.Fatalf("expected no eligible job, processed=%v err=%v", processed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolRequeuesStaleRunningJobs(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithStaleLockTimeout(time.Minute))
|
||||
lockedAt := now.Add(-2 * time.Minute)
|
||||
job := model.BackgroundJob{
|
||||
Queue: model.DefaultBackgroundJobQueue,
|
||||
JobType: "captain_document_sync",
|
||||
Payload: json.RawMessage(`{}`),
|
||||
Status: model.BackgroundJobStatusRunning,
|
||||
MaxAttempts: 3,
|
||||
ScheduledAt: now.Add(-time.Hour),
|
||||
LockedAt: &lockedAt,
|
||||
LockedBy: "dead-worker",
|
||||
}
|
||||
if err := db.Create(&job).Error; err != nil {
|
||||
t.Fatalf("create stale job: %v", err)
|
||||
}
|
||||
|
||||
count, err := wp.RequeueStaleJobs(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("requeue stale: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("expected 1 stale job requeued, got %d", count)
|
||||
}
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusRetrying || reloaded.LockedAt != nil || reloaded.LockedBy != "" {
|
||||
t.Fatalf("expected retrying unlocked stale job: %+v", reloaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolStartAndStopProcessJobs(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithPollInterval(5*time.Millisecond))
|
||||
var handled atomic.Int32
|
||||
wp.Register("event_dispatch", func(ctx context.Context, job *model.BackgroundJob) error {
|
||||
handled.Add(1)
|
||||
return nil
|
||||
})
|
||||
job, err := wp.Enqueue(context.Background(), "event_dispatch", map[string]any{"event": "conversation_created"})
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
if err := wp.Start(); err != nil {
|
||||
t.Fatalf("start worker: %v", err)
|
||||
}
|
||||
defer wp.Stop()
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if handled.Load() == 1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if handled.Load() != 1 {
|
||||
t.Fatalf("worker loop did not process job")
|
||||
}
|
||||
if err := wp.Stop(); err != nil {
|
||||
t.Fatalf("stop worker: %v", err)
|
||||
}
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusCompleted {
|
||||
t.Fatalf("expected completed job after worker loop: %+v", reloaded)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user