feat(worker): add durable background jobs
This commit is contained in:
@@ -0,0 +1,221 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func newWorkerTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open("file:worker-test?mode=memory&cache=shared"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlite db handle: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
if err := db.AutoMigrate(&model.BackgroundJob{}); err != nil {
|
||||
t.Fatalf("migrate background jobs: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
db.Exec("DELETE FROM background_jobs")
|
||||
sqlDB.Close()
|
||||
})
|
||||
return db
|
||||
}
|
||||
|
||||
func loadJob(t *testing.T, db *gorm.DB, id uint) model.BackgroundJob {
|
||||
t.Helper()
|
||||
var job model.BackgroundJob
|
||||
if err := db.First(&job, id).Error; err != nil {
|
||||
t.Fatalf("load job: %v", err)
|
||||
}
|
||||
return job
|
||||
}
|
||||
|
||||
func TestWorkerPoolEnqueueStoresPayloadAndIdempotency(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }))
|
||||
|
||||
job, err := wp.Enqueue(context.Background(), "contact_export", map[string]any{"account_id": 1}, WithIdempotencyKey("contact-export:1"), WithQueue("exports"), WithMaxAttempts(5), WithPriority(10))
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
duplicate, err := wp.Enqueue(context.Background(), "contact_export", map[string]any{"account_id": 2}, WithIdempotencyKey("contact-export:1"))
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue duplicate: %v", err)
|
||||
}
|
||||
if duplicate.ID != job.ID {
|
||||
t.Fatalf("expected duplicate enqueue to return existing job %d, got %d", job.ID, duplicate.ID)
|
||||
}
|
||||
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Queue != "exports" || reloaded.JobType != "contact_export" || reloaded.Status != model.BackgroundJobStatusQueued || reloaded.MaxAttempts != 5 || reloaded.Priority != 10 {
|
||||
t.Fatalf("unexpected job fields: %+v", reloaded)
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(reloaded.Payload, &payload); err != nil {
|
||||
t.Fatalf("unmarshal payload: %v", err)
|
||||
}
|
||||
if payload["account_id"].(float64) != 1 {
|
||||
t.Fatalf("unexpected payload: %s", string(reloaded.Payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolProcessOneCompletesDueJob(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithWorkerID("test-worker"))
|
||||
var handled atomic.Int32
|
||||
wp.Register("send_reply", func(ctx context.Context, job *model.BackgroundJob) error {
|
||||
handled.Add(1)
|
||||
if job.Attempts != 1 || job.LockedBy != "test-worker" || job.LockedAt == nil {
|
||||
t.Fatalf("job was not claimed before handler: %+v", job)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
job, err := wp.Enqueue(context.Background(), "send_reply", map[string]any{"message_id": 7})
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
|
||||
processed, err := wp.ProcessOne(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("process one: %v", err)
|
||||
}
|
||||
if !processed || handled.Load() != 1 {
|
||||
t.Fatalf("expected one handled job, processed=%v handled=%d", processed, handled.Load())
|
||||
}
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusCompleted || reloaded.FinishedAt == nil || reloaded.LockedAt != nil || reloaded.LockedBy != "" {
|
||||
t.Fatalf("expected completed unlocked job: %+v", reloaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolRetriesThenDeadLettersFailures(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithBackoff(func(attempt int) time.Duration { return 0 }))
|
||||
boom := errors.New("provider timeout")
|
||||
wp.Register("webhook_delivery", func(ctx context.Context, job *model.BackgroundJob) error { return boom })
|
||||
job, err := wp.Enqueue(context.Background(), "webhook_delivery", nil, WithMaxAttempts(2))
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
|
||||
processed, err := wp.ProcessOne(context.Background())
|
||||
if !processed || !errors.Is(err, boom) {
|
||||
t.Fatalf("expected first failure, processed=%v err=%v", processed, err)
|
||||
}
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusRetrying || reloaded.Attempts != 1 || reloaded.LastError != boom.Error() || reloaded.FailedAt != nil {
|
||||
t.Fatalf("expected retrying job after first failure: %+v", reloaded)
|
||||
}
|
||||
|
||||
processed, err = wp.ProcessOne(context.Background())
|
||||
if !processed || !errors.Is(err, boom) {
|
||||
t.Fatalf("expected second failure, processed=%v err=%v", processed, err)
|
||||
}
|
||||
reloaded = loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusDead || reloaded.Attempts != 2 || reloaded.FailedAt == nil || reloaded.LockedAt != nil {
|
||||
t.Fatalf("expected dead-lettered job: %+v", reloaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolRespectsScheduleAndQueues(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithQueues("critical"))
|
||||
wp.Register("sla_scan", func(ctx context.Context, job *model.BackgroundJob) error { return nil })
|
||||
if _, err := wp.Enqueue(context.Background(), "sla_scan", nil, WithQueue("default")); err != nil {
|
||||
t.Fatalf("enqueue default: %v", err)
|
||||
}
|
||||
if _, err := wp.Enqueue(context.Background(), "sla_scan", nil, WithQueue("critical"), WithScheduledAt(now.Add(time.Hour))); err != nil {
|
||||
t.Fatalf("enqueue future: %v", err)
|
||||
}
|
||||
processed, err := wp.ProcessOne(context.Background())
|
||||
if err != nil || processed {
|
||||
t.Fatalf("expected no eligible job, processed=%v err=%v", processed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolRequeuesStaleRunningJobs(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithStaleLockTimeout(time.Minute))
|
||||
lockedAt := now.Add(-2 * time.Minute)
|
||||
job := model.BackgroundJob{
|
||||
Queue: model.DefaultBackgroundJobQueue,
|
||||
JobType: "captain_document_sync",
|
||||
Payload: json.RawMessage(`{}`),
|
||||
Status: model.BackgroundJobStatusRunning,
|
||||
MaxAttempts: 3,
|
||||
ScheduledAt: now.Add(-time.Hour),
|
||||
LockedAt: &lockedAt,
|
||||
LockedBy: "dead-worker",
|
||||
}
|
||||
if err := db.Create(&job).Error; err != nil {
|
||||
t.Fatalf("create stale job: %v", err)
|
||||
}
|
||||
|
||||
count, err := wp.RequeueStaleJobs(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("requeue stale: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("expected 1 stale job requeued, got %d", count)
|
||||
}
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusRetrying || reloaded.LockedAt != nil || reloaded.LockedBy != "" {
|
||||
t.Fatalf("expected retrying unlocked stale job: %+v", reloaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolStartAndStopProcessJobs(t *testing.T) {
|
||||
db := newWorkerTestDB(t)
|
||||
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
||||
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithPollInterval(5*time.Millisecond))
|
||||
var handled atomic.Int32
|
||||
wp.Register("event_dispatch", func(ctx context.Context, job *model.BackgroundJob) error {
|
||||
handled.Add(1)
|
||||
return nil
|
||||
})
|
||||
job, err := wp.Enqueue(context.Background(), "event_dispatch", map[string]any{"event": "conversation_created"})
|
||||
if err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
if err := wp.Start(); err != nil {
|
||||
t.Fatalf("start worker: %v", err)
|
||||
}
|
||||
defer wp.Stop()
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if handled.Load() == 1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if handled.Load() != 1 {
|
||||
t.Fatalf("worker loop did not process job")
|
||||
}
|
||||
if err := wp.Stop(); err != nil {
|
||||
t.Fatalf("stop worker: %v", err)
|
||||
}
|
||||
reloaded := loadJob(t, db, job.ID)
|
||||
if reloaded.Status != model.BackgroundJobStatusCompleted {
|
||||
t.Fatalf("expected completed job after worker loop: %+v", reloaded)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user