Restructure the monorepo into clear top-level directories: - backend/: Go module root (cmd, internal, pkg, configs, migrations, docs/swagger, scripts, tests, go.mod, Makefile, .air.toml) - deploy/: Docker (Dockerfile, docker-compose*), quickstart, fluentd - docs/: project documentation + reports/ (moved from repo root) - AGENTS.md: new AI coding-agent guide at repo root Update all references to the new layout: - Dockerfile: COPY backend/go.mod, COPY backend/ (context = repo root) - docker-compose files: context ../.., dockerfile deploy/docker/Dockerfile, env_file ../../.env, volume mounts ../../backend:/app - deploy/quickstart/compose.yaml: dockerfile deploy/docker/Dockerfile - CI: working-directory: backend for go commands, file deploy/docker/Dockerfile, coverage path backend/coverage.out, health_check backend/scripts/ - backend/Makefile: docker target uses -f ../deploy/docker/Dockerfile ../ - README: architecture tree, quickstart, config paths updated Move root stray scripts (rename_models.*, run_m11_tests.sh, verify_build.sh, gorm_bool_main.go) to backend/scripts/legacy/. All moves via git mv to preserve history. Build, vet, SQLite tests, and docker compose config verified.
222 lines
7.8 KiB
Go
222 lines
7.8 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/gochat/gochat/internal/model"
|
|
"gorm.io/driver/sqlite"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/logger"
|
|
)
|
|
|
|
func newWorkerTestDB(t *testing.T) *gorm.DB {
|
|
t.Helper()
|
|
db, err := gorm.Open(sqlite.Open("file:worker-test?mode=memory&cache=shared"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
|
if err != nil {
|
|
t.Fatalf("open sqlite: %v", err)
|
|
}
|
|
sqlDB, err := db.DB()
|
|
if err != nil {
|
|
t.Fatalf("sqlite db handle: %v", err)
|
|
}
|
|
sqlDB.SetMaxOpenConns(1)
|
|
if err := db.AutoMigrate(&model.BackgroundJob{}); err != nil {
|
|
t.Fatalf("migrate background jobs: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
db.Exec("DELETE FROM background_jobs")
|
|
sqlDB.Close()
|
|
})
|
|
return db
|
|
}
|
|
|
|
func loadJob(t *testing.T, db *gorm.DB, id uint) model.BackgroundJob {
|
|
t.Helper()
|
|
var job model.BackgroundJob
|
|
if err := db.First(&job, id).Error; err != nil {
|
|
t.Fatalf("load job: %v", err)
|
|
}
|
|
return job
|
|
}
|
|
|
|
func TestWorkerPoolEnqueueStoresPayloadAndIdempotency(t *testing.T) {
|
|
db := newWorkerTestDB(t)
|
|
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
|
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }))
|
|
|
|
job, err := wp.Enqueue(context.Background(), "contact_export", map[string]any{"account_id": 1}, WithIdempotencyKey("contact-export:1"), WithQueue("exports"), WithMaxAttempts(5), WithPriority(10))
|
|
if err != nil {
|
|
t.Fatalf("enqueue: %v", err)
|
|
}
|
|
duplicate, err := wp.Enqueue(context.Background(), "contact_export", map[string]any{"account_id": 2}, WithIdempotencyKey("contact-export:1"))
|
|
if err != nil {
|
|
t.Fatalf("enqueue duplicate: %v", err)
|
|
}
|
|
if duplicate.ID != job.ID {
|
|
t.Fatalf("expected duplicate enqueue to return existing job %d, got %d", job.ID, duplicate.ID)
|
|
}
|
|
|
|
reloaded := loadJob(t, db, job.ID)
|
|
if reloaded.Queue != "exports" || reloaded.JobType != "contact_export" || reloaded.Status != model.BackgroundJobStatusQueued || reloaded.MaxAttempts != 5 || reloaded.Priority != 10 {
|
|
t.Fatalf("unexpected job fields: %+v", reloaded)
|
|
}
|
|
var payload map[string]any
|
|
if err := json.Unmarshal(reloaded.Payload, &payload); err != nil {
|
|
t.Fatalf("unmarshal payload: %v", err)
|
|
}
|
|
if payload["account_id"].(float64) != 1 {
|
|
t.Fatalf("unexpected payload: %s", string(reloaded.Payload))
|
|
}
|
|
}
|
|
|
|
func TestWorkerPoolProcessOneCompletesDueJob(t *testing.T) {
|
|
db := newWorkerTestDB(t)
|
|
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
|
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithWorkerID("test-worker"))
|
|
var handled atomic.Int32
|
|
wp.Register("send_reply", func(ctx context.Context, job *model.BackgroundJob) error {
|
|
handled.Add(1)
|
|
if job.Attempts != 1 || job.LockedBy != "test-worker" || job.LockedAt == nil {
|
|
t.Fatalf("job was not claimed before handler: %+v", job)
|
|
}
|
|
return nil
|
|
})
|
|
job, err := wp.Enqueue(context.Background(), "send_reply", map[string]any{"message_id": 7})
|
|
if err != nil {
|
|
t.Fatalf("enqueue: %v", err)
|
|
}
|
|
|
|
processed, err := wp.ProcessOne(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("process one: %v", err)
|
|
}
|
|
if !processed || handled.Load() != 1 {
|
|
t.Fatalf("expected one handled job, processed=%v handled=%d", processed, handled.Load())
|
|
}
|
|
reloaded := loadJob(t, db, job.ID)
|
|
if reloaded.Status != model.BackgroundJobStatusCompleted || reloaded.FinishedAt == nil || reloaded.LockedAt != nil || reloaded.LockedBy != "" {
|
|
t.Fatalf("expected completed unlocked job: %+v", reloaded)
|
|
}
|
|
}
|
|
|
|
func TestWorkerPoolRetriesThenDeadLettersFailures(t *testing.T) {
|
|
db := newWorkerTestDB(t)
|
|
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
|
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithBackoff(func(attempt int) time.Duration { return 0 }))
|
|
boom := errors.New("provider timeout")
|
|
wp.Register("webhook_delivery", func(ctx context.Context, job *model.BackgroundJob) error { return boom })
|
|
job, err := wp.Enqueue(context.Background(), "webhook_delivery", nil, WithMaxAttempts(2))
|
|
if err != nil {
|
|
t.Fatalf("enqueue: %v", err)
|
|
}
|
|
|
|
processed, err := wp.ProcessOne(context.Background())
|
|
if !processed || !errors.Is(err, boom) {
|
|
t.Fatalf("expected first failure, processed=%v err=%v", processed, err)
|
|
}
|
|
reloaded := loadJob(t, db, job.ID)
|
|
if reloaded.Status != model.BackgroundJobStatusRetrying || reloaded.Attempts != 1 || reloaded.LastError != boom.Error() || reloaded.FailedAt != nil {
|
|
t.Fatalf("expected retrying job after first failure: %+v", reloaded)
|
|
}
|
|
|
|
processed, err = wp.ProcessOne(context.Background())
|
|
if !processed || !errors.Is(err, boom) {
|
|
t.Fatalf("expected second failure, processed=%v err=%v", processed, err)
|
|
}
|
|
reloaded = loadJob(t, db, job.ID)
|
|
if reloaded.Status != model.BackgroundJobStatusDead || reloaded.Attempts != 2 || reloaded.FailedAt == nil || reloaded.LockedAt != nil {
|
|
t.Fatalf("expected dead-lettered job: %+v", reloaded)
|
|
}
|
|
}
|
|
|
|
func TestWorkerPoolRespectsScheduleAndQueues(t *testing.T) {
|
|
db := newWorkerTestDB(t)
|
|
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
|
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithQueues("critical"))
|
|
wp.Register("sla_scan", func(ctx context.Context, job *model.BackgroundJob) error { return nil })
|
|
if _, err := wp.Enqueue(context.Background(), "sla_scan", nil, WithQueue("default")); err != nil {
|
|
t.Fatalf("enqueue default: %v", err)
|
|
}
|
|
if _, err := wp.Enqueue(context.Background(), "sla_scan", nil, WithQueue("critical"), WithScheduledAt(now.Add(time.Hour))); err != nil {
|
|
t.Fatalf("enqueue future: %v", err)
|
|
}
|
|
processed, err := wp.ProcessOne(context.Background())
|
|
if err != nil || processed {
|
|
t.Fatalf("expected no eligible job, processed=%v err=%v", processed, err)
|
|
}
|
|
}
|
|
|
|
func TestWorkerPoolRequeuesStaleRunningJobs(t *testing.T) {
|
|
db := newWorkerTestDB(t)
|
|
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
|
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithStaleLockTimeout(time.Minute))
|
|
lockedAt := now.Add(-2 * time.Minute)
|
|
job := model.BackgroundJob{
|
|
Queue: model.DefaultBackgroundJobQueue,
|
|
JobType: "captain_document_sync",
|
|
Payload: json.RawMessage(`{}`),
|
|
Status: model.BackgroundJobStatusRunning,
|
|
MaxAttempts: 3,
|
|
ScheduledAt: now.Add(-time.Hour),
|
|
LockedAt: &lockedAt,
|
|
LockedBy: "dead-worker",
|
|
}
|
|
if err := db.Create(&job).Error; err != nil {
|
|
t.Fatalf("create stale job: %v", err)
|
|
}
|
|
|
|
count, err := wp.RequeueStaleJobs(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("requeue stale: %v", err)
|
|
}
|
|
if count != 1 {
|
|
t.Fatalf("expected 1 stale job requeued, got %d", count)
|
|
}
|
|
reloaded := loadJob(t, db, job.ID)
|
|
if reloaded.Status != model.BackgroundJobStatusRetrying || reloaded.LockedAt != nil || reloaded.LockedBy != "" {
|
|
t.Fatalf("expected retrying unlocked stale job: %+v", reloaded)
|
|
}
|
|
}
|
|
|
|
func TestWorkerPoolStartAndStopProcessJobs(t *testing.T) {
|
|
db := newWorkerTestDB(t)
|
|
now := time.Date(2026, 6, 5, 10, 0, 0, 0, time.UTC)
|
|
wp := NewWorkerPoolWithOptions(db, WithNow(func() time.Time { return now }), WithPollInterval(5*time.Millisecond))
|
|
var handled atomic.Int32
|
|
wp.Register("event_dispatch", func(ctx context.Context, job *model.BackgroundJob) error {
|
|
handled.Add(1)
|
|
return nil
|
|
})
|
|
job, err := wp.Enqueue(context.Background(), "event_dispatch", map[string]any{"event": "conversation_created"})
|
|
if err != nil {
|
|
t.Fatalf("enqueue: %v", err)
|
|
}
|
|
if err := wp.Start(); err != nil {
|
|
t.Fatalf("start worker: %v", err)
|
|
}
|
|
defer wp.Stop()
|
|
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if handled.Load() == 1 {
|
|
break
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if handled.Load() != 1 {
|
|
t.Fatalf("worker loop did not process job")
|
|
}
|
|
if err := wp.Stop(); err != nil {
|
|
t.Fatalf("stop worker: %v", err)
|
|
}
|
|
reloaded := loadJob(t, db, job.ID)
|
|
if reloaded.Status != model.BackgroundJobStatusCompleted {
|
|
t.Fatalf("expected completed job after worker loop: %+v", reloaded)
|
|
}
|
|
}
|