270 lines
9.0 KiB
Go
270 lines
9.0 KiB
Go
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/mq"
|
|
)
|
|
|
|
type fakeQueueBroker struct {
|
|
started []string
|
|
drained []string
|
|
consumers []*fakeQueueConsumer
|
|
consumerWaitErr error
|
|
handler mq.MessageHandler
|
|
controlDrain func(context.Context) (int, error)
|
|
controlDrained bool
|
|
}
|
|
|
|
func (f *fakeQueueBroker) StartPredeclaredConsumer(_ context.Context, queue string, handler mq.MessageHandler) (taskQueueConsumer, error) {
|
|
f.started = append(f.started, queue)
|
|
f.handler = handler
|
|
consumer := &fakeQueueConsumer{done: make(chan struct{}), waitErr: f.consumerWaitErr}
|
|
f.consumers = append(f.consumers, consumer)
|
|
return consumer, nil
|
|
}
|
|
|
|
func (f *fakeQueueBroker) DrainPredeclared(_ context.Context, queue string) (int, error) {
|
|
f.drained = append(f.drained, queue)
|
|
return 3, nil
|
|
}
|
|
|
|
func (f *fakeQueueBroker) DrainControlPredeclared(ctx context.Context, _ string, _ mq.MessageHandler) (int, error) {
|
|
f.controlDrained = true
|
|
if f.controlDrain != nil {
|
|
return f.controlDrain(ctx)
|
|
}
|
|
return 0, nil
|
|
}
|
|
|
|
type fakeQueueConsumer struct {
|
|
done chan struct{}
|
|
once sync.Once
|
|
waitErr error
|
|
}
|
|
|
|
func (f *fakeQueueConsumer) Wait(ctx context.Context) error {
|
|
select {
|
|
case <-f.done:
|
|
return f.waitErr
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (f *fakeQueueConsumer) Stop(context.Context) error {
|
|
f.finish()
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeQueueConsumer) finish() {
|
|
f.once.Do(func() { close(f.done) })
|
|
}
|
|
|
|
func TestV3TaskQueueControllerPauseResumeAndStopUsesOnlyPredeclaredQueue(t *testing.T) {
|
|
now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
|
|
d, st, server := newLocalV01TestDispatcher(t, now)
|
|
defer server.Close()
|
|
defer st.Close()
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 1 {
|
|
t.Fatalf("assignments=%+v err=%v", assignments, err)
|
|
}
|
|
assignment := assignments[0]
|
|
broker := &fakeQueueBroker{}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
ctx := context.Background()
|
|
|
|
if err := queues.StartTask(ctx, assignment); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(broker.started) != 1 || broker.started[0] != assignment.Queue.QueueName {
|
|
t.Fatalf("started queues=%v, want only %q", broker.started, assignment.Queue.QueueName)
|
|
}
|
|
|
|
paused, err := st.SetLocalTaskAdmissionBarrier(assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, "paused")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := queues.StopTask(ctx, paused); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(broker.drained) != 0 {
|
|
t.Fatalf("pause discarded queued messages: drained=%v", broker.drained)
|
|
}
|
|
|
|
resumed, err := st.ResumeLocalTaskAdmission(assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, "running", assignment.TaskRevision+1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := queues.StartTask(ctx, resumed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(broker.started) != 2 || broker.started[1] != assignment.Queue.QueueName {
|
|
t.Fatalf("resume did not consume the same backlog queue: %v", broker.started)
|
|
}
|
|
|
|
stopped, err := st.SetLocalTaskAdmissionBarrier(assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, "stopped")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := queues.StopTask(ctx, stopped); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := queues.DrainTask(ctx, stopped); err == nil || len(broker.drained) != 0 {
|
|
t.Fatalf("drain before authoritative stopped status: err=%v drained=%v", err, broker.drained)
|
|
}
|
|
if _, err := st.DB().Exec(`UPDATE local_v01_task_assignments SET saas_status='stopped', task_revision=task_revision+1 WHERE dispatcher_id=? AND task_id=?`, assignment.DispatcherID, assignment.TaskID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := queues.DrainTask(ctx, stopped); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(broker.drained) != 1 || broker.drained[0] != assignment.Queue.QueueName {
|
|
t.Fatalf("stop did not drain the stopped task queue: %v", broker.drained)
|
|
}
|
|
}
|
|
|
|
func TestV3TaskQueueControllerReportsUnexpectedConsumerExit(t *testing.T) {
|
|
now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
|
|
d, st, server := newLocalV01TestDispatcher(t, now)
|
|
defer server.Close()
|
|
defer st.Close()
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 1 {
|
|
t.Fatalf("assignments=%+v err=%v", assignments, err)
|
|
}
|
|
brokerErr := errors.New("broker consumer exited")
|
|
broker := &fakeQueueBroker{consumerWaitErr: brokerErr}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
if err := queues.StartTask(context.Background(), assignments[0]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker.consumers[0].finish()
|
|
select {
|
|
case err := <-queues.Errors():
|
|
if !errors.Is(err, brokerErr) {
|
|
t.Fatalf("consumer error=%v, want wrapped broker error", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("unexpected consumer exit was not reported")
|
|
}
|
|
queues.mu.Lock()
|
|
remaining := len(queues.consumers)
|
|
queues.mu.Unlock()
|
|
if remaining != 0 {
|
|
t.Fatalf("unexpectedly exited consumer remains registered: %d", remaining)
|
|
}
|
|
}
|
|
|
|
func TestV3TaskQueueApplyActiveCallPolicyValidation(t *testing.T) {
|
|
now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
|
|
d, st, server := newLocalV01TestDispatcher(t, now)
|
|
defer server.Close()
|
|
defer st.Close()
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 1 {
|
|
t.Fatalf("assignments=%+v err=%v", assignments, err)
|
|
}
|
|
queues := newTaskQueueController(d, &fakeQueueBroker{}, nil)
|
|
for _, tc := range []struct {
|
|
action, policy string
|
|
wantErr bool
|
|
}{
|
|
{action: "stop", policy: "drain", wantErr: false},
|
|
{action: "pause", policy: "hangup", wantErr: false},
|
|
{action: "resume", policy: "hangup", wantErr: true},
|
|
{action: "stop", policy: "invalid", wantErr: true},
|
|
} {
|
|
err := queues.ApplyActiveCallPolicy(context.Background(), assignments[0], tc.action, tc.policy)
|
|
if (err != nil) != tc.wantErr {
|
|
t.Fatalf("ApplyActiveCallPolicy(%q,%q) error=%v, wantErr=%v", tc.action, tc.policy, err, tc.wantErr)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestV3TaskQueueControllerCannotStartWhenAdmissionIsClosed(t *testing.T) {
|
|
now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
|
|
d, st, server := newLocalV01TestDispatcher(t, now)
|
|
defer server.Close()
|
|
defer st.Close()
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 1 {
|
|
t.Fatalf("assignments=%+v err=%v", assignments, err)
|
|
}
|
|
assignment, err := st.SetLocalTaskAdmissionBarrier(assignments[0].DispatcherID, assignments[0].TaskID, assignments[0].TenantID, assignments[0].TenantKey, "stopped")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker := &fakeQueueBroker{}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
if err := queues.StartTask(context.Background(), assignment); err == nil {
|
|
t.Fatal("started a stopped task queue")
|
|
}
|
|
if len(broker.started) != 0 {
|
|
t.Fatalf("broker subscribed to queues despite closed admission: %v", broker.started)
|
|
}
|
|
}
|
|
|
|
func TestV3TaskQueueControllerStopAllDoesNotChangeDurableAdmission(t *testing.T) {
|
|
now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
|
|
d, st, server := newLocalV01TestDispatcher(t, now)
|
|
defer server.Close()
|
|
defer st.Close()
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 1 {
|
|
t.Fatalf("assignments=%+v err=%v", assignments, err)
|
|
}
|
|
broker := &fakeQueueBroker{}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
if err := queues.StartTask(context.Background(), assignments[0]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := queues.StopAll(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, err := st.LocalTaskAssignment(assignments[0].DispatcherID, assignments[0].TaskID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if current.AdmissionState != "running" {
|
|
t.Fatalf("shutdown changed durable admission to %q", current.AdmissionState)
|
|
}
|
|
}
|
|
|
|
func TestV3TaskQueueControllerPreservesBrokerDrainErrors(t *testing.T) {
|
|
now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
|
|
d, st, server := newLocalV01TestDispatcher(t, now)
|
|
defer server.Close()
|
|
defer st.Close()
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 1 {
|
|
t.Fatalf("assignments=%+v err=%v", assignments, err)
|
|
}
|
|
stopped, err := st.SetLocalTaskAdmissionBarrier(assignments[0].DispatcherID, assignments[0].TaskID, assignments[0].TenantID, assignments[0].TenantKey, "stopped")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := st.DB().Exec(`UPDATE local_v01_task_assignments SET saas_status='stopped' WHERE dispatcher_id=? AND task_id=?`, stopped.DispatcherID, stopped.TaskID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker := &errorDrainBroker{fakeQueueBroker: fakeQueueBroker{}, err: errors.New("drain failed")}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
if err := queues.DrainTask(context.Background(), stopped); !errors.Is(err, broker.err) {
|
|
t.Fatalf("DrainTask error=%v, want %v", err, broker.err)
|
|
}
|
|
}
|
|
|
|
type errorDrainBroker struct {
|
|
fakeQueueBroker
|
|
err error
|
|
}
|
|
|
|
func (f *errorDrainBroker) DrainPredeclared(context.Context, string) (int, error) {
|
|
return 0, f.err
|
|
}
|