300 lines
11 KiB
Go
300 lines
11 KiB
Go
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/configread"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
)
|
|
|
|
func TestLocalV01RuntimeRefreshDiscoveryPersistsCompleteSnapshotAndCursor(t *testing.T) {
|
|
body := localConfigFixture(t, "task-discovery-snapshot-v0.2.json")
|
|
var afterValues []string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet || r.URL.Path != "/internal/v1/dispatcher/tasks" {
|
|
t.Errorf("discovery request = %s %s", r.Method, r.URL.RequestURI())
|
|
}
|
|
afterValues = append(afterValues, r.URL.Query().Get("after"))
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write(body)
|
|
}))
|
|
defer server.Close()
|
|
st, err := store.Open(filepath.Join(t.TempDir(), "dispatcher.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
d, err := NewV3(localTestDispatcherID, st, nil, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := configread.NewClient(server.URL, localTestDispatcherID, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runtime := &LocalV01Runtime{dispatcher: d, client: client}
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(afterValues) != 1 || afterValues[0] != "" {
|
|
t.Fatalf("initial discovery cursors = %v, want one full-snapshot request", afterValues)
|
|
}
|
|
cursor, exists, err := st.LocalTaskDiscoveryCursor(localTestDispatcherID)
|
|
if err != nil || !exists || cursor == "" {
|
|
t.Fatalf("persisted cursor=%q exists=%v err=%v", cursor, exists, err)
|
|
}
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 1 || assignments[0].TaskID != localTestTaskID {
|
|
t.Fatalf("persisted assignments=%+v err=%v", assignments, err)
|
|
}
|
|
}
|
|
|
|
func TestLocalV01RuntimeDiscoveryFailureClosesAdmissionAndStopsTaskConsumers(t *testing.T) {
|
|
var phase atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/internal/v1/dispatcher/tasks" {
|
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch phase.Load() {
|
|
case 0:
|
|
_, _ = w.Write(localConfigFixture(t, "task-discovery-snapshot-v0.2.json"))
|
|
case 1:
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_, _ = w.Write([]byte(`{"schema_version":"task-discovery.v0.2-proposal","resource":"error","error":{"code":"service_unavailable","message":"mock outage"}}`))
|
|
default:
|
|
_, _ = w.Write([]byte(`{"schema_version":"task-discovery.v0.2-proposal","dispatcher_id":"c046b893-8628-4589-ae50-619d049248a6","next_cursor":"opaque-watermark-001","changes":[]}`))
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
st, err := store.Open(filepath.Join(t.TempDir(), "dispatcher.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
d, err := NewV3(localTestDispatcherID, st, nil, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := configread.NewClient(server.URL, localTestDispatcherID, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker := &fakeQueueBroker{}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
runtime := &LocalV01Runtime{dispatcher: d, client: client, queues: queues}
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := queues.StartTask(context.Background(), assignment); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
phase.Store(1)
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err == nil {
|
|
t.Fatal("outage accepted as fresh discovery")
|
|
}
|
|
var ready int
|
|
if err := st.DB().QueryRow(`SELECT ready FROM local_v02_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 0 {
|
|
t.Fatalf("discovery gate after outage=%d err=%v", ready, err)
|
|
}
|
|
select {
|
|
case <-broker.consumers[0].done:
|
|
default:
|
|
t.Fatal("stale task consumer remained active after discovery outage")
|
|
}
|
|
phase.Store(2)
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.DB().QueryRow(`SELECT ready FROM local_v02_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 {
|
|
t.Fatalf("discovery gate after fresh no-change=%d err=%v", ready, err)
|
|
}
|
|
}
|
|
|
|
func TestLocalV01RuntimeExpiredCursorRecoversFullSnapshotAndRetiresTask(t *testing.T) {
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/internal/v1/dispatcher/tasks" {
|
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch calls.Add(1) {
|
|
case 1:
|
|
if r.URL.Query().Has("after") {
|
|
t.Error("initial read must be a full snapshot")
|
|
}
|
|
_, _ = w.Write(localConfigFixture(t, "task-discovery-snapshot-v0.2.json"))
|
|
case 2:
|
|
if r.URL.Query().Get("after") != "opaque-watermark-001" {
|
|
t.Errorf("incremental cursor = %s", r.URL.RawQuery)
|
|
}
|
|
w.WriteHeader(http.StatusGone)
|
|
_, _ = w.Write([]byte(`{"schema_version":"task-discovery.v0.2-proposal","resource":"error","error":{"code":"cursor_expired","message":"mock expired"}}`))
|
|
case 3:
|
|
if r.URL.Query().Has("after") {
|
|
t.Errorf("recovery must be a full snapshot: %s", r.URL.RawQuery)
|
|
}
|
|
_, _ = w.Write([]byte(`{"schema_version":"task-discovery.v0.2-proposal","dispatcher_id":"c046b893-8628-4589-ae50-619d049248a6","cursor":"opaque-watermark-002","tasks":[]}`))
|
|
default:
|
|
t.Error("unexpected fourth request")
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
st, err := store.Open(filepath.Join(t.TempDir(), "dispatcher.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
d, err := NewV3(localTestDispatcherID, st, nil, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := configread.NewClient(server.URL, localTestDispatcherID, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker := &fakeQueueBroker{}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
runtime := &LocalV01Runtime{dispatcher: d, client: client, queues: queues}
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := queues.StartTask(context.Background(), assignment); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := runtime.reconcileTaskQueues(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cursor, exists, err := st.LocalTaskDiscoveryCursor(localTestDispatcherID)
|
|
if err != nil || !exists || cursor != "opaque-watermark-002" || calls.Load() != 3 {
|
|
t.Fatalf("recovered cursor=%q exists=%v requests=%d err=%v", cursor, exists, calls.Load(), err)
|
|
}
|
|
assignment, err = st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID)
|
|
if err != nil || !assignment.Removed || assignment.AdmissionState != "removed" {
|
|
t.Fatalf("retired assignment=%+v err=%v", assignment, err)
|
|
}
|
|
select {
|
|
case <-broker.consumers[0].done:
|
|
default:
|
|
t.Fatal("removed task remained subscribed")
|
|
}
|
|
}
|
|
|
|
func TestLocalV01RuntimeReconcilesOnlyEligibleAssignments(t *testing.T) {
|
|
now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
|
|
d, st, server := newLocalV01TestDispatcher(t, now)
|
|
defer server.Close()
|
|
defer st.Close()
|
|
broker := &fakeQueueBroker{}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
runtime := &LocalV01Runtime{dispatcher: d, queues: queues}
|
|
if err := runtime.reconcileTaskQueues(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(broker.started) != 1 || broker.started[0] == "" {
|
|
t.Fatalf("eligible task queue subscriptions=%v", broker.started)
|
|
}
|
|
if err := queues.StopAll(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestLocalV01RuntimeReconcileDoesNotConsumePausedAssignment(t *testing.T) {
|
|
now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
|
|
d, st, server := newLocalV01TestDispatcher(t, now)
|
|
defer server.Close()
|
|
defer st.Close()
|
|
assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assignment, err = st.SetLocalTaskAdmissionBarrier(assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, "paused")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker := &fakeQueueBroker{}
|
|
queues := newTaskQueueController(d, broker, nil)
|
|
runtime := &LocalV01Runtime{dispatcher: d, queues: queues}
|
|
if err := runtime.reconcileTaskQueues(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(broker.started) != 0 || len(broker.drained) != 0 {
|
|
t.Fatalf("paused task consumed or drained its backlog: started=%v drained=%v", broker.started, broker.drained)
|
|
}
|
|
}
|
|
|
|
func TestTaskAssignmentCanConsumeRequiresRunningAssignedQueue(t *testing.T) {
|
|
base := store.LocalTaskAssignment{
|
|
Status: "running", AdmissionState: "running",
|
|
Queue: store.LocalTaskQueue{QueueName: "saas-owned-queue"},
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
assignment store.LocalTaskAssignment
|
|
want bool
|
|
}{
|
|
{name: "eligible", assignment: base, want: true},
|
|
{name: "paused status", assignment: withTaskStatus(base, "paused"), want: false},
|
|
{name: "closed admission", assignment: withAdmissionState(base, "paused"), want: false},
|
|
{name: "removed", assignment: withRemoved(base), want: false},
|
|
{name: "queue missing", assignment: withQueueName(base, ""), want: false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := taskAssignmentCanConsume(tc.assignment); got != tc.want {
|
|
t.Fatalf("taskAssignmentCanConsume()=%v, want %v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewLocalV01RuntimeRequiresDependencies(t *testing.T) {
|
|
if _, err := NewLocalV01Runtime(nil, nil, nil, nil, nil); err == nil {
|
|
t.Fatal("runtime accepted missing dependencies")
|
|
}
|
|
}
|
|
|
|
func TestLocalV01RuntimeRunFailsClosedWhenUnconfigured(t *testing.T) {
|
|
var runtime LocalV01Runtime
|
|
if err := runtime.Run(context.Background()); err == nil {
|
|
t.Fatal("unconfigured runtime started")
|
|
}
|
|
}
|
|
|
|
func withTaskStatus(assignment store.LocalTaskAssignment, status string) store.LocalTaskAssignment {
|
|
assignment.Status = status
|
|
return assignment
|
|
}
|
|
|
|
func withAdmissionState(assignment store.LocalTaskAssignment, state string) store.LocalTaskAssignment {
|
|
assignment.AdmissionState = state
|
|
return assignment
|
|
}
|
|
|
|
func withRemoved(assignment store.LocalTaskAssignment) store.LocalTaskAssignment {
|
|
assignment.Removed = true
|
|
return assignment
|
|
}
|
|
|
|
func withQueueName(assignment store.LocalTaskAssignment, name string) store.LocalTaskAssignment {
|
|
assignment.Queue.QueueName = name
|
|
return assignment
|
|
}
|