package dispatcher import ( "context" "fmt" "net/http" "net/http/httptest" "path/filepath" "reflect" "sync/atomic" "testing" "time" "git.ipao.vip/rogee/go-sip/internal/configread" "git.ipao.vip/rogee/go-sip/internal/store" ) const testSnapshotID = "22222222-2222-4222-8222-222222222222" func newEventRuntime(t *testing.T, server *httptest.Server) (*LocalV01Runtime, *store.Store) { t.Helper() return newEventRuntimeAt(t, server, filepath.Join(t.TempDir(), "dispatcher.db")) } func newEventRuntimeAt(t *testing.T, server *httptest.Server, path string) (*LocalV01Runtime, *store.Store) { t.Helper() st, err := store.Open(path) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = st.Close() }) d, err := NewV3(localTestDispatcherID, st, nil, time.Now) if err != nil { t.Fatal(err) } client, err := configread.NewClient(server.URL, localTestDispatcherID, "test-secret", server.Client()) if err != nil { t.Fatal(err) } return &LocalV01Runtime{dispatcher: d, client: client}, st } func snapshotTaskPage(taskID, status string, revision int64, next string) string { nextJSON := "null" if next != "" { nextJSON = fmt.Sprintf("%q", next) } return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"snapshot","dispatcher_id":%q,"snapshot_id":%q,"watermark":"42","tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":%q,"task_revision":%d}],"next_page_token":%s}`, localTestDispatcherID, testSnapshotID, taskID, localTestTenantID, localTestTenantKey, status, revision, nextJSON) } func emptySnapshotPage() string { return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"snapshot","dispatcher_id":%q,"snapshot_id":%q,"watermark":"42","tasks":[],"next_page_token":null}`, localTestDispatcherID, testSnapshotID) } func emptyChangesPage(cursor string) string { return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[],"next_cursor":%q}`, localTestDispatcherID, cursor) } func TestRuntimeCommitsCompleteSnapshotBeforeControlAndTaskAdmission(t *testing.T) { var requests []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests = append(requests, r.URL.RawQuery) w.Header().Set("Content-Type", "application/json") switch r.URL.RawQuery { case "mode=snapshot": fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "page-2")) case "page_token=page-2&snapshot_id=" + testSnapshotID: fmt.Fprint(w, snapshotTaskPage("task-b", "paused", 1, "")) case "after=42": fmt.Fprint(w, emptyChangesPage("42")) default: t.Errorf("unexpected query: %s", r.URL.RawQuery) w.WriteHeader(http.StatusBadRequest) } })) defer server.Close() runtime, st := newEventRuntime(t, server) if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } if !reflect.DeepEqual(requests, []string{"mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID}) || !runtime.complete || runtime.cursor != "42" { t.Fatalf("snapshot incomplete or lost in-memory watermark: queries=%v complete=%v cursor=%q", requests, runtime.complete, runtime.cursor) } var ready int if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 0 { t.Fatalf("task admitted before control backlog: ready=%d err=%v", ready, err) } assignments, err := st.LocalTaskAssignments(localTestDispatcherID) if err != nil || len(assignments) != 2 || assignments[0].TaskID != "task-b" || assignments[1].TaskID != localTestTaskID { t.Fatalf("complete snapshot assignments=%+v err=%v", assignments, err) } runtime.queues = newTaskQueueController(runtime.dispatcher, &fakeQueueBroker{}, nil) if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { t.Fatal(err) } if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } if requests[len(requests)-1] != "after=42" || runtime.cursor != "42" { t.Fatalf("live discovery did not start from snapshot watermark: %v cursor=%q", requests, runtime.cursor) } if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 { t.Fatalf("control backlog did not open admission: %d %v", ready, err) } } func TestRuntimeInterruptedSnapshotRestartsFromFirstPage(t *testing.T) { var fail atomic.Bool fail.Store(true) var requests []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests = append(requests, r.URL.RawQuery) w.Header().Set("Content-Type", "application/json") switch r.URL.RawQuery { case "mode=snapshot": fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "page-2")) case "page_token=page-2&snapshot_id=" + testSnapshotID: if fail.Load() { w.WriteHeader(http.StatusServiceUnavailable) fmt.Fprint(w, `{"schema_version":"task-discovery.v0.4-proposal","resource":"error","error":{"code":"service_unavailable","message":"mock outage"}}`) return } fmt.Fprint(w, emptySnapshotPage()) default: t.Errorf("unexpected query %q", r.URL.RawQuery) w.WriteHeader(http.StatusBadRequest) } })) defer server.Close() runtime, st := newEventRuntime(t, server) if err := runtime.refreshDiscoveryState(context.Background()); err == nil { t.Fatal("partial snapshot accepted") } assignments, err := st.LocalTaskAssignments(localTestDispatcherID) if err != nil || len(assignments) != 0 || runtime.complete || runtime.cursor != "" { t.Fatalf("failed snapshot persisted partial assignments: %+v err=%v complete=%v cursor=%q", assignments, err, runtime.complete, runtime.cursor) } fail.Store(false) if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } want := []string{"mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID, "mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID} if !reflect.DeepEqual(requests, want) || !runtime.complete || runtime.cursor != "42" { t.Fatalf("partial snapshot resumed instead of restarting: requests=%v complete=%v cursor=%q", requests, runtime.complete, runtime.cursor) } } func TestRuntimeRestartDiscardsLiveCursorAndReloadsCompleteSnapshot(t *testing.T) { var requests []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests = append(requests, r.URL.RawQuery) w.Header().Set("Content-Type", "application/json") switch r.URL.RawQuery { case "mode=snapshot": fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "")) case "after=42": fmt.Fprint(w, emptyChangesPage("42")) default: t.Errorf("unexpected query %q", r.URL.RawQuery) w.WriteHeader(http.StatusBadRequest) } })) defer server.Close() path := filepath.Join(t.TempDir(), "dispatcher.db") first, st := newEventRuntimeAt(t, server, path) if err := first.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } if err := st.Close(); err != nil { t.Fatal(err) } second, _ := newEventRuntimeAt(t, server, path) if second.complete || second.cursor != "" { t.Fatalf("new runtime reused old cursor: %q complete=%v", second.cursor, second.complete) } if err := second.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } if !reflect.DeepEqual(requests, []string{"mode=snapshot", "mode=snapshot"}) { t.Fatalf("restart skipped full snapshot: requests=%v", requests) } } func TestRuntimeCatchesChangesArrivingAfterSnapshotWatermark(t *testing.T) { var requests []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests = append(requests, r.URL.RawQuery) w.Header().Set("Content-Type", "application/json") switch r.URL.RawQuery { case "mode=snapshot": fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "")) case "after=42": fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":"removed","task_revision":2}],"next_cursor":"43"}`, localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey) case "after=43": fmt.Fprint(w, emptyChangesPage("43")) default: t.Errorf("unexpected query %q", r.URL.RawQuery) w.WriteHeader(http.StatusBadRequest) } })) defer server.Close() runtime, st := newEventRuntime(t, server) if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } runtime.queues = newTaskQueueController(runtime.dispatcher, &fakeQueueBroker{}, nil) if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { t.Fatal(err) } if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID) if err != nil || !assignment.Removed || assignment.TaskRevision != 2 || runtime.cursor != "43" { t.Fatalf("event after snapshot was lost: %+v err=%v cursor=%q", assignment, err, runtime.cursor) } if !reflect.DeepEqual(requests, []string{"mode=snapshot", "after=42", "after=43"}) { t.Fatalf("changes sequence=%v", requests) } }