222 lines
9.2 KiB
Go
222 lines
9.2 KiB
Go
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/configread"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
)
|
|
|
|
const testSnapshotID = "22222222-2222-4222-8222-222222222222"
|
|
|
|
func newEventRuntime(t *testing.T, server *httptest.Server) (*LocalV01Runtime, *store.Store) {
|
|
t.Helper()
|
|
return newEventRuntimeAt(t, server, filepath.Join(t.TempDir(), "dispatcher.db"))
|
|
}
|
|
|
|
func newEventRuntimeAt(t *testing.T, server *httptest.Server, path string) (*LocalV01Runtime, *store.Store) {
|
|
t.Helper()
|
|
st, err := store.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = st.Close() })
|
|
d, err := NewV3(localTestDispatcherID, st, nil, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := configread.NewClient(server.URL, localTestDispatcherID, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return &LocalV01Runtime{dispatcher: d, client: client}, st
|
|
}
|
|
|
|
func snapshotTaskPage(taskID, status string, revision int64, next string) string {
|
|
nextJSON := "null"
|
|
if next != "" {
|
|
nextJSON = fmt.Sprintf("%q", next)
|
|
}
|
|
return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"snapshot","dispatcher_id":%q,"snapshot_id":%q,"watermark":"42","tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":%q,"task_revision":%d}],"next_page_token":%s}`,
|
|
localTestDispatcherID, testSnapshotID, taskID, localTestTenantID, localTestTenantKey, status, revision, nextJSON)
|
|
}
|
|
|
|
func emptySnapshotPage() string {
|
|
return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"snapshot","dispatcher_id":%q,"snapshot_id":%q,"watermark":"42","tasks":[],"next_page_token":null}`, localTestDispatcherID, testSnapshotID)
|
|
}
|
|
|
|
func emptyChangesPage(cursor string) string {
|
|
return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[],"next_cursor":%q}`, localTestDispatcherID, cursor)
|
|
}
|
|
|
|
func TestRuntimeCommitsCompleteSnapshotBeforeControlAndTaskAdmission(t *testing.T) {
|
|
var requests []string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests = append(requests, r.URL.RawQuery)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.RawQuery {
|
|
case "mode=snapshot":
|
|
fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "page-2"))
|
|
case "page_token=page-2&snapshot_id=" + testSnapshotID:
|
|
fmt.Fprint(w, snapshotTaskPage("task-b", "paused", 1, ""))
|
|
case "after=42":
|
|
fmt.Fprint(w, emptyChangesPage("42"))
|
|
default:
|
|
t.Errorf("unexpected query: %s", r.URL.RawQuery)
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
runtime, st := newEventRuntime(t, server)
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(requests, []string{"mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID}) || !runtime.complete || runtime.cursor != "42" {
|
|
t.Fatalf("snapshot incomplete or lost in-memory watermark: queries=%v complete=%v cursor=%q", requests, runtime.complete, runtime.cursor)
|
|
}
|
|
var ready int
|
|
if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 0 {
|
|
t.Fatalf("task admitted before control backlog: ready=%d err=%v", ready, err)
|
|
}
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 2 || assignments[0].TaskID != "task-b" || assignments[1].TaskID != localTestTaskID {
|
|
t.Fatalf("complete snapshot assignments=%+v err=%v", assignments, err)
|
|
}
|
|
runtime.queues = newTaskQueueController(runtime.dispatcher, &fakeQueueBroker{}, nil)
|
|
if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if requests[len(requests)-1] != "after=42" || runtime.cursor != "42" {
|
|
t.Fatalf("live discovery did not start from snapshot watermark: %v cursor=%q", requests, runtime.cursor)
|
|
}
|
|
if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 {
|
|
t.Fatalf("control backlog did not open admission: %d %v", ready, err)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeInterruptedSnapshotRestartsFromFirstPage(t *testing.T) {
|
|
var fail atomic.Bool
|
|
fail.Store(true)
|
|
var requests []string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests = append(requests, r.URL.RawQuery)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.RawQuery {
|
|
case "mode=snapshot":
|
|
fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "page-2"))
|
|
case "page_token=page-2&snapshot_id=" + testSnapshotID:
|
|
if fail.Load() {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
fmt.Fprint(w, `{"schema_version":"task-discovery.v0.4-proposal","resource":"error","error":{"code":"service_unavailable","message":"mock outage"}}`)
|
|
return
|
|
}
|
|
fmt.Fprint(w, emptySnapshotPage())
|
|
default:
|
|
t.Errorf("unexpected query %q", r.URL.RawQuery)
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
runtime, st := newEventRuntime(t, server)
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err == nil {
|
|
t.Fatal("partial snapshot accepted")
|
|
}
|
|
assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
|
|
if err != nil || len(assignments) != 0 || runtime.complete || runtime.cursor != "" {
|
|
t.Fatalf("failed snapshot persisted partial assignments: %+v err=%v complete=%v cursor=%q", assignments, err, runtime.complete, runtime.cursor)
|
|
}
|
|
fail.Store(false)
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []string{"mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID, "mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID}
|
|
if !reflect.DeepEqual(requests, want) || !runtime.complete || runtime.cursor != "42" {
|
|
t.Fatalf("partial snapshot resumed instead of restarting: requests=%v complete=%v cursor=%q", requests, runtime.complete, runtime.cursor)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeRestartDiscardsLiveCursorAndReloadsCompleteSnapshot(t *testing.T) {
|
|
var requests []string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests = append(requests, r.URL.RawQuery)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.RawQuery {
|
|
case "mode=snapshot":
|
|
fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, ""))
|
|
case "after=42":
|
|
fmt.Fprint(w, emptyChangesPage("42"))
|
|
default:
|
|
t.Errorf("unexpected query %q", r.URL.RawQuery)
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
path := filepath.Join(t.TempDir(), "dispatcher.db")
|
|
first, st := newEventRuntimeAt(t, server, path)
|
|
if err := first.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, _ := newEventRuntimeAt(t, server, path)
|
|
if second.complete || second.cursor != "" {
|
|
t.Fatalf("new runtime reused old cursor: %q complete=%v", second.cursor, second.complete)
|
|
}
|
|
if err := second.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(requests, []string{"mode=snapshot", "mode=snapshot"}) {
|
|
t.Fatalf("restart skipped full snapshot: requests=%v", requests)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeCatchesChangesArrivingAfterSnapshotWatermark(t *testing.T) {
|
|
var requests []string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests = append(requests, r.URL.RawQuery)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.RawQuery {
|
|
case "mode=snapshot":
|
|
fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, ""))
|
|
case "after=42":
|
|
fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":"removed","task_revision":2}],"next_cursor":"43"}`, localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey)
|
|
case "after=43":
|
|
fmt.Fprint(w, emptyChangesPage("43"))
|
|
default:
|
|
t.Errorf("unexpected query %q", r.URL.RawQuery)
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
runtime, st := newEventRuntime(t, server)
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runtime.queues = newTaskQueueController(runtime.dispatcher, &fakeQueueBroker{}, nil)
|
|
if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := runtime.refreshDiscoveryState(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID)
|
|
if err != nil || !assignment.Removed || assignment.TaskRevision != 2 || runtime.cursor != "43" {
|
|
t.Fatalf("event after snapshot was lost: %+v err=%v cursor=%q", assignment, err, runtime.cursor)
|
|
}
|
|
if !reflect.DeepEqual(requests, []string{"mode=snapshot", "after=42", "after=43"}) {
|
|
t.Fatalf("changes sequence=%v", requests)
|
|
}
|
|
}
|