//go:build integration package dispatcher import ( "context" "encoding/json" "errors" "fmt" "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" "git.ipao.vip/rogee/go-sip/internal/configread" "git.ipao.vip/rogee/go-sip/internal/mq" "git.ipao.vip/rogee/go-sip/internal/store" "git.ipao.vip/rogee/go-sip/internal/tenant" amqp "github.com/rabbitmq/amqp091-go" ) type currentRuntimeMockAgent struct { calls chan CurrentCallSpec controls chan CurrentControlSpec } func (a *currentRuntimeMockAgent) LoadedTrunks(context.Context) (map[string]int64, error) { return map[string]int64{"trunk-mock": 8}, nil } func (a *currentRuntimeMockAgent) Originate(_ context.Context, spec CurrentCallSpec) error { a.calls <- spec return nil } func (a *currentRuntimeMockAgent) SendControl(_ context.Context, spec CurrentControlSpec) error { a.controls <- spec return nil } func TestCurrentRuntimeIsolatedControlBacklogExecuteAndSharedResult(t *testing.T) { brokerURL, adminURL := os.Getenv("RABBITMQ_URL"), os.Getenv("RABBITMQ_PROVISIONER_URL") if brokerURL == "" || adminURL == "" { t.Skip("requires isolated RabbitMQ mock with provisioner account") } id := "c046b893-8628-4589-ae50-619d049248a6" adminConn, err := amqp.Dial(adminURL) if err != nil { t.Fatal(err) } defer adminConn.Close() admin, err := adminConn.Channel() if err != nil { t.Fatal(err) } defer admin.Close() for _, exchange := range []string{mq.CommandsExchangeCurrent, mq.ResultsExchangeCurrent, mq.DeadLetterExchangeCurrent} { if err := admin.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil { t.Fatal(err) } } controlRoute, _ := tenant.CurrentControlRoute(id) taskRoute, _ := tenant.CurrentTaskRoute(id, "task-asr") resultRoute, _ := tenant.CurrentResultRoute(id) shared := "agent-call.saas.events.v1" for _, queue := range []string{controlRoute.Queue, taskRoute.Queue, shared} { if _, err := admin.QueueDeclare(queue, true, false, false, false, nil); err != nil { t.Fatal(err) } defer func(name string) { _, _ = admin.QueueDelete(name, false, false, false) }(queue) if _, err := admin.QueuePurge(queue, false); err != nil { t.Fatal(err) } } for _, route := range []tenant.CurrentRoute{controlRoute, taskRoute, resultRoute} { queue := route.Queue if route.BindingKey == resultRoute.BindingKey { queue = shared } if err := admin.QueueBind(queue, route.BindingKey, route.Exchange, false, nil); err != nil { t.Fatal(err) } defer func(q, key, exchange string) { _ = admin.QueueUnbind(q, key, exchange, nil) }(queue, route.BindingKey, route.Exchange) } publish := func(route tenant.CurrentRoute, body []byte) { t.Helper() if err := admin.PublishWithContext(context.Background(), route.Exchange, route.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil { t.Fatal(err) } } publish(controlRoute, currentControlBody(t, "control-example", "pause", "drain")) publish(taskRoute, currentExecuteBody(t, "call-example", "15003164745")) snapshot := currentPolicySnapshot(t) sipJSON, err := json.Marshal(snapshot.SIP) if err != nil { t.Fatal(err) } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") var body []byte switch r.URL.Path { case "/internal/v1/dispatcher/sip": body = sipJSON case "/internal/v1/dispatcher/tasks": if r.URL.Query().Get("after") == "" { body = currentConfigExample(t, "task-discovery-page") } else if r.URL.Query().Get("after") != "" { body = currentConfigExample(t, "task-discovery-end") } else { w.WriteHeader(http.StatusBadRequest) return } case "/internal/v1/dispatcher/task/task-asr": body = currentConfigExample(t, "config-read-task-asr") case "/internal/v1/dispatcher/ai-providers": body = currentConfigExample(t, "config-read-providers") case "/internal/v1/dispatcher/tenant/1001/quota": body = currentConfigExample(t, "config-read-quota") default: t.Errorf("unexpected HTTP configuration path %s", r.URL.Path) w.WriteHeader(http.StatusNotFound) return } _, _ = w.Write(body) })) defer server.Close() client, err := configread.NewClient(server.URL, id, "test-secret", server.Client()) if err != nil { t.Fatal(err) } db, err := store.OpenCurrent(filepath.Join(t.TempDir(), "dispatcher.db")) if err != nil { t.Fatal(err) } defer db.Close() broker, err := mq.OpenCurrent(brokerURL, id, 1) if err != nil { t.Fatal(err) } defer broker.Close() agent := ¤tRuntimeMockAgent{calls: make(chan CurrentCallSpec, 3), controls: make(chan CurrentControlSpec, 3)} var windowAllowed atomic.Bool windowAllowed.Store(true) verify := func(_ context.Context, sip configread.CurrentSIP) error { if sip.Revision != 8 || sip.DispatcherID != id { return fmt.Errorf("unloaded SIP revision") } return nil } runtime := &CurrentRuntime{ Broker: broker, Bootstrap: CurrentBootstrap{DispatcherID: id, Client: client, Store: db, VerifySIP: verify}, Execute: CurrentExecuteController{DispatcherID: id, Store: db, Originator: agent, Publisher: broker, Now: func() time.Time { if windowAllowed.Load() { return currentMonday(9, 30) } return currentMonday(8, 59) }}, Control: CurrentControlController{DispatcherID: id, Store: db, Client: client, Agent: agent, VerifySIP: verify, Now: func() time.Time { return currentMonday(9, 30) }}, PollInterval: 30 * time.Millisecond, DiscoveryInterval: 120 * time.Millisecond, Logger: slog.Default(), } ctx, cancel := context.WithCancel(context.Background()) finished := make(chan struct{}) var runtimeErr error go func() { runtimeErr = runtime.Serve(ctx); close(finished) }() defer func() { cancel() select { case <-finished: if runtimeErr != nil && !errors.Is(runtimeErr, context.Canceled) { t.Errorf("runtime stopped: %v", runtimeErr) } case <-time.After(5 * time.Second): t.Error("runtime did not stop") } }() select { case spec := <-agent.controls: if spec.Action != "pause" || spec.ActiveCallPolicy != "drain" { t.Fatalf("wrong drained control: %+v", spec) } case <-finished: t.Fatalf("runtime failed during bootstrap: %v", runtimeErr) case <-time.After(5 * time.Second): t.Fatal("control backlog was not processed") } time.Sleep(120 * time.Millisecond) select { case call := <-agent.calls: t.Fatalf("paused task originated call: %s", call.EventID) default: } state, err := admin.QueueInspect(taskRoute.Queue) if err != nil || state.Messages != 1 { t.Fatalf("task queue was consumed before resume: %+v %v", state, err) } publish(controlRoute, currentControlBody(t, "resume-integration", "resume", "")) select { case spec := <-agent.controls: if spec.Action != "resume" { t.Fatalf("wrong control action: %+v", spec) } case <-finished: t.Fatalf("runtime failed before resume: %v", runtimeErr) case <-time.After(5 * time.Second): t.Fatal("resume control not delivered") } select { case spec := <-agent.calls: if spec.EventID != "call-example" || spec.TrunkID != "trunk-mock" { t.Fatalf("wrong dispatch: %+v", spec) } case <-finished: t.Fatalf("runtime failed before call dispatch: %v", runtimeErr) case <-time.After(5 * time.Second): t.Fatal("queued call not dispatched after resume") } seen := map[string]string{} until := time.After(5 * time.Second) for len(seen) < 3 { select { case <-until: t.Fatalf("missing SaaS shared-queue results: %+v", seen) default: } msg, ok, err := admin.Get(shared, false) if err != nil { t.Fatal(err) } if !ok { time.Sleep(20 * time.Millisecond) continue } var event struct { EventID string `json:"event_id"` EventType string `json:"event_type"` Payload struct { Status string `json:"status"` } `json:"payload"` } if err := json.Unmarshal(msg.Body, &event); err != nil { t.Fatal(err) } seen[event.EventID] = event.Payload.Status if err := msg.Ack(false); err != nil { t.Fatal(err) } } if seen["control-example"] != "applied" || seen["resume-integration"] != "applied" || seen["call-example"] != "dispatched" { t.Fatalf("wrong shared SaaS results: %+v", seen) } publish(taskRoute, currentExecuteBody(t, "call-example", "15003164745")) select { case spec := <-agent.calls: t.Fatalf("redelivery reoriginated call %s", spec.EventID) case <-time.After(200 * time.Millisecond): } // A temporary task rule wait is retained durably, then its consumer // stops so further instructions remain in SaaS's task queue. Admission // resumes from the original identity when the configured window opens. windowAllowed.Store(false) waiting := []byte(strings.Replace(string(currentExecuteBody(t, "window-wait-1", "15003164745")), `"issued_at":"2026-09-21T01:00:00Z"`, `"issued_at":"2026-09-20T00:00:00Z"`, 1)) publish(taskRoute, waiting) waitDeadline := time.After(5 * time.Second) for { count, err := db.PendingExecuteCount(id, 1001, "task-asr") if err != nil { t.Fatal(err) } queueState, err := admin.QueueInspect(taskRoute.Queue) if err != nil { t.Fatal(err) } if count == 1 && queueState.Consumers == 0 { break } select { case <-waitDeadline: t.Fatalf("task-local waiting consumer stayed active: pending=%d consumers=%d", count, queueState.Consumers) case <-time.After(20 * time.Millisecond): } } publish(taskRoute, currentExecuteBody(t, "window-wait-2", "15003164745")) time.Sleep(100 * time.Millisecond) queueState, err := admin.QueueInspect(taskRoute.Queue) if err != nil || queueState.Messages != 1 { t.Fatalf("rule-wait backlog was not retained in SaaS queue: %+v %v", queueState, err) } select { case spec := <-agent.calls: t.Fatalf("called outside allowed window: %s", spec.EventID) default: } windowAllowed.Store(true) select { case spec := <-agent.calls: if spec.EventID != "window-wait-1" { t.Fatalf("wrong waiting instruction resumed: %s", spec.EventID) } case <-finished: t.Fatalf("runtime failed before window reopened: %v", runtimeErr) case <-time.After(5 * time.Second): t.Fatal("eligible retained instruction did not resume") } // A SIP notification is ACKed only after its revision is durable. The // already-dispatched calls keep the revision change fenced, while the // independent control queue and shared result publisher continue. sipChange := []byte(strings.Replace(string(currentConfigExample(t, "mq-sip-change")), `"revision":8`, `"revision":9`, 1)) publish(controlRoute, sipChange) deadline := time.After(5 * time.Second) for { applied, pending, err := db.SIPState(id) if err != nil { t.Fatal(err) } if applied == 8 && pending == 9 { break } select { case <-deadline: t.Fatalf("SIP notification was not persisted: applied=%d pending=%d", applied, pending) case <-time.After(20 * time.Millisecond): } } publish(taskRoute, currentExecuteBody(t, "after-sip-change", "15003164745")) select { case spec := <-agent.calls: t.Fatalf("pending SIP change originated call %s", spec.EventID) case <-time.After(120 * time.Millisecond): } publish(controlRoute, currentControlBody(t, "stop-after-sip", "stop", "")) select { case spec := <-agent.controls: if spec.Action != "stop" || spec.ActiveCallPolicy != "hangup" { t.Fatalf("control blocked or changed by SIP reload: %+v", spec) } case <-finished: t.Fatalf("runtime failed while SIP was pending: %v", runtimeErr) case <-time.After(5 * time.Second): t.Fatal("stop control was blocked by pending SIP revision") } deadline = time.After(5 * time.Second) for { select { case <-deadline: t.Fatal("control acknowledgment did not reach shared SaaS result queue during SIP reload") default: } msg, ok, err := admin.Get(shared, false) if err != nil { t.Fatal(err) } if !ok { time.Sleep(20 * time.Millisecond) continue } var ack struct { EventID string `json:"event_id"` Payload struct { Status string `json:"status"` } `json:"payload"` } if err := json.Unmarshal(msg.Body, &ack); err != nil { t.Fatal(err) } if err := msg.Ack(false); err != nil { t.Fatal(err) } if ack.EventID == "stop-after-sip" { if ack.Payload.Status != "applied" { t.Fatalf("stop control falsely acknowledged during SIP reload: %+v", ack) } break } } }