381 lines
12 KiB
Go
381 lines
12 KiB
Go
//go:build integration
|
|
|
|
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/configread"
|
|
"git.ipao.vip/rogee/go-sip/internal/mq"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
"git.ipao.vip/rogee/go-sip/internal/tenant"
|
|
amqp "github.com/rabbitmq/amqp091-go"
|
|
)
|
|
|
|
type currentRuntimeMockAgent struct {
|
|
calls chan CurrentCallSpec
|
|
controls chan CurrentControlSpec
|
|
}
|
|
|
|
func (a *currentRuntimeMockAgent) LoadedTrunks(context.Context) (map[string]int64, error) {
|
|
return map[string]int64{"trunk-mock": 8}, nil
|
|
}
|
|
func (a *currentRuntimeMockAgent) Originate(_ context.Context, spec CurrentCallSpec) error {
|
|
a.calls <- spec
|
|
return nil
|
|
}
|
|
func (a *currentRuntimeMockAgent) SendControl(_ context.Context, spec CurrentControlSpec) error {
|
|
a.controls <- spec
|
|
return nil
|
|
}
|
|
|
|
func TestCurrentRuntimeIsolatedControlBacklogExecuteAndSharedResult(t *testing.T) {
|
|
brokerURL, adminURL := os.Getenv("RABBITMQ_URL"), os.Getenv("RABBITMQ_PROVISIONER_URL")
|
|
if brokerURL == "" || adminURL == "" {
|
|
t.Skip("requires isolated RabbitMQ mock with provisioner account")
|
|
}
|
|
id := "c046b893-8628-4589-ae50-619d049248a6"
|
|
adminConn, err := amqp.Dial(adminURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer adminConn.Close()
|
|
admin, err := adminConn.Channel()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer admin.Close()
|
|
for _, exchange := range []string{mq.CommandsExchangeCurrent, mq.ResultsExchangeCurrent, mq.DeadLetterExchangeCurrent} {
|
|
if err := admin.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
controlRoute, _ := tenant.CurrentControlRoute(id)
|
|
taskRoute, _ := tenant.CurrentTaskRoute(id, "task-asr")
|
|
resultRoute, _ := tenant.CurrentResultRoute(id)
|
|
shared := "agent-call.saas.events.v1"
|
|
for _, queue := range []string{controlRoute.Queue, taskRoute.Queue, shared} {
|
|
if _, err := admin.QueueDeclare(queue, true, false, false, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func(name string) { _, _ = admin.QueueDelete(name, false, false, false) }(queue)
|
|
if _, err := admin.QueuePurge(queue, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, route := range []tenant.CurrentRoute{controlRoute, taskRoute, resultRoute} {
|
|
queue := route.Queue
|
|
if route.BindingKey == resultRoute.BindingKey {
|
|
queue = shared
|
|
}
|
|
if err := admin.QueueBind(queue, route.BindingKey, route.Exchange, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func(q, key, exchange string) { _ = admin.QueueUnbind(q, key, exchange, nil) }(queue, route.BindingKey, route.Exchange)
|
|
}
|
|
publish := func(route tenant.CurrentRoute, body []byte) {
|
|
t.Helper()
|
|
if err := admin.PublishWithContext(context.Background(), route.Exchange, route.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
publish(controlRoute, currentControlBody(t, "control-example", "pause", "drain"))
|
|
publish(taskRoute, currentExecuteBody(t, "call-example", "15003164745"))
|
|
|
|
snapshot := currentPolicySnapshot(t)
|
|
sipJSON, err := json.Marshal(snapshot.SIP)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
var body []byte
|
|
switch r.URL.Path {
|
|
case "/internal/v1/dispatcher/sip":
|
|
body = sipJSON
|
|
case "/internal/v1/dispatcher/tasks":
|
|
if r.URL.Query().Get("after") == "" {
|
|
body = currentConfigExample(t, "task-discovery-page")
|
|
} else if r.URL.Query().Get("after") != "" {
|
|
body = currentConfigExample(t, "task-discovery-end")
|
|
} else {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
case "/internal/v1/dispatcher/task/task-asr":
|
|
body = currentConfigExample(t, "config-read-task-asr")
|
|
case "/internal/v1/dispatcher/ai-providers":
|
|
body = currentConfigExample(t, "config-read-providers")
|
|
case "/internal/v1/dispatcher/tenant/1001/quota":
|
|
body = currentConfigExample(t, "config-read-quota")
|
|
default:
|
|
t.Errorf("unexpected HTTP configuration path %s", r.URL.Path)
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
_, _ = w.Write(body)
|
|
}))
|
|
defer server.Close()
|
|
client, err := configread.NewClient(server.URL, id, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db, err := store.OpenCurrent(filepath.Join(t.TempDir(), "dispatcher.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
broker, err := mq.OpenCurrent(brokerURL, id, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer broker.Close()
|
|
agent := ¤tRuntimeMockAgent{calls: make(chan CurrentCallSpec, 3), controls: make(chan CurrentControlSpec, 3)}
|
|
var windowAllowed atomic.Bool
|
|
windowAllowed.Store(true)
|
|
verify := func(_ context.Context, sip configread.CurrentSIP) error {
|
|
if sip.Revision != 8 || sip.DispatcherID != id {
|
|
return fmt.Errorf("unloaded SIP revision")
|
|
}
|
|
return nil
|
|
}
|
|
runtime := &CurrentRuntime{
|
|
Broker: broker,
|
|
Bootstrap: CurrentBootstrap{DispatcherID: id, Client: client, Store: db, VerifySIP: verify},
|
|
Execute: CurrentExecuteController{DispatcherID: id, Store: db, Originator: agent, Publisher: broker, Now: func() time.Time {
|
|
if windowAllowed.Load() {
|
|
return currentMonday(9, 30)
|
|
}
|
|
return currentMonday(8, 59)
|
|
}},
|
|
Control: CurrentControlController{DispatcherID: id, Store: db, Client: client, Agent: agent, VerifySIP: verify, Now: func() time.Time { return currentMonday(9, 30) }},
|
|
PollInterval: 30 * time.Millisecond, DiscoveryInterval: 120 * time.Millisecond, Logger: slog.Default(),
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
finished := make(chan struct{})
|
|
var runtimeErr error
|
|
go func() { runtimeErr = runtime.Serve(ctx); close(finished) }()
|
|
defer func() {
|
|
cancel()
|
|
select {
|
|
case <-finished:
|
|
if runtimeErr != nil && !errors.Is(runtimeErr, context.Canceled) {
|
|
t.Errorf("runtime stopped: %v", runtimeErr)
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Error("runtime did not stop")
|
|
}
|
|
}()
|
|
select {
|
|
case spec := <-agent.controls:
|
|
if spec.Action != "pause" || spec.ActiveCallPolicy != "drain" {
|
|
t.Fatalf("wrong drained control: %+v", spec)
|
|
}
|
|
case <-finished:
|
|
t.Fatalf("runtime failed during bootstrap: %v", runtimeErr)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("control backlog was not processed")
|
|
}
|
|
time.Sleep(120 * time.Millisecond)
|
|
select {
|
|
case call := <-agent.calls:
|
|
t.Fatalf("paused task originated call: %s", call.EventID)
|
|
default:
|
|
}
|
|
state, err := admin.QueueInspect(taskRoute.Queue)
|
|
if err != nil || state.Messages != 1 {
|
|
t.Fatalf("task queue was consumed before resume: %+v %v", state, err)
|
|
}
|
|
publish(controlRoute, currentControlBody(t, "resume-integration", "resume", ""))
|
|
select {
|
|
case spec := <-agent.controls:
|
|
if spec.Action != "resume" {
|
|
t.Fatalf("wrong control action: %+v", spec)
|
|
}
|
|
case <-finished:
|
|
t.Fatalf("runtime failed before resume: %v", runtimeErr)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("resume control not delivered")
|
|
}
|
|
select {
|
|
case spec := <-agent.calls:
|
|
if spec.EventID != "call-example" || spec.TrunkID != "trunk-mock" {
|
|
t.Fatalf("wrong dispatch: %+v", spec)
|
|
}
|
|
case <-finished:
|
|
t.Fatalf("runtime failed before call dispatch: %v", runtimeErr)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("queued call not dispatched after resume")
|
|
}
|
|
seen := map[string]string{}
|
|
until := time.After(5 * time.Second)
|
|
for len(seen) < 3 {
|
|
select {
|
|
case <-until:
|
|
t.Fatalf("missing SaaS shared-queue results: %+v", seen)
|
|
default:
|
|
}
|
|
msg, ok, err := admin.Get(shared, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !ok {
|
|
time.Sleep(20 * time.Millisecond)
|
|
continue
|
|
}
|
|
var event struct {
|
|
EventID string `json:"event_id"`
|
|
EventType string `json:"event_type"`
|
|
Payload struct {
|
|
Status string `json:"status"`
|
|
} `json:"payload"`
|
|
}
|
|
if err := json.Unmarshal(msg.Body, &event); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seen[event.EventID] = event.Payload.Status
|
|
if err := msg.Ack(false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if seen["control-example"] != "applied" || seen["resume-integration"] != "applied" || seen["call-example"] != "dispatched" {
|
|
t.Fatalf("wrong shared SaaS results: %+v", seen)
|
|
}
|
|
publish(taskRoute, currentExecuteBody(t, "call-example", "15003164745"))
|
|
select {
|
|
case spec := <-agent.calls:
|
|
t.Fatalf("redelivery reoriginated call %s", spec.EventID)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
// A temporary task rule wait is retained durably, then its consumer
|
|
// stops so further instructions remain in SaaS's task queue. Admission
|
|
// resumes from the original identity when the configured window opens.
|
|
windowAllowed.Store(false)
|
|
waiting := []byte(strings.Replace(string(currentExecuteBody(t, "window-wait-1", "15003164745")), `"issued_at":"2026-09-21T01:00:00Z"`, `"issued_at":"2026-09-20T00:00:00Z"`, 1))
|
|
publish(taskRoute, waiting)
|
|
waitDeadline := time.After(5 * time.Second)
|
|
for {
|
|
count, err := db.PendingExecuteCount(id, 1001, "task-asr")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
queueState, err := admin.QueueInspect(taskRoute.Queue)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count == 1 && queueState.Consumers == 0 {
|
|
break
|
|
}
|
|
select {
|
|
case <-waitDeadline:
|
|
t.Fatalf("task-local waiting consumer stayed active: pending=%d consumers=%d", count, queueState.Consumers)
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
}
|
|
publish(taskRoute, currentExecuteBody(t, "window-wait-2", "15003164745"))
|
|
time.Sleep(100 * time.Millisecond)
|
|
queueState, err := admin.QueueInspect(taskRoute.Queue)
|
|
if err != nil || queueState.Messages != 1 {
|
|
t.Fatalf("rule-wait backlog was not retained in SaaS queue: %+v %v", queueState, err)
|
|
}
|
|
select {
|
|
case spec := <-agent.calls:
|
|
t.Fatalf("called outside allowed window: %s", spec.EventID)
|
|
default:
|
|
}
|
|
windowAllowed.Store(true)
|
|
select {
|
|
case spec := <-agent.calls:
|
|
if spec.EventID != "window-wait-1" {
|
|
t.Fatalf("wrong waiting instruction resumed: %s", spec.EventID)
|
|
}
|
|
case <-finished:
|
|
t.Fatalf("runtime failed before window reopened: %v", runtimeErr)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("eligible retained instruction did not resume")
|
|
}
|
|
// A SIP notification is ACKed only after its revision is durable. The
|
|
// already-dispatched calls keep the revision change fenced, while the
|
|
// independent control queue and shared result publisher continue.
|
|
sipChange := []byte(strings.Replace(string(currentConfigExample(t, "mq-sip-change")), `"revision":8`, `"revision":9`, 1))
|
|
publish(controlRoute, sipChange)
|
|
deadline := time.After(5 * time.Second)
|
|
for {
|
|
applied, pending, err := db.SIPState(id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if applied == 8 && pending == 9 {
|
|
break
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatalf("SIP notification was not persisted: applied=%d pending=%d", applied, pending)
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
}
|
|
publish(taskRoute, currentExecuteBody(t, "after-sip-change", "15003164745"))
|
|
select {
|
|
case spec := <-agent.calls:
|
|
t.Fatalf("pending SIP change originated call %s", spec.EventID)
|
|
case <-time.After(120 * time.Millisecond):
|
|
}
|
|
publish(controlRoute, currentControlBody(t, "stop-after-sip", "stop", ""))
|
|
select {
|
|
case spec := <-agent.controls:
|
|
if spec.Action != "stop" || spec.ActiveCallPolicy != "hangup" {
|
|
t.Fatalf("control blocked or changed by SIP reload: %+v", spec)
|
|
}
|
|
case <-finished:
|
|
t.Fatalf("runtime failed while SIP was pending: %v", runtimeErr)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("stop control was blocked by pending SIP revision")
|
|
}
|
|
deadline = time.After(5 * time.Second)
|
|
for {
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("control acknowledgment did not reach shared SaaS result queue during SIP reload")
|
|
default:
|
|
}
|
|
msg, ok, err := admin.Get(shared, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !ok {
|
|
time.Sleep(20 * time.Millisecond)
|
|
continue
|
|
}
|
|
var ack struct {
|
|
EventID string `json:"event_id"`
|
|
Payload struct {
|
|
Status string `json:"status"`
|
|
} `json:"payload"`
|
|
}
|
|
if err := json.Unmarshal(msg.Body, &ack); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := msg.Ack(false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ack.EventID == "stop-after-sip" {
|
|
if ack.Payload.Status != "applied" {
|
|
t.Fatalf("stop control falsely acknowledged during SIP reload: %+v", ack)
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|