//go:build integration package mq import ( "context" "fmt" "os" "strings" "testing" "time" "git.ipao.vip/rogee/go-sip/internal/tenant" amqp "github.com/rabbitmq/amqp091-go" ) func TestBrokerSharedResultQueueAndNoConfigure(t *testing.T) { url := os.Getenv("RABBITMQ_URL") provisionerURL := os.Getenv("RABBITMQ_PROVISIONER_URL") if url == "" || provisionerURL == "" { t.Skip("isolated RabbitMQ Mock URLs not configured") } ids := []string{testDispatcherID, "550e8400-e29b-41d4-a716-446655440000"} conn, err := amqp.Dial(provisionerURL) if err != nil { t.Fatal(err) } defer conn.Close() admin, err := conn.Channel() if err != nil { t.Fatal(err) } defer admin.Close() // Only the two business exchanges are provisioned. Opening a Dispatcher // must not depend on an unused dead-letter exchange. for _, exchange := range []string{CommandsExchange, ResultsExchange} { if err := admin.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil { t.Fatal(err) } } shared := "agent-call.saas.events.v1" if _, err := admin.QueueDeclare(shared, true, false, false, false, nil); err != nil { t.Fatal(err) } defer func() { _, _ = admin.QueueDelete(shared, false, false, false) }() queues := []string{} for _, id := range ids { control, _ := tenant.ControlRoute(id) task, _ := tenant.TaskRoute(id, "task-asr") result, _ := tenant.ResultRoute(id) for _, route := range []tenant.Route{control, task} { if _, err := admin.QueueDeclare(route.Queue, true, false, false, false, nil); err != nil { t.Fatal(err) } queues = append(queues, route.Queue) if err := admin.QueueBind(route.Queue, route.BindingKey, route.Exchange, false, nil); err != nil { t.Fatal(err) } } if err := admin.QueueBind(shared, result.BindingKey, result.Exchange, false, nil); err != nil { t.Fatal(err) } defer func(key string) { _ = admin.QueueUnbind(shared, key, ResultsExchange, nil) }(result.BindingKey) } defer func() { for _, name := range queues { _, _ = admin.QueueDelete(name, false, false, false) } }() for _, name := range append(append([]string(nil), queues...), shared) { if _, err := admin.QueuePurge(name, false); err != nil { t.Fatal(err) } } // The Dispatcher account must not configure the topology, including the // SaaS result queue. A failed declare closes its channel, not the connection. probeConn, err := amqp.Dial(url) if err != nil { t.Fatal(err) } probe, err := probeConn.Channel() if err != nil { t.Fatal(err) } if _, err := probe.QueueDeclare("should-never-be-created", true, false, false, false, nil); err == nil { _, _ = admin.QueueDelete("should-never-be-created", false, false, false) t.Fatal("Dispatcher unexpectedly has configure permission") } _ = probe.Close() _ = probeConn.Close() brokers := make([]*Broker, 0, 2) for _, id := range ids { broker, err := Open(url, id, 1) if err != nil { t.Fatalf("Dispatcher without configure could not open %s: %v", id, err) } brokers = append(brokers, broker) defer broker.Close() } control, _ := tenant.ControlRoute(ids[0]) task, _ := tenant.TaskRoute(ids[0], "task-asr") controlBody := currentMessage(t, "mq-control") if err := admin.PublishWithContext(context.Background(), CommandsExchange, control.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: controlBody}); err != nil { t.Fatal(err) } processed, err := brokers[0].DrainControlPredeclared(context.Background(), control.Queue, func(_ context.Context, key string, body []byte) error { if key != control.BindingKey || string(body) != string(controlBody) { return fmt.Errorf("wrong control route or body") } return nil }) if err != nil || processed != 1 { t.Fatalf("control backlog: processed=%d err=%v", processed, err) } body := currentMessage(t, "mq-execute") if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) got := make(chan []byte, 1) done := make(chan error, 1) go func() { done <- brokers[0].ConsumePredeclared(ctx, task.Queue, func(_ context.Context, key string, message []byte) error { if key != task.BindingKey { return fmt.Errorf("wrong task routing key %q", key) } got <- append([]byte(nil), message...) return nil }) }() select { case message := <-got: if string(message) != string(body) { t.Errorf("received wrong task body") } case err := <-done: t.Fatalf("task consumer ended: %v", err) case <-time.After(5 * time.Second): t.Fatal("task message not received") } cancel() select { case <-done: case <-time.After(5 * time.Second): t.Fatal("consumer did not stop") } otherTask, _ := tenant.TaskRoute(ids[1], "task-asr") if state, err := admin.QueueInspect(otherTask.Queue); err != nil || state.Messages != 0 { t.Fatalf("D2 stole D1 command: %+v %v", state, err) } // Purge is permitted with read access and affects only the chosen task; // it must not remove another Dispatcher's backlog or the control queue. for i := 0; i < 3; i++ { if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil { t.Fatal(err) } } if err := admin.PublishWithContext(context.Background(), CommandsExchange, otherTask.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil { t.Fatal(err) } count, err := brokers[0].PurgeTaskQueue(context.Background(), "task-asr") if err != nil || count != 3 { t.Fatalf("task-only purge: removed=%d err=%v", count, err) } if state, err := admin.QueueInspect(task.Queue); err != nil || state.Messages != 0 { t.Fatalf("D1 queue not empty: %+v %v", state, err) } if state, err := admin.QueueInspect(otherTask.Queue); err != nil || state.Messages != 1 { t.Fatalf("D2 queue was purged: %+v %v", state, err) } if _, err := brokers[0].PurgeTaskQueue(context.Background(), "../foreign"); err == nil { t.Fatal("invalid task ID accepted for purge") } // Stop closes the consumer channel before purge; an in-flight delivery // must become Ready rather than surviving the purge as Unacked. inFlight := make(chan struct{}, 1) consumer, err := brokers[0].StartPredeclaredConsumer(context.Background(), task.Queue, func(ctx context.Context, _ string, _ []byte) error { select { case inFlight <- struct{}{}: default: } <-ctx.Done() return ctx.Err() }) if err != nil { t.Fatal(err) } if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: currentMessage(t, "mq-execute")}); err != nil { t.Fatal(err) } select { case <-inFlight: case <-time.After(3 * time.Second): t.Fatal("task consumer did not receive in-flight delivery") } stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := consumer.Stop(stopCtx); err != nil { t.Fatal(err) } count, err = brokers[0].PurgeTaskQueue(context.Background(), "task-asr") if err != nil || count != 1 { t.Fatalf("in-flight delivery survived stop and purge: removed=%d err=%v", count, err) } resultBody := currentMessage(t, "mq-result-no-recording") for i, broker := range brokers { message := resultBody if i == 1 { message = []byte(strings.Replace(string(resultBody), ids[0], ids[1], 1)) } route, _ := tenant.ResultRoute(ids[i]) if err := broker.Publish(context.Background(), route.Exchange, route.BindingKey, message); err != nil { t.Fatalf("result publish for D%d: %v", i+1, err) } } seen := map[string]bool{} for range ids { delivery, ok, err := admin.Get(shared, false) if err != nil || !ok { t.Fatalf("shared SaaS queue missing confirmed result: %v", err) } seen[delivery.RoutingKey] = true if err := delivery.Ack(false); err != nil { t.Fatal(err) } } for _, id := range ids { if !seen["d."+id+".out"] { t.Fatalf("missing D-specific output binding for %s", id) } } // A dead connection must not claim delivery. An explicit new session can // publish the unchanged pending identity to the SaaS-owned shared queue. route, _ := tenant.ResultRoute(ids[0]) pending := []byte(strings.Replace(string(resultBody), `"event_id":"call-result-example"`, `"event_id":"call-result-reopened"`, 1)) if err := brokers[0].Close(); err != nil { t.Fatal(err) } if err := brokers[0].Publish(context.Background(), route.Exchange, route.BindingKey, pending); err == nil || !strings.Contains(err.Error(), "connection is closed") { t.Fatalf("closed connection reported a successful publish: %v", err) } reopened, err := Open(url, ids[0], 1) if err != nil { t.Fatalf("explicit reconnect failed: %v", err) } defer reopened.Close() brokers[0] = reopened if err := reopened.Publish(context.Background(), route.Exchange, route.BindingKey, pending); err != nil { t.Fatalf("original pending identity was not delivered after reconnect: %v", err) } delivery, ok, err := admin.Get(shared, false) if err != nil || !ok || delivery.MessageId != "call-result-reopened" || string(delivery.Body) != string(pending) || delivery.DeliveryMode != amqp.Persistent { t.Fatalf("reconnected result was not received by the SaaS-owned queue with its original identity: ok=%t err=%v", ok, err) } if err := delivery.Ack(false); err != nil { t.Fatal(err) } for _, id := range ids { route, _ := tenant.ResultRoute(id) if err := admin.QueueUnbind(shared, route.BindingKey, route.Exchange, nil); err != nil { t.Fatal(err) } } resultRoute, _ := tenant.ResultRoute(ids[0]) if err := brokers[0].Publish(context.Background(), resultRoute.Exchange, resultRoute.BindingKey, resultBody); err == nil { t.Fatal("unroutable mandatory result falsely confirmed") } controlRoute, err := tenant.ControlRoute(ids[0]) if err != nil { t.Fatal(err) } if err := admin.PublishWithContext(context.Background(), controlRoute.Exchange, controlRoute.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: []byte("{")}); err != nil { t.Fatal(err) } processed, err = brokers[0].DrainControlPredeclared(context.Background(), controlRoute.Queue, func(context.Context, string, []byte) error { t.Fatal("invalid backlog control reached handler") return nil }) if err == nil || processed != 1 || !strings.Contains(err.Error(), "invalid control backlog") { t.Fatalf("malformed control backlog was silently cleared: processed=%d err=%v", processed, err) } }