286 lines
10 KiB
Go
286 lines
10 KiB
Go
//go:build integration
|
|
|
|
package mq
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/tenant"
|
|
amqp "github.com/rabbitmq/amqp091-go"
|
|
)
|
|
|
|
func TestBrokerSharedResultQueueAndNoConfigure(t *testing.T) {
|
|
url := os.Getenv("RABBITMQ_URL")
|
|
provisionerURL := os.Getenv("RABBITMQ_PROVISIONER_URL")
|
|
if url == "" || provisionerURL == "" {
|
|
t.Skip("isolated RabbitMQ Mock URLs not configured")
|
|
}
|
|
ids := []string{testDispatcherID, "550e8400-e29b-41d4-a716-446655440000"}
|
|
conn, err := amqp.Dial(provisionerURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.Close()
|
|
admin, err := conn.Channel()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer admin.Close()
|
|
for _, exchange := range []string{CommandsExchange, ResultsExchange, DeadLetterExchange} {
|
|
if err := admin.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
shared := "agent-call.saas.events.v1"
|
|
if _, err := admin.QueueDeclare(shared, true, false, false, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _, _ = admin.QueueDelete(shared, false, false, false) }()
|
|
queues := []string{}
|
|
for _, id := range ids {
|
|
control, _ := tenant.ControlRoute(id)
|
|
task, _ := tenant.TaskRoute(id, "task-asr")
|
|
result, _ := tenant.ResultRoute(id)
|
|
for _, route := range []tenant.Route{control, task} {
|
|
if _, err := admin.QueueDeclare(route.Queue, true, false, false, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
queues = append(queues, route.Queue)
|
|
if err := admin.QueueBind(route.Queue, route.BindingKey, route.Exchange, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := admin.QueueBind(shared, result.BindingKey, result.Exchange, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func(key string) { _ = admin.QueueUnbind(shared, key, ResultsExchange, nil) }(result.BindingKey)
|
|
}
|
|
defer func() {
|
|
for _, name := range queues {
|
|
_, _ = admin.QueueDelete(name, false, false, false)
|
|
}
|
|
}()
|
|
for _, name := range append(append([]string(nil), queues...), shared) {
|
|
if _, err := admin.QueuePurge(name, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// The Dispatcher account must not configure the topology, including the
|
|
// SaaS result queue. A failed declare closes its channel, not the connection.
|
|
probeConn, err := amqp.Dial(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
probe, err := probeConn.Channel()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := probe.QueueDeclare("should-never-be-created", true, false, false, false, nil); err == nil {
|
|
_, _ = admin.QueueDelete("should-never-be-created", false, false, false)
|
|
t.Fatal("Dispatcher unexpectedly has configure permission")
|
|
}
|
|
_ = probe.Close()
|
|
_ = probeConn.Close()
|
|
|
|
brokers := make([]*Broker, 0, 2)
|
|
for _, id := range ids {
|
|
broker, err := Open(url, id, 1)
|
|
if err != nil {
|
|
t.Fatalf("Dispatcher without configure could not open %s: %v", id, err)
|
|
}
|
|
brokers = append(brokers, broker)
|
|
defer broker.Close()
|
|
}
|
|
control, _ := tenant.ControlRoute(ids[0])
|
|
task, _ := tenant.TaskRoute(ids[0], "task-asr")
|
|
controlBody := currentMessage(t, "mq-control")
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, control.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: controlBody}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
processed, err := brokers[0].DrainControlPredeclared(context.Background(), control.Queue, func(_ context.Context, key string, body []byte) error {
|
|
if key != control.BindingKey || string(body) != string(controlBody) {
|
|
return fmt.Errorf("wrong control route or body")
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil || processed != 1 {
|
|
t.Fatalf("control backlog: processed=%d err=%v", processed, err)
|
|
}
|
|
body := currentMessage(t, "mq-execute")
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
got := make(chan []byte, 1)
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- brokers[0].ConsumePredeclared(ctx, task.Queue, func(_ context.Context, key string, message []byte) error {
|
|
if key != task.BindingKey {
|
|
return fmt.Errorf("wrong task routing key %q", key)
|
|
}
|
|
got <- append([]byte(nil), message...)
|
|
return nil
|
|
})
|
|
}()
|
|
select {
|
|
case message := <-got:
|
|
if string(message) != string(body) {
|
|
t.Errorf("received wrong task body")
|
|
}
|
|
case err := <-done:
|
|
t.Fatalf("task consumer ended: %v", err)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("task message not received")
|
|
}
|
|
cancel()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("consumer did not stop")
|
|
}
|
|
otherTask, _ := tenant.TaskRoute(ids[1], "task-asr")
|
|
if state, err := admin.QueueInspect(otherTask.Queue); err != nil || state.Messages != 0 {
|
|
t.Fatalf("D2 stole D1 command: %+v %v", state, err)
|
|
}
|
|
|
|
// Purge is permitted with read access and affects only the chosen task;
|
|
// it must not remove another Dispatcher's backlog or the control queue.
|
|
for i := 0; i < 3; i++ {
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, otherTask.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count, err := brokers[0].PurgeTaskQueue(context.Background(), "task-asr")
|
|
if err != nil || count != 3 {
|
|
t.Fatalf("task-only purge: removed=%d err=%v", count, err)
|
|
}
|
|
if state, err := admin.QueueInspect(task.Queue); err != nil || state.Messages != 0 {
|
|
t.Fatalf("D1 queue not empty: %+v %v", state, err)
|
|
}
|
|
if state, err := admin.QueueInspect(otherTask.Queue); err != nil || state.Messages != 1 {
|
|
t.Fatalf("D2 queue was purged: %+v %v", state, err)
|
|
}
|
|
if _, err := brokers[0].PurgeTaskQueue(context.Background(), "../foreign"); err == nil {
|
|
t.Fatal("invalid task ID accepted for purge")
|
|
}
|
|
|
|
// Stop closes the consumer channel before purge; an in-flight delivery
|
|
// must become Ready rather than surviving the purge as Unacked.
|
|
inFlight := make(chan struct{}, 1)
|
|
consumer, err := brokers[0].StartPredeclaredConsumer(context.Background(), task.Queue, func(ctx context.Context, _ string, _ []byte) error {
|
|
select {
|
|
case inFlight <- struct{}{}:
|
|
default:
|
|
}
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: currentMessage(t, "mq-execute")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-inFlight:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("task consumer did not receive in-flight delivery")
|
|
}
|
|
stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := consumer.Stop(stopCtx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count, err = brokers[0].PurgeTaskQueue(context.Background(), "task-asr")
|
|
if err != nil || count != 1 {
|
|
t.Fatalf("in-flight delivery survived stop and purge: removed=%d err=%v", count, err)
|
|
}
|
|
|
|
resultBody := currentMessage(t, "mq-result-no-recording")
|
|
for i, broker := range brokers {
|
|
message := resultBody
|
|
if i == 1 {
|
|
message = []byte(strings.Replace(string(resultBody), ids[0], ids[1], 1))
|
|
}
|
|
route, _ := tenant.ResultRoute(ids[i])
|
|
if err := broker.Publish(context.Background(), route.Exchange, route.BindingKey, message); err != nil {
|
|
t.Fatalf("result publish for D%d: %v", i+1, err)
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for range ids {
|
|
delivery, ok, err := admin.Get(shared, false)
|
|
if err != nil || !ok {
|
|
t.Fatalf("shared SaaS queue missing confirmed result: %v", err)
|
|
}
|
|
seen[delivery.RoutingKey] = true
|
|
if err := delivery.Ack(false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, id := range ids {
|
|
if !seen["d."+id+".out"] {
|
|
t.Fatalf("missing D-specific output binding for %s", id)
|
|
}
|
|
}
|
|
// A dead connection must not claim delivery. An explicit new session can
|
|
// publish the unchanged pending identity to the SaaS-owned shared queue.
|
|
route, _ := tenant.ResultRoute(ids[0])
|
|
pending := []byte(strings.Replace(string(resultBody), `"event_id":"call-result-example"`, `"event_id":"call-result-reopened"`, 1))
|
|
if err := brokers[0].Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := brokers[0].Publish(context.Background(), route.Exchange, route.BindingKey, pending); err == nil || !strings.Contains(err.Error(), "connection is closed") {
|
|
t.Fatalf("closed connection reported a successful publish: %v", err)
|
|
}
|
|
reopened, err := Open(url, ids[0], 1)
|
|
if err != nil {
|
|
t.Fatalf("explicit reconnect failed: %v", err)
|
|
}
|
|
defer reopened.Close()
|
|
brokers[0] = reopened
|
|
if err := reopened.Publish(context.Background(), route.Exchange, route.BindingKey, pending); err != nil {
|
|
t.Fatalf("original pending identity was not delivered after reconnect: %v", err)
|
|
}
|
|
delivery, ok, err := admin.Get(shared, false)
|
|
if err != nil || !ok || delivery.MessageId != "call-result-reopened" || string(delivery.Body) != string(pending) || delivery.DeliveryMode != amqp.Persistent {
|
|
t.Fatalf("reconnected result was not received by the SaaS-owned queue with its original identity: ok=%t err=%v", ok, err)
|
|
}
|
|
if err := delivery.Ack(false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, id := range ids {
|
|
route, _ := tenant.ResultRoute(id)
|
|
if err := admin.QueueUnbind(shared, route.BindingKey, route.Exchange, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
resultRoute, _ := tenant.ResultRoute(ids[0])
|
|
if err := brokers[0].Publish(context.Background(), resultRoute.Exchange, resultRoute.BindingKey, resultBody); err == nil {
|
|
t.Fatal("unroutable mandatory result falsely confirmed")
|
|
}
|
|
controlRoute, err := tenant.ControlRoute(ids[0])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := admin.PublishWithContext(context.Background(), controlRoute.Exchange, controlRoute.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: []byte("{")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
processed, err = brokers[0].DrainControlPredeclared(context.Background(), controlRoute.Queue, func(context.Context, string, []byte) error {
|
|
t.Fatal("invalid backlog control reached handler")
|
|
return nil
|
|
})
|
|
if err == nil || processed != 1 || !strings.Contains(err.Error(), "invalid control backlog") {
|
|
t.Fatalf("malformed control backlog was silently cleared: processed=%d err=%v", processed, err)
|
|
}
|
|
}
|