400 lines
15 KiB
Go
400 lines
15 KiB
Go
//go:build integration
|
|
|
|
package mq
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/configread"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
"git.ipao.vip/rogee/go-sip/internal/tenant"
|
|
amqp "github.com/rabbitmq/amqp091-go"
|
|
)
|
|
|
|
func TestBrokerSharedResultQueueAndNoConfigure(t *testing.T) {
|
|
url := os.Getenv("RABBITMQ_URL")
|
|
provisionerURL := os.Getenv("RABBITMQ_PROVISIONER_URL")
|
|
if url == "" || provisionerURL == "" {
|
|
t.Skip("isolated RabbitMQ Mock URLs not configured")
|
|
}
|
|
ids := []string{testDispatcherID, "550e8400-e29b-41d4-a716-446655440000"}
|
|
conn, err := amqp.Dial(provisionerURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.Close()
|
|
admin, err := conn.Channel()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer admin.Close()
|
|
// Only the two business exchanges are provisioned. Opening a Dispatcher
|
|
// must not depend on an unused dead-letter exchange.
|
|
for _, exchange := range []string{CommandsExchange, ResultsExchange} {
|
|
if err := admin.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
shared := "agent-call.saas.events.v1"
|
|
if _, err := admin.QueueDeclare(shared, true, false, false, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _, _ = admin.QueueDelete(shared, false, false, false) }()
|
|
queues := []string{}
|
|
for _, id := range ids {
|
|
control, _ := tenant.ControlRoute(id)
|
|
task, _ := tenant.TaskRoute(id, "task-asr")
|
|
result, _ := tenant.ResultRoute(id)
|
|
for _, route := range []tenant.Route{control, task} {
|
|
if _, err := admin.QueueDeclare(route.Queue, true, false, false, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
queues = append(queues, route.Queue)
|
|
if err := admin.QueueBind(route.Queue, route.BindingKey, route.Exchange, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := admin.QueueBind(shared, result.BindingKey, result.Exchange, false, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func(key string) { _ = admin.QueueUnbind(shared, key, ResultsExchange, nil) }(result.BindingKey)
|
|
}
|
|
defer func() {
|
|
for _, name := range queues {
|
|
_, _ = admin.QueueDelete(name, false, false, false)
|
|
}
|
|
}()
|
|
for _, name := range append(append([]string(nil), queues...), shared) {
|
|
if _, err := admin.QueuePurge(name, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// The Dispatcher account must not configure the topology, including the
|
|
// SaaS result queue. A failed declare closes its channel, not the connection.
|
|
probeConn, err := amqp.Dial(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
probe, err := probeConn.Channel()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := probe.QueueDeclare("should-never-be-created", true, false, false, false, nil); err == nil {
|
|
_, _ = admin.QueueDelete("should-never-be-created", false, false, false)
|
|
t.Fatal("Dispatcher unexpectedly has configure permission")
|
|
}
|
|
_ = probe.Close()
|
|
_ = probeConn.Close()
|
|
|
|
brokers := make([]*Broker, 0, 2)
|
|
for _, id := range ids {
|
|
broker, err := Open(url, id, 1)
|
|
if err != nil {
|
|
t.Fatalf("Dispatcher without configure could not open %s: %v", id, err)
|
|
}
|
|
brokers = append(brokers, broker)
|
|
defer broker.Close()
|
|
}
|
|
control, _ := tenant.ControlRoute(ids[0])
|
|
task, _ := tenant.TaskRoute(ids[0], "task-asr")
|
|
controlBody := currentMessage(t, "mq-control")
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, control.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: controlBody}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
processed, err := brokers[0].DrainControlPredeclared(context.Background(), control.Queue, func(_ context.Context, key string, body []byte) error {
|
|
if key != control.BindingKey || string(body) != string(controlBody) {
|
|
return fmt.Errorf("wrong control route or body")
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil || processed != 1 {
|
|
t.Fatalf("control backlog: processed=%d err=%v", processed, err)
|
|
}
|
|
body := currentMessage(t, "mq-execute")
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
got := make(chan []byte, 1)
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- brokers[0].ConsumePredeclared(ctx, task.Queue, func(_ context.Context, key string, message []byte) error {
|
|
if key != task.BindingKey {
|
|
return fmt.Errorf("wrong task routing key %q", key)
|
|
}
|
|
got <- append([]byte(nil), message...)
|
|
return nil
|
|
})
|
|
}()
|
|
select {
|
|
case message := <-got:
|
|
if string(message) != string(body) {
|
|
t.Errorf("received wrong task body")
|
|
}
|
|
case err := <-done:
|
|
t.Fatalf("task consumer ended: %v", err)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("task message not received")
|
|
}
|
|
cancel()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("consumer did not stop")
|
|
}
|
|
otherTask, _ := tenant.TaskRoute(ids[1], "task-asr")
|
|
if state, err := admin.QueueInspect(otherTask.Queue); err != nil || state.Messages != 0 {
|
|
t.Fatalf("D2 stole D1 command: %+v %v", state, err)
|
|
}
|
|
|
|
// Purge is permitted with read access and affects only the chosen task;
|
|
// it must not remove another Dispatcher's backlog or the control queue.
|
|
for i := 0; i < 3; i++ {
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, otherTask.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count, err := brokers[0].PurgeTaskQueue(context.Background(), "task-asr")
|
|
if err != nil || count != 3 {
|
|
t.Fatalf("task-only purge: removed=%d err=%v", count, err)
|
|
}
|
|
if state, err := admin.QueueInspect(task.Queue); err != nil || state.Messages != 0 {
|
|
t.Fatalf("D1 queue not empty: %+v %v", state, err)
|
|
}
|
|
if state, err := admin.QueueInspect(otherTask.Queue); err != nil || state.Messages != 1 {
|
|
t.Fatalf("D2 queue was purged: %+v %v", state, err)
|
|
}
|
|
if _, err := brokers[0].PurgeTaskQueue(context.Background(), "../foreign"); err == nil {
|
|
t.Fatal("invalid task ID accepted for purge")
|
|
}
|
|
|
|
// Stop closes the consumer channel before purge; an in-flight delivery
|
|
// must become Ready rather than surviving the purge as Unacked.
|
|
inFlight := make(chan struct{}, 1)
|
|
consumer, err := brokers[0].StartPredeclaredConsumer(context.Background(), task.Queue, func(ctx context.Context, _ string, _ []byte) error {
|
|
select {
|
|
case inFlight <- struct{}{}:
|
|
default:
|
|
}
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := admin.PublishWithContext(context.Background(), CommandsExchange, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: currentMessage(t, "mq-execute")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-inFlight:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("task consumer did not receive in-flight delivery")
|
|
}
|
|
stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := consumer.Stop(stopCtx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count, err = brokers[0].PurgeTaskQueue(context.Background(), "task-asr")
|
|
if err != nil || count != 1 {
|
|
t.Fatalf("in-flight delivery survived stop and purge: removed=%d err=%v", count, err)
|
|
}
|
|
|
|
resultBody := currentMessage(t, "mq-result-no-recording")
|
|
for i, broker := range brokers {
|
|
message := resultBody
|
|
if i == 1 {
|
|
message = []byte(strings.Replace(string(resultBody), ids[0], ids[1], 1))
|
|
}
|
|
route, _ := tenant.ResultRoute(ids[i])
|
|
if err := broker.Publish(context.Background(), route.Exchange, route.BindingKey, message); err != nil {
|
|
t.Fatalf("result publish for D%d: %v", i+1, err)
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for range ids {
|
|
delivery, ok, err := admin.Get(shared, false)
|
|
if err != nil || !ok {
|
|
t.Fatalf("shared SaaS queue missing confirmed result: %v", err)
|
|
}
|
|
seen[delivery.RoutingKey] = true
|
|
if err := delivery.Ack(false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, id := range ids {
|
|
if !seen["d."+id+".out"] {
|
|
t.Fatalf("missing D-specific output binding for %s", id)
|
|
}
|
|
}
|
|
// A dead connection must not claim delivery. An explicit new session can
|
|
// publish the unchanged pending identity to the SaaS-owned shared queue.
|
|
route, _ := tenant.ResultRoute(ids[0])
|
|
pending := []byte(strings.Replace(string(resultBody), `"event_id":"call-example"`, `"event_id":"numberID-reopened-0000123"`, 1))
|
|
if err := brokers[0].Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := brokers[0].Publish(context.Background(), route.Exchange, route.BindingKey, pending); err == nil || !strings.Contains(err.Error(), "connection is closed") {
|
|
t.Fatalf("closed connection reported a successful publish: %v", err)
|
|
}
|
|
reopened, err := Open(url, ids[0], 1)
|
|
if err != nil {
|
|
t.Fatalf("explicit reconnect failed: %v", err)
|
|
}
|
|
defer reopened.Close()
|
|
brokers[0] = reopened
|
|
if err := reopened.Publish(context.Background(), route.Exchange, route.BindingKey, pending); err != nil {
|
|
t.Fatalf("original pending identity was not delivered after reconnect: %v", err)
|
|
}
|
|
delivery, ok, err := admin.Get(shared, false)
|
|
if err != nil || !ok || delivery.MessageId != "numberID-reopened-0000123" || string(delivery.Body) != string(pending) || delivery.DeliveryMode != amqp.Persistent {
|
|
t.Fatalf("reconnected result was not received by the SaaS-owned queue with its original identity: ok=%t err=%v", ok, err)
|
|
}
|
|
if err := delivery.Ack(false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err := store.Open(filepath.Join(t.TempDir(), "rejections.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
if err := s.ApplyDiscoverySnapshot(ids[0], []configread.DiscoveredTask{{TaskID: "task-asr", TenantID: 1001, TaskRevision: 1, Status: "running"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
brokers[0].SetInvalidMessageHandler(func(_ context.Context, _ string, _ string, raw []byte, cause error) error {
|
|
err := s.RejectData(ids[0], raw, cause)
|
|
if errors.Is(err, store.ErrUnidentifiableData) {
|
|
return nil
|
|
}
|
|
return err
|
|
})
|
|
badControl := []byte(`{"event_id":"original-missing-issued-at","event_type":"task.control","dispatcher_id":"` + ids[0] + `","tenant_id":1001,"payload":{"task_id":"task-asr","action":"resume","reason":"PRIVATE ORIGINAL"}}`)
|
|
controlSource, _ := tenant.ControlRoute(ids[0])
|
|
if err := admin.PublishWithContext(context.Background(), controlSource.Exchange, controlSource.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: badControl}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count, err = brokers[0].DrainControlPredeclared(context.Background(), controlSource.Queue, func(context.Context, string, []byte) error { t.Fatal("malformed original executed"); return nil })
|
|
if err != nil || count != 1 {
|
|
t.Fatalf("malformed original not durably handled: %d %v", count, err)
|
|
}
|
|
feedback, err := s.ListPendingOutbox(ids[0])
|
|
if err != nil || len(feedback) != 1 {
|
|
t.Fatalf("correlated feedback missing: %+v %v", feedback, err)
|
|
}
|
|
resultDestination, _ := tenant.ResultRoute(ids[0])
|
|
if err := brokers[0].Publish(context.Background(), resultDestination.Exchange, resultDestination.BindingKey, feedback[0].Body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rejected, ok, err := admin.Get(shared, false)
|
|
if err != nil || !ok {
|
|
t.Fatalf("SaaS-owned queue did not receive rejection: %t %v", ok, err)
|
|
}
|
|
var receipt struct {
|
|
EventID string `json:"event_id"`
|
|
Payload struct {
|
|
TaskID string `json:"task_id"`
|
|
Action string `json:"action"`
|
|
Status string `json:"status"`
|
|
ReasonCode string `json:"reason_code"`
|
|
TaskState string `json:"task_state"`
|
|
Reason string `json:"reason_message"`
|
|
} `json:"payload"`
|
|
}
|
|
if err := json.Unmarshal(rejected.Body, &receipt); err != nil || receipt.EventID != "original-missing-issued-at" || receipt.Payload.TaskID != "task-asr" || receipt.Payload.Action != "resume" || receipt.Payload.TaskState != "running" || receipt.Payload.ReasonCode != "invalid_data" || receipt.Payload.Status != "rejected" || !strings.Contains(receipt.Payload.Reason, "issued_at") || strings.Contains(string(rejected.Body), "PRIVATE ORIGINAL") {
|
|
t.Fatalf("untruthful/unsafe rejected feedback: %s %v", rejected.Body, err)
|
|
}
|
|
if err := rejected.Ack(false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Inspect the actual queued success body as well as the correlated
|
|
// structural rejection above. This is isolated MQ, not external SaaS.
|
|
if err := s.PrepareControl(ids[0], 1001, "task-asr", "pause"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CompleteControl(ids[0], 1001, "task-asr", "pause", "control-applied-fields"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
feedback, err = s.ListPendingOutbox(ids[0])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var appliedBody []byte
|
|
for _, row := range feedback {
|
|
if row.EventID == "control-applied-fields" {
|
|
appliedBody = row.Body
|
|
}
|
|
}
|
|
if len(appliedBody) == 0 {
|
|
t.Fatal("applied receipt missing")
|
|
}
|
|
if err := brokers[0].Publish(context.Background(), resultDestination.Exchange, resultDestination.BindingKey, appliedBody); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
applied, ok, err := admin.Get(shared, false)
|
|
if err != nil || !ok || !bytes.Equal(applied.Body, appliedBody) {
|
|
t.Fatalf("success receipt changed or missing on MQ: %t %v", ok, err)
|
|
}
|
|
var success struct {
|
|
Payload map[string]any `json:"payload"`
|
|
}
|
|
if err := json.Unmarshal(applied.Body, &success); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := map[string]any{"task_id": "task-asr", "action": "pause", "status": "applied", "reason_code": "applied", "task_state": "paused"}
|
|
if len(success.Payload) != len(want) {
|
|
t.Fatalf("success receipt must contain exactly five fields: %+v", success.Payload)
|
|
}
|
|
for k, v := range want {
|
|
if success.Payload[k] != v {
|
|
t.Errorf("MQ success %s: got %v, want %v", k, success.Payload[k], v)
|
|
}
|
|
}
|
|
if err := applied.Ack(false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.MarkOutboxConfirmed(ids[0], "task.control", "control-applied-fields"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, id := range ids {
|
|
route, _ := tenant.ResultRoute(id)
|
|
if err := admin.QueueUnbind(shared, route.BindingKey, route.Exchange, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
resultRoute, _ := tenant.ResultRoute(ids[0])
|
|
if err := brokers[0].Publish(context.Background(), resultRoute.Exchange, resultRoute.BindingKey, resultBody); err == nil {
|
|
t.Fatal("unroutable mandatory result falsely confirmed")
|
|
}
|
|
controlRoute, err := tenant.ControlRoute(ids[0])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := admin.PublishWithContext(context.Background(), controlRoute.Exchange, controlRoute.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: []byte("{")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
processed, err = brokers[0].DrainControlPredeclared(context.Background(), controlRoute.Queue, func(context.Context, string, []byte) error {
|
|
t.Fatal("invalid backlog control reached handler")
|
|
return nil
|
|
})
|
|
if err != nil || processed != 1 {
|
|
t.Fatalf("unidentifiable garbage was not handled without fabricated feedback: processed=%d err=%v", processed, err)
|
|
}
|
|
feedback, err = s.ListPendingOutbox(ids[0])
|
|
if err != nil || len(feedback) != 1 {
|
|
t.Fatalf("garbage invented another identity: %+v %v", feedback, err)
|
|
}
|
|
}
|