314 lines
8.6 KiB
Go
314 lines
8.6 KiB
Go
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/contracts"
|
|
agentv1 "git.ipao.vip/rogee/go-sip/gen/agent/v1"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
type fakePublisher struct {
|
|
exchanges []string
|
|
keys []string
|
|
bodies [][]byte
|
|
err error
|
|
}
|
|
|
|
func (p *fakePublisher) Publish(_ context.Context, exchange, key string, body []byte) error {
|
|
if p.err != nil {
|
|
return p.err
|
|
}
|
|
p.exchanges = append(p.exchanges, exchange)
|
|
p.keys = append(p.keys, key)
|
|
p.bodies = append(p.bodies, append([]byte(nil), body...))
|
|
return nil
|
|
}
|
|
|
|
func TestFlushOutboxPublishesAfterDurableIngest(t *testing.T) {
|
|
st, err := store.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
now := time.Date(2026, 9, 18, 0, 0, 0, 0, time.UTC)
|
|
pub := &fakePublisher{}
|
|
d, err := New(st, pub, func() time.Time { return now })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := contracts.Read("examples/call.execute.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := d.AcceptCommand(raw, "agent-call.tenant.tenant-demo-key.call.execute"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count, err := d.FlushOutbox(context.Background(), 10)
|
|
if err != nil || count != 1 || len(pub.bodies) != 1 {
|
|
t.Fatalf("flush count=%d err=%v published=%d", count, err, len(pub.bodies))
|
|
}
|
|
if pub.exchanges[0] != "agent-call.events.v1" {
|
|
t.Fatalf("exchange=%q", pub.exchanges[0])
|
|
}
|
|
}
|
|
|
|
func TestOutboxClaimRecoveryPublishesAfterRestart(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "dispatcher.db")
|
|
first, err := store.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
firstDispatcher, err := New(first, nil, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := contracts.Read("examples/call.execute.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := firstDispatcher.AcceptCommand(raw, "agent-call.tenant.tenant-demo-key.call.execute"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := first.ClaimOutbox(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
t.Fatalf("claimed=%d err=%v", len(claimed), err)
|
|
}
|
|
if err := first.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
second, err := store.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
if err := second.Close(); err != nil {
|
|
t.Error(err)
|
|
}
|
|
})
|
|
publisher := &fakePublisher{}
|
|
secondDispatcher, err := New(second, publisher, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
published, err := secondDispatcher.FlushOutbox(t.Context(), 1)
|
|
if err != nil || published != 1 || len(publisher.bodies) != 1 {
|
|
t.Fatalf("published=%d err=%v bodies=%d", published, err, len(publisher.bodies))
|
|
}
|
|
var status string
|
|
if err := second.DB().QueryRow(`SELECT status FROM outbox WHERE event_id = ?`, claimed[0].EventID).Scan(&status); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status != "published" {
|
|
t.Fatalf("status=%q, want published", status)
|
|
}
|
|
}
|
|
|
|
func TestOutboxProcessCrashRecovery(t *testing.T) {
|
|
const (
|
|
helperEnv = "SIP_GO_AGENT_OUTBOX_CRASH_HELPER"
|
|
dbEnv = "SIP_GO_AGENT_OUTBOX_CRASH_DB"
|
|
exitCode = 97
|
|
)
|
|
if os.Getenv(helperEnv) == "1" {
|
|
path := os.Getenv(dbEnv)
|
|
st, err := store.Open(path)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(2)
|
|
}
|
|
d, err := New(st, nil, time.Now)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(3)
|
|
}
|
|
raw, err := contracts.Read("examples/call.execute.json")
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(4)
|
|
}
|
|
if _, err := d.AcceptCommand(raw, "agent-call.tenant.tenant-demo-key.call.execute"); err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(5)
|
|
}
|
|
claimed, err := st.ClaimOutbox(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
fmt.Fprintf(os.Stderr, "claimed=%d err=%v\\n", len(claimed), err)
|
|
os.Exit(6)
|
|
}
|
|
// Simulate a process dying after the durable claim and before publish.
|
|
os.Exit(exitCode)
|
|
}
|
|
|
|
path := filepath.Join(t.TempDir(), "dispatcher.db")
|
|
cmd := exec.Command(os.Args[0], "-test.run=^TestOutboxProcessCrashRecovery$")
|
|
cmd.Env = append(os.Environ(), helperEnv+"=1", dbEnv+"="+path)
|
|
output, err := cmd.CombinedOutput()
|
|
var exitErr *exec.ExitError
|
|
if err == nil || !errors.As(err, &exitErr) || exitErr.ExitCode() != exitCode {
|
|
t.Fatalf("crash helper err=%v output=%s", err, output)
|
|
}
|
|
|
|
st, err := store.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = st.Close() })
|
|
publisher := &fakePublisher{}
|
|
d, err := New(st, publisher, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
published, err := d.FlushOutbox(t.Context(), 1)
|
|
if err != nil || published != 1 || len(publisher.bodies) != 1 {
|
|
t.Fatalf("published=%d err=%v bodies=%d", published, err, len(publisher.bodies))
|
|
}
|
|
}
|
|
|
|
func TestFlushOutboxMarksRetryOnPublishFailure(t *testing.T) {
|
|
st, err := store.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
pub := &fakePublisher{err: errors.New("broker unavailable")}
|
|
d, err := New(st, pub, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := contracts.Read("examples/call.execute.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := d.AcceptCommand(raw, "agent-call.tenant.tenant-demo-key.call.execute"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := d.FlushOutbox(context.Background(), 10); err == nil {
|
|
t.Fatal("expected publish failure")
|
|
}
|
|
rows, err := st.DB().Query(`SELECT status FROM outbox`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
if !rows.Next() {
|
|
t.Fatal("missing outbox row")
|
|
}
|
|
var status string
|
|
if err := rows.Scan(&status); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status != "retry" {
|
|
t.Fatalf("status=%q, want retry", status)
|
|
}
|
|
}
|
|
|
|
func TestExecuteReservedBindsQuotaAndAgentExecution(t *testing.T) {
|
|
st, err := store.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
for _, scope := range []string{"tenant:tenant-demo-key", "global", "cell:cell-1"} {
|
|
if err := st.SetQuota(scope, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
now := time.Date(2026, 9, 18, 1, 0, 0, 0, time.UTC)
|
|
d, err := New(st, nil, func() time.Time { return now })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := contracts.Read("examples/call.execute.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := d.AcceptCommand(raw, "agent-call.tenant.tenant-demo-key.call.execute"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
task, err := d.ReserveTask("tenant-demo-key", "reservation-integration", []string{"tenant:tenant-demo-key", "global", "cell:cell-1"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
coordinator := NewAgentCoordinator(func() time.Time { return now })
|
|
client := startMockAgent(t, &agentv1.AgentStatus{AgentId: "agent-1", CellId: "cell-1"})
|
|
if err := coordinator.Register("agent-1", client); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := coordinator.Activate(context.Background(), "agent-1", "cell-1", "boot-1", "epoch-1", 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err := d.ExecuteReserved(context.Background(), coordinator, "agent-1", task, raw, "reservation-integration", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Permit == nil || result.Receipt == nil || result.Unknown {
|
|
t.Fatalf("unexpected result: %+v", result)
|
|
}
|
|
var taskStatus string
|
|
if err := st.DB().QueryRow(`SELECT status FROM tasks WHERE execution_id = ?`, task.ExecutionID).Scan(&taskStatus); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if taskStatus != "running" {
|
|
t.Fatalf("task status=%q, want running", taskStatus)
|
|
}
|
|
}
|
|
|
|
func TestFairSchedulerRoundRobinAndRestore(t *testing.T) {
|
|
s := NewFairScheduler([]string{"a", "b", "c"})
|
|
for i, want := range []string{"a", "b", "c", "a"} {
|
|
got, ok := s.NextTenant()
|
|
if !ok || got != want {
|
|
t.Fatalf("turn %d = %q/%v, want %q", i, got, ok, want)
|
|
}
|
|
}
|
|
s.RestoreCursor(2)
|
|
got, _ := s.NextTenant()
|
|
if got != "c" {
|
|
t.Fatalf("restored cursor = %q, want c", got)
|
|
}
|
|
}
|
|
|
|
func TestFairSchedulerPersistsCursorAcrossRestart(t *testing.T) {
|
|
dbPath := filepath.Join(t.TempDir(), "dispatcher.db")
|
|
st, err := store.Open(dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first, err := NewFairSchedulerFromStore(st, "tenant-rotation", []string{"a", "b", "c"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i, want := range []string{"a", "b"} {
|
|
got, ok, err := first.NextTenantDurable(st, "tenant-rotation")
|
|
if err != nil || !ok || got != want {
|
|
t.Fatalf("turn %d = %q/%v err=%v, want %q", i, got, ok, err, want)
|
|
}
|
|
}
|
|
if err := st.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
st, err = store.Open(dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
second, err := NewFairSchedulerFromStore(st, "tenant-rotation", []string{"a", "b", "c"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, ok, err := second.NextTenantDurable(st, "tenant-rotation")
|
|
if err != nil || !ok || got != "c" {
|
|
t.Fatalf("restart turn = %q/%v err=%v, want c", got, ok, err)
|
|
}
|
|
}
|