212 lines
8.1 KiB
Go
212 lines
8.1 KiB
Go
package store
|
|
|
|
import (
|
|
"database/sql"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func confirmCleanupFeedback(t *testing.T, s *Store, id, eventType string) {
|
|
t.Helper()
|
|
if err := s.MarkOutboxConfirmed(currentCall(id).DispatcherID, eventType, id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func requireCleanupCount(t *testing.T, s *Store, table string, expected int) {
|
|
t.Helper()
|
|
var count int
|
|
if err := s.db.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&count); err != nil || count != expected {
|
|
t.Fatalf("%s rows=%d expected=%d err=%v", table, count, expected, err)
|
|
}
|
|
}
|
|
|
|
func TestCleanupCallNeedsSuccessfulResultAndBothTypedConfirms(t *testing.T) {
|
|
for _, scenario := range []struct {
|
|
name string
|
|
result, ackConfirm, resultConfirm bool
|
|
deleted int
|
|
}{
|
|
{"no_result_rpc", false, true, false, 0},
|
|
{"no_dispatch_confirm", true, false, true, 0},
|
|
{"no_result_confirm", true, true, false, 0},
|
|
{"all_complete", true, true, true, 3},
|
|
} {
|
|
t.Run(scenario.name, func(t *testing.T) {
|
|
s := preparedCurrentCallStore(t)
|
|
cmd := currentResultCall(t, s, "numberID-cleanup", true)
|
|
if scenario.result {
|
|
if _, _, err := s.RecordCallResult(cmd.DispatcherID, cmd.EventID, currentResultPayload(t)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if scenario.ackConfirm {
|
|
confirmCleanupFeedback(t, s, cmd.EventID, "call.execute")
|
|
}
|
|
if scenario.resultConfirm {
|
|
confirmCleanupFeedback(t, s, cmd.EventID, "call.execute.result")
|
|
}
|
|
result, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID)
|
|
if err != nil || result.DeletedRows != scenario.deleted {
|
|
t.Fatalf("cleanup=%+v err=%v", result, err)
|
|
}
|
|
requireCleanupCount(t, s, "dispatcher_tasks", 1)
|
|
requireCleanupCount(t, s, "dispatcher_configs", 1)
|
|
if scenario.deleted == 0 {
|
|
requireCleanupCount(t, s, "dispatcher_inbox", 1)
|
|
if result.RetainedReason == "" {
|
|
t.Fatal("retained call has no reason")
|
|
}
|
|
return
|
|
}
|
|
requireCleanupCount(t, s, "dispatcher_inbox", 0)
|
|
requireCleanupCount(t, s, "dispatcher_outbox", 0)
|
|
if repeated, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err != nil || repeated.DeletedRows != 0 {
|
|
t.Fatalf("repeat cleanup=%+v err=%v", repeated, err)
|
|
}
|
|
if _, inserted, err := s.RecordExecute(cmd); err != nil || !inserted {
|
|
t.Fatalf("same event ID was not reusable: inserted=%t err=%v", inserted, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCleanupRejectedNeedsOnlyItsConfirmedRefusal(t *testing.T) {
|
|
s := preparedCurrentCallStore(t)
|
|
cmd := currentCall("rejected-cleanup")
|
|
if _, _, err := s.RecordExecute(cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.RejectExecute(cmd.DispatcherID, cmd.EventID, "not issued"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err != nil || result.DeletedRows != 0 {
|
|
t.Fatalf("unconfirmed refusal removed: %+v %v", result, err)
|
|
}
|
|
confirmCleanupFeedback(t, s, cmd.EventID, "call.execute")
|
|
if result, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err != nil || result.DeletedRows != 2 {
|
|
t.Fatalf("refusal cleanup=%+v err=%v", result, err)
|
|
}
|
|
requireCleanupCount(t, s, "dispatcher_inbox", 0)
|
|
requireCleanupCount(t, s, "dispatcher_outbox", 0)
|
|
}
|
|
|
|
func TestCleanupNeverRemovesUnfinishedOrUnknownCalls(t *testing.T) {
|
|
for _, state := range []string{"pending", "dispatching", "dispatched", "unknown"} {
|
|
t.Run(state, func(t *testing.T) {
|
|
s := preparedCurrentCallStore(t)
|
|
cmd := currentCall("retain-" + state)
|
|
if _, _, err := s.RecordExecute(cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if state != "pending" {
|
|
if err := s.ReserveExecute(cmd.DispatcherID, cmd.EventID, currentReservation(), time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if state == "dispatched" {
|
|
if err := s.MarkExecuteDispatched(cmd.DispatcherID, cmd.EventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if state == "unknown" {
|
|
if err := s.MarkExecuteUnknown(cmd.DispatcherID, cmd.EventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if result, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err != nil || result.DeletedRows != 0 || result.RetainedReason == "" {
|
|
t.Fatalf("unfinished cleanup=%+v err=%v", result, err)
|
|
}
|
|
requireCleanupCount(t, s, "dispatcher_inbox", 1)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCleanupCallRollbackPreservesRecordingAndOriginalFacts(t *testing.T) {
|
|
s := preparedCurrentCallStore(t)
|
|
cmd := currentResultCall(t, s, "cleanup-rollback", true)
|
|
binding := currentUploadBinding(cmd)
|
|
if _, _, err := s.BindRecordingUpload(binding); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := s.RecordUploadedCallResult(cmd.DispatcherID, cmd.EventID, currentUploadedResultPayload(t, binding), currentUploadProof(binding)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
confirmCleanupFeedback(t, s, cmd.EventID, "call.execute")
|
|
confirmCleanupFeedback(t, s, cmd.EventID, "call.execute.result")
|
|
if _, err := s.db.Exec(`CREATE TRIGGER cleanup_fault BEFORE DELETE ON dispatcher_inbox BEGIN SELECT RAISE(ABORT,'cleanup interrupted'); END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err == nil {
|
|
t.Fatal("cleanup swallowed SQLite deletion failure")
|
|
}
|
|
requireCleanupCount(t, s, "dispatcher_recordings", 1)
|
|
requireCleanupCount(t, s, "dispatcher_outbox", 2)
|
|
requireCleanupCount(t, s, "dispatcher_inbox", 1)
|
|
if _, err := s.db.Exec(`DROP TRIGGER cleanup_fault`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err != nil || result.DeletedRows != 4 {
|
|
t.Fatalf("retry cleanup=%+v err=%v", result, err)
|
|
}
|
|
requireCleanupCount(t, s, "dispatcher_recordings", 0)
|
|
}
|
|
|
|
func TestCleanupSuppressedRequiresCompletedConfirmedStop(t *testing.T) {
|
|
s := preparedCurrentCallStore(t)
|
|
cmd := currentCall("cancelled-cleanup")
|
|
if _, _, err := s.RecordExecute(cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.PrepareControl(cmd.DispatcherID, cmd.TenantID, cmd.TaskID, "stop"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err != nil || result.DeletedRows != 0 {
|
|
t.Fatalf("stopping task cleaned early: %+v %v", result, err)
|
|
}
|
|
if err := s.CompleteControl(cmd.DispatcherID, cmd.TenantID, cmd.TaskID, "stop", "stop-cleanup"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err != nil || result.DeletedRows != 0 {
|
|
t.Fatalf("unconfirmed stop cleaned early: %+v %v", result, err)
|
|
}
|
|
confirmCleanupFeedback(t, s, "stop-cleanup", "task.control")
|
|
if result, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err != nil || result.DeletedRows != 1 {
|
|
t.Fatalf("cancelled cleanup=%+v err=%v", result, err)
|
|
}
|
|
requireCleanupCount(t, s, "dispatcher_outbox", 1) // stop proof belongs to the task sweep
|
|
requireCleanupCount(t, s, "dispatcher_tasks", 1)
|
|
if admitted, err := s.CanAdmit(cmd.DispatcherID, cmd.TenantID, cmd.TaskID); err != nil || admitted {
|
|
t.Fatalf("stop barrier lost: %t %v", admitted, err)
|
|
}
|
|
if _, _, err := s.RecordExecute(cmd); !errors.Is(err, ErrStopped) {
|
|
t.Fatalf("cleaned stopped command was readmitted: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCleanupCallRefusesMismatchedFinalIdentity(t *testing.T) {
|
|
s := preparedCurrentCallStore(t)
|
|
cmd := currentResultCall(t, s, "cleanup-corrupt", true)
|
|
if _, _, err := s.RecordCallResult(cmd.DispatcherID, cmd.EventID, currentResultPayload(t)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
confirmCleanupFeedback(t, s, cmd.EventID, "call.execute")
|
|
confirmCleanupFeedback(t, s, cmd.EventID, "call.execute.result")
|
|
if _, err := s.db.Exec(`UPDATE dispatcher_outbox SET body=json_set(body,'$.payload.task_id','other-task') WHERE event_type='call.execute.result'`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.CleanupCall(cmd.DispatcherID, cmd.EventID); err == nil {
|
|
t.Fatal("conflicting result identity was deleted")
|
|
}
|
|
requireCleanupCount(t, s, "dispatcher_outbox", 2)
|
|
requireCleanupCount(t, s, "dispatcher_inbox", 1)
|
|
if _, _, err := s.RecordExecute(currentCall("neighbor")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var state string
|
|
if err := s.db.QueryRow(`SELECT status FROM dispatcher_inbox WHERE event_id='neighbor'`).Scan(&state); errors.Is(err, sql.ErrNoRows) || err != nil || state != "pending" {
|
|
t.Fatalf("neighbor changed: state=%s err=%v", state, err)
|
|
}
|
|
}
|