380 lines
13 KiB
Go
380 lines
13 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func deliveredRecordingFixture(t *testing.T, r *RecordingRecovery) RecordingRecoveryEntry {
|
|
t.Helper()
|
|
e := failedRecordingFixture()
|
|
if _, err := r.SaveFailure([]byte("recording bytes"), e, &UploadHTTPError{StatusCode: 503}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e, err := r.Load(e.Bucket, e.ObjectKey)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e.State = "delivered"
|
|
e.Uploaded = &UploadResult{StatusCode: 200, SizeBytes: e.SizeBytes, SHA256: e.SHA256}
|
|
_, _, info, err := r.paths(e.Bucket, e.ObjectKey)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.persistState(info, e); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return e
|
|
}
|
|
|
|
func TestDeliveredCleanupHardExitHelper(t *testing.T) {
|
|
root := os.Getenv("RECORDING_CLEANUP_EXIT_ROOT")
|
|
if root == "" {
|
|
return
|
|
}
|
|
r := &RecordingRecovery{Root: root}
|
|
e := failedRecordingFixture()
|
|
dir, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
r.removeFile = func(path string) error {
|
|
if err := os.Remove(path); err != nil {
|
|
return err
|
|
}
|
|
step := os.Getenv("RECORDING_CLEANUP_EXIT_STEP")
|
|
if (step == "audio" && path == audio) || (step == "info" && path == info) || (step == "leaf" && path == dir) || (step == "proof" && filepath.Dir(path) == filepath.Join(root, recordingCleanupDirectory)) {
|
|
if err := syncDirectory(filepath.Dir(path)); err != nil {
|
|
return err
|
|
}
|
|
os.Exit(23) // no defers, proof restoration, or orderly shutdown
|
|
}
|
|
return nil
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Fatal("hard-exit injection did not run")
|
|
}
|
|
|
|
func TestDeliveredCleanupResumesAfterHardExitAtEveryFinalDeletion(t *testing.T) {
|
|
for _, step := range []string{"audio", "info", "leaf", "proof"} {
|
|
for _, midnight := range []bool{false, true} {
|
|
t.Run(step+"/midnight="+fmt.Sprint(midnight), func(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, _, _, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
cmd := exec.Command(os.Args[0], "-test.run=^TestDeliveredCleanupHardExitHelper$")
|
|
cmd.Env = append(os.Environ(), "RECORDING_CLEANUP_EXIT_ROOT="+r.Root, "RECORDING_CLEANUP_EXIT_STEP="+step)
|
|
if err := cmd.Run(); err == nil || cmd.ProcessState.ExitCode() != 23 {
|
|
t.Fatalf("helper did not exit at boundary: %v", err)
|
|
}
|
|
unknown := filepath.Join(r.Root, "unowned-empty-leaf")
|
|
if err := os.Mkdir(unknown, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
restarted := &RecordingRecovery{Root: r.Root}
|
|
calls := 0
|
|
var err error
|
|
if midnight {
|
|
_, err = restarted.CleanupDelivered(context.Background())
|
|
} else {
|
|
_, err = restarted.RecoverDue(context.Background(), func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
calls++
|
|
return RecoveryTarget{}, errors.New("no grant")
|
|
}, func(context.Context, RecordingRecoveryEntry) error { calls++; return errors.New("no report") })
|
|
}
|
|
if err != nil || calls != 0 {
|
|
t.Fatalf("restart replayed work: calls=%d err=%v", calls, err)
|
|
}
|
|
if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("completed leaf remains after hard exit: %v", err)
|
|
}
|
|
if _, err := os.Stat(unknown); err != nil {
|
|
t.Fatal("unknown empty directory was removed")
|
|
}
|
|
files := 0
|
|
if err := filepath.WalkDir(r.Root, func(_ string, item fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !item.IsDir() {
|
|
files++
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if files != 0 {
|
|
t.Fatalf("completion proof or files accumulated: %d", files)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDeliveredRecordingCleanupAfterRestartWithoutAudio(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private"), Now: func() time.Time { return time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC) }}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
if err := os.Remove(audio); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
restarted := &RecordingRecovery{Root: r.Root}
|
|
if loaded, err := restarted.Load(e.Bucket, e.ObjectKey); err != nil || loaded.State != "delivered" {
|
|
t.Fatalf("valid deletion intermediate state unreadable: %+v %v", loaded, err)
|
|
}
|
|
calls := 0
|
|
after, err := restarted.Retry(context.Background(), e.Bucket, e.ObjectKey,
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
calls++
|
|
return RecoveryTarget{}, errors.New("must not request a grant")
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error {
|
|
calls++
|
|
return errors.New("must not report again")
|
|
})
|
|
if err != nil || after.State != "delivered" || calls != 0 {
|
|
t.Fatalf("delivered cleanup replayed work: %+v calls=%d err=%v", after, calls, err)
|
|
}
|
|
for _, path := range []string{dir, audio, info} {
|
|
if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("completed recording remains at %q: %v", path, err)
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Dir(dir)); err != nil {
|
|
t.Fatal("cleanup deleted a shared parent")
|
|
}
|
|
}
|
|
|
|
func TestDeliveredRecordingRetainedWhileResultReferenceUnfinished(t *testing.T) {
|
|
for _, state := range []string{"prepared", "ready", "end_pending"} {
|
|
t.Run(state, func(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
j := ResultJournal{Root: r.Root}
|
|
entry := ResultEntry{SourceEventID: e.SourceEventID, DispatcherID: "dispatcher-1", TenantID: 1001, Payload: []byte(`{}`)}
|
|
if err := j.save(entry, state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
for _, path := range []string{audio, info} {
|
|
if _, err := os.Stat(path); err != nil {
|
|
t.Fatalf("pending reference lost recovery file: %v", err)
|
|
}
|
|
}
|
|
if state == "prepared" {
|
|
if err := j.DiscardPrepared(e.SourceEventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
} else {
|
|
if state == "end_pending" {
|
|
if err := j.MarkEnded(e.SourceEventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := j.Ack(e.SourceEventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(info); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("reference-free delivered recording remains: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDeliveredCleanupFailureResumesWithoutPUTOrReport(t *testing.T) {
|
|
for _, failedStep := range []string{"info", "directory"} {
|
|
t.Run(failedStep, func(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
r.removeFile = func(path string) error {
|
|
if (failedStep == "info" && path == info) || (failedStep == "directory" && path == dir) {
|
|
return errors.New("injected cleanup interruption")
|
|
}
|
|
return os.Remove(path)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err == nil {
|
|
t.Fatal("deletion interruption was swallowed")
|
|
}
|
|
if _, err := os.Stat(audio); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("original audio deletion did not finish: %v", err)
|
|
}
|
|
if loaded, err := r.Load(e.Bucket, e.ObjectKey); err != nil || loaded.State != "delivered" {
|
|
t.Fatalf("interrupted cleanup lost its completion proof: %+v %v", loaded, err)
|
|
}
|
|
restarted := &RecordingRecovery{Root: r.Root}
|
|
if _, err := restarted.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("restart did not finish exact-leaf cleanup: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDeliveredMissingAudioStillValidatesOriginalReceipt(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
_, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
if err := os.Remove(audio); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e.SHA256 = "abc"
|
|
e.Uploaded.SHA256 = "abc"
|
|
if err := r.persistState(info, e); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Load(e.Bucket, e.ObjectKey); !errors.Is(err, ErrRecordingRecoveryIncomplete) {
|
|
t.Fatalf("malformed original receipt accepted after audio deletion: %v", err)
|
|
}
|
|
if _, err := os.Stat(info); err != nil {
|
|
t.Fatal("malformed evidence was removed")
|
|
}
|
|
}
|
|
|
|
func TestDeliveredCleanupPreservesUnexpectedFilesAndNeighborRecords(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
unknown := filepath.Join(dir, "operator-notes")
|
|
if err := os.WriteFile(unknown, []byte("retained"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err == nil {
|
|
t.Fatal("unknown leaf file was silently ignored")
|
|
}
|
|
for _, path := range []string{unknown, audio, info} {
|
|
if _, err := os.Stat(path); err != nil {
|
|
t.Fatalf("unknown leaf cleanup deleted data: %v", err)
|
|
}
|
|
}
|
|
if err := os.Remove(unknown); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestDeliveredRecoveryScanFinishesPartialDeletionDespiteIndependentBadDirectory(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
_, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
if err := os.Remove(audio); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
blockedDir := filepath.Join(r.Root, "blocked-bucket")
|
|
if err := os.Mkdir(blockedDir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Chmod(blockedDir, 0755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
calls := 0
|
|
processed, err := r.RecoverDue(context.Background(),
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
calls++
|
|
return RecoveryTarget{}, errors.New("must not request a grant")
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error {
|
|
calls++
|
|
return errors.New("must not report again")
|
|
})
|
|
if err == nil || processed != 0 || calls != 0 {
|
|
t.Fatalf("independent bad directory handling: processed=%d calls=%d err=%v", processed, calls, err)
|
|
}
|
|
if _, err := os.Stat(info); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("delivered intermediate state was not cleaned: %v", err)
|
|
}
|
|
if _, err := os.Stat(blockedDir); err != nil {
|
|
t.Fatal("unknown directory was removed")
|
|
}
|
|
}
|
|
|
|
func TestRecordingCleanupLeavesNoPermanentInMemoryLocks(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, _, _, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
resultPath, err := (ResultJournal{Root: r.Root}).path(e.SourceEventID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
privateStateLocks.Lock()
|
|
recordingHeld, resultHeld := privateStateLocks.paths[dir] != nil, privateStateLocks.paths[resultPath] != nil
|
|
privateStateLocks.Unlock()
|
|
if recordingHeld || resultHeld {
|
|
t.Fatalf("completed entry kept permanent path locks: recording=%t result=%t", recordingHeld, resultHeld)
|
|
}
|
|
}
|
|
|
|
func TestMidnightDeliveredOnlySweepPreservesUnfinishedFiles(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
completed := deliveredRecordingFixture(t, r)
|
|
pending := failedRecordingFixture()
|
|
pending.SourceEventID = "execute-pending"
|
|
pending.ObjectKey = "tenant/call/pending.wav"
|
|
if _, err := r.SaveFailure([]byte("pending audio"), pending, &UploadHTTPError{StatusCode: 503}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count, err := r.CleanupDelivered(context.Background())
|
|
if err != nil || count != 1 {
|
|
t.Fatalf("midnight result=%d err=%v", count, err)
|
|
}
|
|
_, _, completedInfo, _ := r.paths(completed.Bucket, completed.ObjectKey)
|
|
if _, err := os.Stat(completedInfo); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatal("delivered info remains")
|
|
}
|
|
if entry, err := r.Load(pending.Bucket, pending.ObjectKey); err != nil || entry.State != "retry_pending" {
|
|
t.Fatalf("unfinished entry changed: %+v %v", entry, err)
|
|
}
|
|
}
|
|
|
|
func TestRepeatedDeliveredRecordingCleanupDoesNotAccumulateFiles(t *testing.T) {
|
|
root := filepath.Join(t.TempDir(), "private")
|
|
for round := 0; round < 3; round++ {
|
|
r := &RecordingRecovery{Root: root}
|
|
for i := 0; i < 20; i++ {
|
|
e := deliveredRecordingFixture(t, r)
|
|
if count, err := r.CleanupDelivered(context.Background()); err != nil || count != 1 {
|
|
t.Fatalf("round=%d entry=%d count=%d err=%v", round, i, count, err)
|
|
}
|
|
dir, _, _, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("completed files accumulated: %v", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMissingAudioDoesNotRelaxUnfinishedRecordingValidation(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
entry := failedRecordingFixture()
|
|
if _, err := r.SaveFailure([]byte("recording bytes"), entry, &UploadHTTPError{StatusCode: 503}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, audio, _, _ := r.paths(entry.Bucket, entry.ObjectKey)
|
|
if err := os.Remove(audio); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Load(entry.Bucket, entry.ObjectKey); !errors.Is(err, ErrRecordingRecoveryIncomplete) {
|
|
t.Fatalf("unfinished missing audio was accepted: %v", err)
|
|
}
|
|
}
|