255 lines
8.9 KiB
Go
255 lines
8.9 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func deliveredRecordingFixture(t *testing.T, r *RecordingRecovery) RecordingRecoveryEntry {
|
|
t.Helper()
|
|
e := failedRecordingFixture()
|
|
if _, err := r.SaveFailure([]byte("recording bytes"), e, &UploadHTTPError{StatusCode: 503}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e, err := r.Load(e.Bucket, e.ObjectKey)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e.State = "delivered"
|
|
e.Uploaded = &UploadResult{StatusCode: 200, SizeBytes: e.SizeBytes, SHA256: e.SHA256}
|
|
_, _, info, err := r.paths(e.Bucket, e.ObjectKey)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.persistState(info, e); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return e
|
|
}
|
|
|
|
func TestDeliveredRecordingCleanupAfterRestartWithoutAudio(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private"), Now: func() time.Time { return time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC) }}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
if err := os.Remove(audio); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
restarted := &RecordingRecovery{Root: r.Root}
|
|
if loaded, err := restarted.Load(e.Bucket, e.ObjectKey); err != nil || loaded.State != "delivered" {
|
|
t.Fatalf("valid deletion intermediate state unreadable: %+v %v", loaded, err)
|
|
}
|
|
calls := 0
|
|
after, err := restarted.Retry(context.Background(), e.Bucket, e.ObjectKey,
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
calls++
|
|
return RecoveryTarget{}, errors.New("must not request a grant")
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error {
|
|
calls++
|
|
return errors.New("must not report again")
|
|
})
|
|
if err != nil || after.State != "delivered" || calls != 0 {
|
|
t.Fatalf("delivered cleanup replayed work: %+v calls=%d err=%v", after, calls, err)
|
|
}
|
|
for _, path := range []string{dir, audio, info} {
|
|
if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("completed recording remains at %q: %v", path, err)
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Dir(dir)); err != nil {
|
|
t.Fatal("cleanup deleted a shared parent")
|
|
}
|
|
}
|
|
|
|
func TestDeliveredRecordingRetainedWhileResultReferenceUnfinished(t *testing.T) {
|
|
for _, state := range []string{"prepared", "ready", "end_pending"} {
|
|
t.Run(state, func(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
j := ResultJournal{Root: r.Root}
|
|
entry := ResultEntry{SourceEventID: e.SourceEventID, DispatcherID: "dispatcher-1", TenantID: 1001, Payload: []byte(`{}`)}
|
|
if err := j.save(entry, state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
for _, path := range []string{audio, info} {
|
|
if _, err := os.Stat(path); err != nil {
|
|
t.Fatalf("pending reference lost recovery file: %v", err)
|
|
}
|
|
}
|
|
if state == "prepared" {
|
|
if err := j.DiscardPrepared(e.SourceEventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
} else {
|
|
if state == "end_pending" {
|
|
if err := j.MarkEnded(e.SourceEventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := j.Ack(e.SourceEventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(info); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("reference-free delivered recording remains: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDeliveredCleanupFailureResumesWithoutPUTOrReport(t *testing.T) {
|
|
for _, failedStep := range []string{"info", "directory"} {
|
|
t.Run(failedStep, func(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
r.removeFile = func(path string) error {
|
|
if (failedStep == "info" && path == info) || (failedStep == "directory" && path == dir) {
|
|
return errors.New("injected cleanup interruption")
|
|
}
|
|
return os.Remove(path)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err == nil {
|
|
t.Fatal("deletion interruption was swallowed")
|
|
}
|
|
if _, err := os.Stat(audio); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("original audio deletion did not finish: %v", err)
|
|
}
|
|
if loaded, err := r.Load(e.Bucket, e.ObjectKey); err != nil || loaded.State != "delivered" {
|
|
t.Fatalf("interrupted cleanup lost its completion proof: %+v %v", loaded, err)
|
|
}
|
|
restarted := &RecordingRecovery{Root: r.Root}
|
|
if _, err := restarted.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("restart did not finish exact-leaf cleanup: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDeliveredMissingAudioStillValidatesOriginalReceipt(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
_, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
if err := os.Remove(audio); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e.SHA256 = "abc"
|
|
e.Uploaded.SHA256 = "abc"
|
|
if err := r.persistState(info, e); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Load(e.Bucket, e.ObjectKey); !errors.Is(err, ErrRecordingRecoveryIncomplete) {
|
|
t.Fatalf("malformed original receipt accepted after audio deletion: %v", err)
|
|
}
|
|
if _, err := os.Stat(info); err != nil {
|
|
t.Fatal("malformed evidence was removed")
|
|
}
|
|
}
|
|
|
|
func TestDeliveredCleanupPreservesUnexpectedFilesAndNeighborRecords(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
unknown := filepath.Join(dir, "operator-notes")
|
|
if err := os.WriteFile(unknown, []byte("retained"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err == nil {
|
|
t.Fatal("unknown leaf file was silently ignored")
|
|
}
|
|
for _, path := range []string{unknown, audio, info} {
|
|
if _, err := os.Stat(path); err != nil {
|
|
t.Fatalf("unknown leaf cleanup deleted data: %v", err)
|
|
}
|
|
}
|
|
if err := os.Remove(unknown); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestDeliveredRecoveryScanFinishesPartialDeletionDespiteIndependentBadDirectory(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
_, audio, info, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
if err := os.Remove(audio); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
blockedDir := filepath.Join(r.Root, "blocked-bucket")
|
|
if err := os.Mkdir(blockedDir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Chmod(blockedDir, 0755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
calls := 0
|
|
processed, err := r.RecoverDue(context.Background(),
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
calls++
|
|
return RecoveryTarget{}, errors.New("must not request a grant")
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error {
|
|
calls++
|
|
return errors.New("must not report again")
|
|
})
|
|
if err == nil || processed != 0 || calls != 0 {
|
|
t.Fatalf("independent bad directory handling: processed=%d calls=%d err=%v", processed, calls, err)
|
|
}
|
|
if _, err := os.Stat(info); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("delivered intermediate state was not cleaned: %v", err)
|
|
}
|
|
if _, err := os.Stat(blockedDir); err != nil {
|
|
t.Fatal("unknown directory was removed")
|
|
}
|
|
}
|
|
|
|
func TestRecordingCleanupLeavesNoPermanentInMemoryLocks(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
e := deliveredRecordingFixture(t, r)
|
|
dir, _, _, _ := r.paths(e.Bucket, e.ObjectKey)
|
|
resultPath, err := (ResultJournal{Root: r.Root}).path(e.SourceEventID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Retry(context.Background(), e.Bucket, e.ObjectKey, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
privateStateLocks.Lock()
|
|
recordingHeld, resultHeld := privateStateLocks.paths[dir] != nil, privateStateLocks.paths[resultPath] != nil
|
|
privateStateLocks.Unlock()
|
|
if recordingHeld || resultHeld {
|
|
t.Fatalf("completed entry kept permanent path locks: recording=%t result=%t", recordingHeld, resultHeld)
|
|
}
|
|
}
|
|
|
|
func TestMissingAudioDoesNotRelaxUnfinishedRecordingValidation(t *testing.T) {
|
|
r := &RecordingRecovery{Root: filepath.Join(t.TempDir(), "private")}
|
|
entry := failedRecordingFixture()
|
|
if _, err := r.SaveFailure([]byte("recording bytes"), entry, &UploadHTTPError{StatusCode: 503}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, audio, _, _ := r.paths(entry.Bucket, entry.ObjectKey)
|
|
if err := os.Remove(audio); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := r.Load(entry.Bucket, entry.ObjectKey); !errors.Is(err, ErrRecordingRecoveryIncomplete) {
|
|
t.Fatalf("unfinished missing audio was accepted: %v", err)
|
|
}
|
|
}
|