package agent import ( "context" "crypto/sha256" "encoding/hex" "errors" "io" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" ) func TestRecordingRecoveryDiscoversAndResumesAfterProcessRestart(t *testing.T) { root := privateRecoveryRoot(t) now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) original := RecordingRecovery{Root: root, Now: func() time.Time { return now }} entry, err := original.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } now = now.Add(time.Minute) var puts, reports atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) puts.Add(1) })) defer server.Close() restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}} grant := func(_ context.Context, e RecordingRecoveryEntry) (RecoveryTarget, error) { return recoveryTarget(now, e, server.URL+"/original"), nil } report := func(_ context.Context, e RecordingRecoveryEntry) error { if e.SourceEventID != entry.SourceEventID || e.Uploaded == nil { t.Errorf("restarted reporter lost durable original upload identity: %+v", e) } reports.Add(1) return nil } processed, err := restarted.RecoverDue(context.Background(), grant, report) if err != nil || processed != 1 || puts.Load() != 1 || reports.Load() != 1 { t.Fatalf("startup recovery did not restore the original result: processed=%d puts=%d reports=%d err=%v", processed, puts.Load(), reports.Load(), err) } processed, err = restarted.RecoverDue(context.Background(), grant, report) if err != nil || processed != 0 || puts.Load() != 1 || reports.Load() != 1 { t.Fatalf("delivered result was re-uploaded after second scan: processed=%d puts=%d reports=%d err=%v", processed, puts.Load(), reports.Load(), err) } } func TestRecordingRecoveryScannerLeavesResultJournalToItsOwnRecovery(t *testing.T) { root := privateRecoveryRoot(t) journal := ResultJournal{Root: root} if err := journal.SaveReady(ResultEntry{SourceEventID: "original", DispatcherID: "11111111-1111-4111-8111-111111111111", TenantID: 1, Payload: []byte(`{"recording":{}}`)}); err != nil { t.Fatal(err) } processed, err := (&RecordingRecovery{Root: root}).RecoverDue(context.Background(), func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { t.Fatal("no failed PUT exists") return RecoveryTarget{}, nil }, func(context.Context, RecordingRecoveryEntry) error { t.Fatal("no failed PUT exists"); return nil }) if processed != 0 || err != nil { t.Fatalf("result-only journal is not a failed PUT: processed=%d err=%v", processed, err) } } func TestRecordingRecoveryScannerReportsOrphanInsteadOfInventingResult(t *testing.T) { root := privateRecoveryRoot(t) broken := RecordingRecovery{Root: root, writeState: func(string, any) error { return errors.New("metadata disk full") }} entry := failedRecordingFixture() if _, err := broken.SaveFailure([]byte("RIFF original audio"), entry, &UploadHTTPError{StatusCode: 503}); err == nil { t.Fatal("metadata disk fault was hidden") } restarted := RecordingRecovery{Root: root} processed, err := restarted.RecoverDue(context.Background(), func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { t.Fatal("orphan must not PUT") return RecoveryTarget{}, nil }, func(context.Context, RecordingRecoveryEntry) error { t.Fatal("orphan must not report a result") return nil }) if processed != 0 || !errors.Is(err, ErrRecordingRecoveryIncomplete) { t.Fatalf("orphan was silently skipped: processed=%d err=%v", processed, err) } trace := sha256.Sum256([]byte(filepath.ToSlash(filepath.Join(entry.Bucket, entry.ObjectKey)))) if !strings.Contains(err.Error(), hex.EncodeToString(trace[:6])) { t.Fatalf("orphan error lacks a safe stable path reference: %v", err) } if _, err := os.Stat(filepath.Join(root, "mock-bucket", "tenant", "call", "rec.wav", "recording.wav")); err != nil { t.Fatalf("original audio was removed while blocked: %v", err) } } func TestRecordingRecoveryConcurrentRetryOnlyPutsOnce(t *testing.T) { root := privateRecoveryRoot(t) now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } now = now.Add(time.Minute) var puts, reports atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) puts.Add(1) time.Sleep(15 * time.Millisecond) })) defer server.Close() grant := func(_ context.Context, e RecordingRecoveryEntry) (RecoveryTarget, error) { return recoveryTarget(now, e, server.URL), nil } report := func(context.Context, RecordingRecoveryEntry) error { reports.Add(1); return nil } var wg sync.WaitGroup errs := make(chan error, 2) for i := 0; i < 2; i++ { wg.Add(1) go func() { defer wg.Done() _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, report) errs <- err }() } wg.Wait() close(errs) for err := range errs { if err != nil { t.Fatalf("serialized retry failed: %v", err) } } if puts.Load() != 1 || reports.Load() != 1 { t.Fatalf("concurrent retry repeated a completed side effect: puts=%d reports=%d", puts.Load(), reports.Load()) } } func TestRecordingRecoveryRootMustBePrivate(t *testing.T) { root := filepath.Join(t.TempDir(), "world-readable") if err := os.Mkdir(root, 0700); err != nil { t.Fatal(err) } if err := os.Chmod(root, 0755); err != nil { t.Fatal(err) } recovery := RecordingRecovery{Root: root} _, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err == nil { t.Fatal("restricted recording was saved under a public recovery root") } files, err := os.ReadDir(root) if err != nil || len(files) != 0 { t.Fatalf("rejected root nevertheless contains recording files: files=%v err=%v", files, err) } } func TestRecordingRecoveryGrantFailureAlsoBacksOff(t *testing.T) { now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) recovery := RecordingRecovery{Root: privateRecoveryRoot(t), Now: func() time.Time { return now }} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } now = now.Add(time.Minute) grantCalls := 0 grant := func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { grantCalls++ return RecoveryTarget{}, errors.New("temporary Dispatcher outage") } _, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, func(context.Context, RecordingRecoveryEntry) error { return nil }) if err == nil || grantCalls != 1 { t.Fatalf("failed grant request was hidden: calls=%d err=%v", grantCalls, err) } pending, err := recovery.Load(entry.Bucket, entry.ObjectKey) if err != nil || pending.State != "retry_pending" || pending.Attempts != 2 || pending.NextAttemptAt != now.Add(2*time.Minute) { t.Fatalf("grant failure caused an immediate unbounded retry: state=%+v err=%v", pending, err) } now = now.Add(time.Minute) if _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, func(context.Context, RecordingRecoveryEntry) error { return nil }); !errors.Is(err, ErrRecordingNotDue) || grantCalls != 1 { t.Fatalf("grant request retried before the next window: calls=%d err=%v", grantCalls, err) } }