package agent import ( "context" "errors" "io" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" agentpb "git.ipao.vip/rogee/go-sip/gen/agent" ) func recoveryTarget(now time.Time, entry RecordingRecoveryEntry, signedURL string) RecoveryTarget { return RecoveryTarget{Bucket: entry.Bucket, Grant: &agentpb.UploadGrant{ UploadId: entry.UploadID, Bucket: entry.Bucket, ObjectKey: entry.ObjectKey, TargetUrl: signedURL, ExpiresAtUnixMs: now.Add(15 * time.Minute).UnixMilli(), MaxBytes: entry.SizeBytes, RequiredChecksumSha256: entry.SHA256, }} } func TestRecordingRecoveryBackoffAndResultRetryDoNotRepeatSuccessfulPUT(t *testing.T) { root := privateRecoveryRoot(t) now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } var puts atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPut || r.URL.Path != "/original" { http.Error(w, "changed target", http.StatusBadRequest) return } _, _ = io.Copy(io.Discard, r.Body) if puts.Add(1) == 1 { w.WriteHeader(http.StatusServiceUnavailable) } })) defer server.Close() var grants atomic.Int32 grant := func(_ context.Context, e RecordingRecoveryEntry) (RecoveryTarget, error) { grants.Add(1) return recoveryTarget(now, e, server.URL+"/original?signature=DO_NOT_LOG"), nil } var reports atomic.Int32 report := func(_ context.Context, e RecordingRecoveryEntry) error { if !strings.EqualFold(e.SHA256, entry.SHA256) || e.SourceEventID != entry.SourceEventID || string(e.ResultPayload) != string(entry.ResultPayload) || e.Uploaded == nil { t.Errorf("final result lost its original identity or confirmed upload: %+v", e) } if reports.Add(1) == 1 { return errors.New("Dispatcher result receipt unavailable") } return nil } if _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, report); !errors.Is(err, ErrRecordingNotDue) || puts.Load() != 0 { t.Fatalf("first retry must wait at least one minute: puts=%d err=%v", puts.Load(), err) } now = now.Add(time.Minute) if _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, report); err == nil || puts.Load() != 1 { t.Fatalf("first due retry should preserve the original failed target: puts=%d err=%v", puts.Load(), err) } pending, err := recovery.Load(entry.Bucket, entry.ObjectKey) if err != nil || pending.State != "retry_pending" || pending.Attempts != 2 || pending.NextAttemptAt != now.Add(2*time.Minute) { t.Fatalf("failure did not persist its next bounded retry: entry=%+v err=%v", pending, err) } now = now.Add(2 * time.Minute) if _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, report); err == nil || puts.Load() != 2 || reports.Load() != 1 { t.Fatalf("successful retry must not be called successful if reporting failed: puts=%d reports=%d err=%v", puts.Load(), reports.Load(), err) } uploaded, err := recovery.Load(entry.Bucket, entry.ObjectKey) if err != nil || uploaded.State != "uploaded_unreported" || uploaded.Uploaded == nil || uploaded.Uploaded.StatusCode != 200 { t.Fatalf("successful PUT was not durably marked before reporting: entry=%+v err=%v", uploaded, err) } now = now.Add(time.Minute) restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: recovery.Upload} neverGrant := func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { t.Fatal("confirmed successful OSS upload requested another PUT grant") return RecoveryTarget{}, errors.New("duplicate grant") } processed, err := restarted.RecoverDue(context.Background(), neverGrant, report) finished, loadErr := restarted.Load(entry.Bucket, entry.ObjectKey) if err != nil || loadErr != nil || processed != 1 || finished.State != "delivered" || puts.Load() != 2 || grants.Load() != 2 || reports.Load() != 2 { t.Fatalf("restart must resend only original result: recovered=%d state=%s puts=%d grants=%d reports=%d err=%v load=%v", processed, finished.State, puts.Load(), grants.Load(), reports.Load(), err, loadErr) } if _, err := os.Stat(filepath.Join(root, "mock-bucket", "tenant", "call", "rec.wav", "recording.wav")); err != nil { t.Fatalf("failed-path original audio must not be cleared automatically: %v", err) } } func TestRecordingRecoveryExpiresAfterExactly48HoursWithoutResult(t *testing.T) { now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) recovery := RecordingRecovery{Root: privateRecoveryRoot(t), Now: func() time.Time { return now }} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } now = now.Add(48 * time.Hour) grantCalls, reportCalls := 0, 0 _, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { grantCalls++ return RecoveryTarget{}, nil }, func(context.Context, RecordingRecoveryEntry) error { reportCalls++; return nil }) if !errors.Is(err, ErrRecordingRetryExpired) || grantCalls != 0 || reportCalls != 0 { t.Fatalf("expired recording was uploaded or given a fabricated result: grant=%d report=%d err=%v", grantCalls, reportCalls, err) } retained, err := recovery.Load(entry.Bucket, entry.ObjectKey) if err != nil || retained.State != "expired" { t.Fatalf("expired files must remain for manual handling: state=%s err=%v", retained.State, err) } } func TestRecordingRecoveryUnknownPUTNeverRetriesAfterRestart(t *testing.T) { now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) root := privateRecoveryRoot(t) recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), ErrUploadOutcomeUnknown) if err != nil { t.Fatal(err) } now = now.Add(time.Hour) grants := 0 restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }} _, err = restarted.Retry(context.Background(), entry.Bucket, entry.ObjectKey, func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { grants++ return RecoveryTarget{}, nil }, func(context.Context, RecordingRecoveryEntry) error { t.Fatal("unknown PUT fabricated a result") return nil }) if !errors.Is(err, ErrUploadOutcomeUnknown) || grants != 0 { t.Fatalf("unknown PUT was retried: grants=%d err=%v", grants, err) } } func TestRecordingRecoveryRejectsChangedBucketBeforePUT(t *testing.T) { now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) recovery := RecordingRecovery{Root: privateRecoveryRoot(t), Now: func() time.Time { return now }} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } now = now.Add(time.Minute) _, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { return RecoveryTarget{Bucket: "changed-bucket"}, nil }, func(context.Context, RecordingRecoveryEntry) error { t.Fatal("changed target fabricated a result") return nil }) if err == nil || !strings.Contains(err.Error(), "target") { t.Fatalf("changed OSS bucket was silently used: %v", err) } pending, err := recovery.Load(entry.Bucket, entry.ObjectKey) if err != nil || pending.State != "retry_pending" { t.Fatalf("original target was overwritten: state=%s err=%v", pending.State, err) } } func TestRecordingRecoveryRejectsChangedSignedGrantBucketBeforePUT(t *testing.T) { now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) var puts atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { puts.Add(1) })) defer server.Close() recovery := RecordingRecovery{Root: privateRecoveryRoot(t), Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } now = now.Add(time.Minute) _, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { target := recoveryTarget(now, entry, server.URL+"/original?signature=DO_NOT_LOG") target.Grant.Bucket = "changed-bucket" return target, nil }, func(context.Context, RecordingRecoveryEntry) error { return errors.New("unexpected report") }) if err == nil || !strings.Contains(err.Error(), "target") || puts.Load() != 0 { t.Fatalf("changed signed-grant bucket reached OSS: puts=%d err=%v", puts.Load(), err) } pending, err := recovery.Load(entry.Bucket, entry.ObjectKey) if err != nil || pending.State != "retry_pending" { t.Fatalf("changed grant moved original recording: state=%s err=%v", pending.State, err) } } func TestRecordingRecoveryPersistsInFlightBeforeUnknownPUT(t *testing.T) { now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) root := privateRecoveryRoot(t) var puts atomic.Int32 recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{Now: func() time.Time { return now }, HTTPClient: &http.Client{Transport: ambiguousBytesTransport{requests: &puts}}}} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } now = now.Add(time.Minute) grant := func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { return recoveryTarget(now, entry, "https://oss.example.invalid/original?signature=DO_NOT_LOG"), nil } _, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, func(context.Context, RecordingRecoveryEntry) error { t.Fatal("unknown PUT fabricated a result") return nil }) if !errors.Is(err, ErrUploadOutcomeUnknown) || puts.Load() != 1 { t.Fatalf("transport outcome was hidden: puts=%d err=%v", puts.Load(), err) } loaded, err := recovery.Load(entry.Bucket, entry.ObjectKey) if err != nil || loaded.State != "put_in_flight" { t.Fatalf("PUT started without a durable in-flight barrier: state=%s err=%v", loaded.State, err) } now = now.Add(time.Minute) restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }} _, err = restarted.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, func(context.Context, RecordingRecoveryEntry) error { t.Fatal("repeated unknown PUT"); return nil }) if !errors.Is(err, ErrUploadOutcomeUnknown) || puts.Load() != 1 { t.Fatalf("restart repeated an uncertain OSS PUT: puts=%d err=%v", puts.Load(), err) } } func TestRecordingRetryBackoffStopsAtOneHour(t *testing.T) { for _, tc := range []struct { attempts int want time.Duration }{{1, time.Minute}, {2, 2 * time.Minute}, {3, 4 * time.Minute}, {4, 8 * time.Minute}, {5, 16 * time.Minute}, {6, 32 * time.Minute}, {7, time.Hour}, {30, time.Hour}} { if got := recordingRetryDelay(tc.attempts); got != tc.want { t.Fatalf("attempt %d has retry delay %s, want %s", tc.attempts, got, tc.want) } } } func TestRecordingRecoveryCannotRepeatSuccessfulPUTAfterMetadataDiskFault(t *testing.T) { now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) root := privateRecoveryRoot(t) var puts, reports atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) puts.Add(1) })) defer server.Close() recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}} entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503}) if err != nil { t.Fatal(err) } recovery.writeState = func(path string, value any) error { state, ok := value.(RecordingRecoveryEntry) if ok && state.State == "uploaded_unreported" { return errors.New("disk full after confirmed PUT") } return writeJSONAtomic(path, value) } now = now.Add(time.Minute) _, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, func(_ context.Context, e RecordingRecoveryEntry) (RecoveryTarget, error) { return recoveryTarget(now, e, server.URL), nil }, func(context.Context, RecordingRecoveryEntry) error { reports.Add(1); return nil }) if err == nil || !strings.Contains(err.Error(), "persist confirmed OSS upload") || puts.Load() != 1 || reports.Load() != 0 { t.Fatalf("unrecorded PUT success fabricated a result or triggered another PUT: puts=%d reports=%d err=%v", puts.Load(), reports.Load(), err) } persisted, err := recovery.Load(entry.Bucket, entry.ObjectKey) if err != nil || persisted.State != "put_in_flight" { t.Fatalf("crash barrier was not retained: state=%s err=%v", persisted.State, err) } restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }} _, err = restarted.Retry(context.Background(), entry.Bucket, entry.ObjectKey, func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) { t.Fatal("duplicate grant after successful PUT") return RecoveryTarget{}, nil }, func(context.Context, RecordingRecoveryEntry) error { t.Fatal("unconfirmed upload fabricated a result") return nil }) if !errors.Is(err, ErrUploadOutcomeUnknown) || puts.Load() != 1 { t.Fatalf("restart retried a confirmed but unjournaled upload: puts=%d err=%v", puts.Load(), err) } }