289 lines
14 KiB
Go
289 lines
14 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
|
)
|
|
|
|
func recoveryTarget(now time.Time, entry RecordingRecoveryEntry, signedURL string) RecoveryTarget {
|
|
return RecoveryTarget{Bucket: entry.Bucket, Grant: &agentpb.UploadGrant{
|
|
UploadId: entry.UploadID, Bucket: entry.Bucket, ObjectKey: entry.ObjectKey, TargetUrl: signedURL,
|
|
ExpiresAtUnixMs: now.Add(15 * time.Minute).UnixMilli(), MaxBytes: entry.SizeBytes,
|
|
RequiredChecksumSha256: entry.SHA256,
|
|
}}
|
|
}
|
|
|
|
func TestRecordingRecoveryBackoffAndResultRetryDoNotRepeatSuccessfulPUT(t *testing.T) {
|
|
root := privateRecoveryRoot(t)
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var puts atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPut || r.URL.Path != "/original" {
|
|
http.Error(w, "changed target", http.StatusBadRequest)
|
|
return
|
|
}
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
if puts.Add(1) == 1 {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
var grants atomic.Int32
|
|
grant := func(_ context.Context, e RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
grants.Add(1)
|
|
return recoveryTarget(now, e, server.URL+"/original?signature=DO_NOT_LOG"), nil
|
|
}
|
|
var reports atomic.Int32
|
|
report := func(_ context.Context, e RecordingRecoveryEntry) error {
|
|
if !strings.EqualFold(e.SHA256, entry.SHA256) || e.SourceEventID != entry.SourceEventID || string(e.ResultPayload) != string(entry.ResultPayload) || e.Uploaded == nil {
|
|
t.Errorf("final result lost its original identity or confirmed upload: %+v", e)
|
|
}
|
|
if reports.Add(1) == 1 {
|
|
return errors.New("Dispatcher result receipt unavailable")
|
|
}
|
|
return nil
|
|
}
|
|
if _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, report); !errors.Is(err, ErrRecordingNotDue) || puts.Load() != 0 {
|
|
t.Fatalf("first retry must wait at least one minute: puts=%d err=%v", puts.Load(), err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
if _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, report); err == nil || puts.Load() != 1 {
|
|
t.Fatalf("first due retry should preserve the original failed target: puts=%d err=%v", puts.Load(), err)
|
|
}
|
|
pending, err := recovery.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || pending.State != "retry_pending" || pending.Attempts != 2 || pending.NextAttemptAt != now.Add(2*time.Minute) {
|
|
t.Fatalf("failure did not persist its next bounded retry: entry=%+v err=%v", pending, err)
|
|
}
|
|
now = now.Add(2 * time.Minute)
|
|
if _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, report); err == nil || puts.Load() != 2 || reports.Load() != 1 {
|
|
t.Fatalf("successful retry must not be called successful if reporting failed: puts=%d reports=%d err=%v", puts.Load(), reports.Load(), err)
|
|
}
|
|
uploaded, err := recovery.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || uploaded.State != "uploaded_unreported" || uploaded.Uploaded == nil || uploaded.Uploaded.StatusCode != 200 {
|
|
t.Fatalf("successful PUT was not durably marked before reporting: entry=%+v err=%v", uploaded, err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: recovery.Upload}
|
|
neverGrant := func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
t.Fatal("confirmed successful OSS upload requested another PUT grant")
|
|
return RecoveryTarget{}, errors.New("duplicate grant")
|
|
}
|
|
processed, err := restarted.RecoverDue(context.Background(), neverGrant, report)
|
|
finished, loadErr := restarted.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || loadErr != nil || processed != 1 || finished.State != "delivered" || puts.Load() != 2 || grants.Load() != 2 || reports.Load() != 2 {
|
|
t.Fatalf("restart must resend only original result: recovered=%d state=%s puts=%d grants=%d reports=%d err=%v load=%v", processed, finished.State, puts.Load(), grants.Load(), reports.Load(), err, loadErr)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "mock-bucket", "tenant", "call", "rec.wav", "recording.wav")); err != nil {
|
|
t.Fatalf("failed-path original audio must not be cleared automatically: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryExpiresAfterExactly48HoursWithoutResult(t *testing.T) {
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
recovery := RecordingRecovery{Root: privateRecoveryRoot(t), Now: func() time.Time { return now }}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(48 * time.Hour)
|
|
grantCalls, reportCalls := 0, 0
|
|
_, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey,
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
grantCalls++
|
|
return RecoveryTarget{}, nil
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error { reportCalls++; return nil })
|
|
if !errors.Is(err, ErrRecordingRetryExpired) || grantCalls != 0 || reportCalls != 0 {
|
|
t.Fatalf("expired recording was uploaded or given a fabricated result: grant=%d report=%d err=%v", grantCalls, reportCalls, err)
|
|
}
|
|
retained, err := recovery.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || retained.State != "expired" {
|
|
t.Fatalf("expired files must remain for manual handling: state=%s err=%v", retained.State, err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryUnknownPUTNeverRetriesAfterRestart(t *testing.T) {
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
root := privateRecoveryRoot(t)
|
|
recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), ErrUploadOutcomeUnknown)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(time.Hour)
|
|
grants := 0
|
|
restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }}
|
|
_, err = restarted.Retry(context.Background(), entry.Bucket, entry.ObjectKey,
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
grants++
|
|
return RecoveryTarget{}, nil
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error {
|
|
t.Fatal("unknown PUT fabricated a result")
|
|
return nil
|
|
})
|
|
if !errors.Is(err, ErrUploadOutcomeUnknown) || grants != 0 {
|
|
t.Fatalf("unknown PUT was retried: grants=%d err=%v", grants, err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryRejectsChangedBucketBeforePUT(t *testing.T) {
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
recovery := RecordingRecovery{Root: privateRecoveryRoot(t), Now: func() time.Time { return now }}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
_, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey,
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
return RecoveryTarget{Bucket: "changed-bucket"}, nil
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error {
|
|
t.Fatal("changed target fabricated a result")
|
|
return nil
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "target") {
|
|
t.Fatalf("changed OSS bucket was silently used: %v", err)
|
|
}
|
|
pending, err := recovery.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || pending.State != "retry_pending" {
|
|
t.Fatalf("original target was overwritten: state=%s err=%v", pending.State, err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryRejectsChangedSignedGrantBucketBeforePUT(t *testing.T) {
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
var puts atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { puts.Add(1) }))
|
|
defer server.Close()
|
|
recovery := RecordingRecovery{Root: privateRecoveryRoot(t), Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
_, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey,
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
target := recoveryTarget(now, entry, server.URL+"/original?signature=DO_NOT_LOG")
|
|
target.Grant.Bucket = "changed-bucket"
|
|
return target, nil
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error { return errors.New("unexpected report") })
|
|
if err == nil || !strings.Contains(err.Error(), "target") || puts.Load() != 0 {
|
|
t.Fatalf("changed signed-grant bucket reached OSS: puts=%d err=%v", puts.Load(), err)
|
|
}
|
|
pending, err := recovery.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || pending.State != "retry_pending" {
|
|
t.Fatalf("changed grant moved original recording: state=%s err=%v", pending.State, err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryPersistsInFlightBeforeUnknownPUT(t *testing.T) {
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
root := privateRecoveryRoot(t)
|
|
var puts atomic.Int32
|
|
recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{Now: func() time.Time { return now }, HTTPClient: &http.Client{Transport: ambiguousBytesTransport{requests: &puts}}}}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
grant := func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
return recoveryTarget(now, entry, "https://oss.example.invalid/original?signature=DO_NOT_LOG"), nil
|
|
}
|
|
_, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, func(context.Context, RecordingRecoveryEntry) error {
|
|
t.Fatal("unknown PUT fabricated a result")
|
|
return nil
|
|
})
|
|
if !errors.Is(err, ErrUploadOutcomeUnknown) || puts.Load() != 1 {
|
|
t.Fatalf("transport outcome was hidden: puts=%d err=%v", puts.Load(), err)
|
|
}
|
|
loaded, err := recovery.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || loaded.State != "put_in_flight" {
|
|
t.Fatalf("PUT started without a durable in-flight barrier: state=%s err=%v", loaded.State, err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }}
|
|
_, err = restarted.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, func(context.Context, RecordingRecoveryEntry) error { t.Fatal("repeated unknown PUT"); return nil })
|
|
if !errors.Is(err, ErrUploadOutcomeUnknown) || puts.Load() != 1 {
|
|
t.Fatalf("restart repeated an uncertain OSS PUT: puts=%d err=%v", puts.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRetryBackoffStopsAtOneHour(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
attempts int
|
|
want time.Duration
|
|
}{{1, time.Minute}, {2, 2 * time.Minute}, {3, 4 * time.Minute}, {6, 32 * time.Minute}, {7, time.Hour}, {30, time.Hour}} {
|
|
if got := recordingRetryDelay(tc.attempts); got != tc.want {
|
|
t.Fatalf("attempt %d has retry delay %s, want %s", tc.attempts, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryCannotRepeatSuccessfulPUTAfterMetadataDiskFault(t *testing.T) {
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
root := privateRecoveryRoot(t)
|
|
var puts, reports atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
puts.Add(1)
|
|
}))
|
|
defer server.Close()
|
|
recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recovery.writeState = func(path string, value any) error {
|
|
state, ok := value.(RecordingRecoveryEntry)
|
|
if ok && state.State == "uploaded_unreported" {
|
|
return errors.New("disk full after confirmed PUT")
|
|
}
|
|
return writeJSONAtomic(path, value)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
_, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey,
|
|
func(_ context.Context, e RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
return recoveryTarget(now, e, server.URL), nil
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error { reports.Add(1); return nil })
|
|
if err == nil || !strings.Contains(err.Error(), "persist confirmed OSS upload") || puts.Load() != 1 || reports.Load() != 0 {
|
|
t.Fatalf("unrecorded PUT success fabricated a result or triggered another PUT: puts=%d reports=%d err=%v", puts.Load(), reports.Load(), err)
|
|
}
|
|
persisted, err := recovery.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || persisted.State != "put_in_flight" {
|
|
t.Fatalf("crash barrier was not retained: state=%s err=%v", persisted.State, err)
|
|
}
|
|
restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }}
|
|
_, err = restarted.Retry(context.Background(), entry.Bucket, entry.ObjectKey,
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
t.Fatal("duplicate grant after successful PUT")
|
|
return RecoveryTarget{}, nil
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error {
|
|
t.Fatal("unconfirmed upload fabricated a result")
|
|
return nil
|
|
})
|
|
if !errors.Is(err, ErrUploadOutcomeUnknown) || puts.Load() != 1 {
|
|
t.Fatalf("restart retried a confirmed but unjournaled upload: puts=%d err=%v", puts.Load(), err)
|
|
}
|
|
}
|