172 lines
6.9 KiB
Go
172 lines
6.9 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestRecordingRecoveryDiscoversAndResumesAfterProcessRestart(t *testing.T) {
|
|
root := privateRecoveryRoot(t)
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
original := RecordingRecovery{Root: root, Now: func() time.Time { return now }}
|
|
entry, err := original.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
var puts, reports atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
puts.Add(1)
|
|
}))
|
|
defer server.Close()
|
|
restarted := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}}
|
|
grant := func(_ context.Context, e RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
return recoveryTarget(now, e, server.URL+"/original"), nil
|
|
}
|
|
report := func(_ context.Context, e RecordingRecoveryEntry) error {
|
|
if e.SourceEventID != entry.SourceEventID || e.Uploaded == nil {
|
|
t.Errorf("restarted reporter lost durable original upload identity: %+v", e)
|
|
}
|
|
reports.Add(1)
|
|
return nil
|
|
}
|
|
processed, err := restarted.RecoverDue(context.Background(), grant, report)
|
|
if err != nil || processed != 1 || puts.Load() != 1 || reports.Load() != 1 {
|
|
t.Fatalf("startup recovery did not restore the original result: processed=%d puts=%d reports=%d err=%v", processed, puts.Load(), reports.Load(), err)
|
|
}
|
|
processed, err = restarted.RecoverDue(context.Background(), grant, report)
|
|
if err != nil || processed != 0 || puts.Load() != 1 || reports.Load() != 1 {
|
|
t.Fatalf("delivered result was re-uploaded after second scan: processed=%d puts=%d reports=%d err=%v", processed, puts.Load(), reports.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryScannerReportsOrphanInsteadOfInventingResult(t *testing.T) {
|
|
root := privateRecoveryRoot(t)
|
|
broken := RecordingRecovery{Root: root, writeState: func(string, any) error { return errors.New("metadata disk full") }}
|
|
entry := failedRecordingFixture()
|
|
if _, err := broken.SaveFailure([]byte("RIFF original audio"), entry, &UploadHTTPError{StatusCode: 503}); err == nil {
|
|
t.Fatal("metadata disk fault was hidden")
|
|
}
|
|
restarted := RecordingRecovery{Root: root}
|
|
processed, err := restarted.RecoverDue(context.Background(),
|
|
func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
t.Fatal("orphan must not PUT")
|
|
return RecoveryTarget{}, nil
|
|
},
|
|
func(context.Context, RecordingRecoveryEntry) error {
|
|
t.Fatal("orphan must not report a result")
|
|
return nil
|
|
})
|
|
if processed != 0 || !errors.Is(err, ErrRecordingRecoveryIncomplete) {
|
|
t.Fatalf("orphan was silently skipped: processed=%d err=%v", processed, err)
|
|
}
|
|
trace := sha256.Sum256([]byte(filepath.ToSlash(filepath.Join(entry.Bucket, entry.ObjectKey))))
|
|
if !strings.Contains(err.Error(), hex.EncodeToString(trace[:6])) {
|
|
t.Fatalf("orphan error lacks a safe stable path reference: %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "mock-bucket", "tenant", "call", "rec.wav", "recording.wav")); err != nil {
|
|
t.Fatalf("original audio was removed while blocked: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryConcurrentRetryOnlyPutsOnce(t *testing.T) {
|
|
root := privateRecoveryRoot(t)
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
recovery := RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
var puts, reports atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
puts.Add(1)
|
|
time.Sleep(15 * time.Millisecond)
|
|
}))
|
|
defer server.Close()
|
|
grant := func(_ context.Context, e RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
return recoveryTarget(now, e, server.URL), nil
|
|
}
|
|
report := func(context.Context, RecordingRecoveryEntry) error { reports.Add(1); return nil }
|
|
var wg sync.WaitGroup
|
|
errs := make(chan error, 2)
|
|
for i := 0; i < 2; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, report)
|
|
errs <- err
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
close(errs)
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatalf("serialized retry failed: %v", err)
|
|
}
|
|
}
|
|
if puts.Load() != 1 || reports.Load() != 1 {
|
|
t.Fatalf("concurrent retry repeated a completed side effect: puts=%d reports=%d", puts.Load(), reports.Load())
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryRootMustBePrivate(t *testing.T) {
|
|
root := filepath.Join(t.TempDir(), "world-readable")
|
|
if err := os.Mkdir(root, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Chmod(root, 0755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recovery := RecordingRecovery{Root: root}
|
|
_, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err == nil {
|
|
t.Fatal("restricted recording was saved under a public recovery root")
|
|
}
|
|
files, err := os.ReadDir(root)
|
|
if err != nil || len(files) != 0 {
|
|
t.Fatalf("rejected root nevertheless contains recording files: files=%v err=%v", files, err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingRecoveryGrantFailureAlsoBacksOff(t *testing.T) {
|
|
now := time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC)
|
|
recovery := RecordingRecovery{Root: privateRecoveryRoot(t), Now: func() time.Time { return now }}
|
|
entry, err := recovery.SaveFailure([]byte("RIFF original audio"), failedRecordingFixture(), &UploadHTTPError{StatusCode: 503})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
grantCalls := 0
|
|
grant := func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) {
|
|
grantCalls++
|
|
return RecoveryTarget{}, errors.New("temporary Dispatcher outage")
|
|
}
|
|
_, err = recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, func(context.Context, RecordingRecoveryEntry) error { return nil })
|
|
if err == nil || grantCalls != 1 {
|
|
t.Fatalf("failed grant request was hidden: calls=%d err=%v", grantCalls, err)
|
|
}
|
|
pending, err := recovery.Load(entry.Bucket, entry.ObjectKey)
|
|
if err != nil || pending.State != "retry_pending" || pending.Attempts != 2 || pending.NextAttemptAt != now.Add(2*time.Minute) {
|
|
t.Fatalf("grant failure caused an immediate unbounded retry: state=%+v err=%v", pending, err)
|
|
}
|
|
now = now.Add(time.Minute)
|
|
if _, err := recovery.Retry(context.Background(), entry.Bucket, entry.ObjectKey, grant, func(context.Context, RecordingRecoveryEntry) error { return nil }); !errors.Is(err, ErrRecordingNotDue) || grantCalls != 1 {
|
|
t.Fatalf("grant request retried before the next window: calls=%d err=%v", grantCalls, err)
|
|
}
|
|
}
|