package agent import ( "context" "crypto/sha256" "errors" "fmt" "log/slog" "strings" "time" agentpb "git.ipao.vip/rogee/go-sip/gen/agent" ) var ErrRecordingNotDue = errors.New("recording recovery retry is not due") var ErrRecordingRetryExpired = errors.New("recording recovery retry window expired") // RecoveryTarget must come from the Dispatcher for the original upload. The // fresh grant may change its token, but never its bucket, key, ID or checksum. type RecoveryTarget struct { Bucket string Grant *agentpb.UploadGrant } type RecoveryGrant func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error) type RecordingResultReporter func(context.Context, RecordingRecoveryEntry) error // Retry runs one due recovery entry. The persisted in-flight barrier prevents // a process restart from sending another PUT when its previous outcome is // unknown. Once a PUT succeeds, only the original result is reported again. func (r *RecordingRecovery) Retry(ctx context.Context, bucket, objectKey string, grant RecoveryGrant, report RecordingResultReporter) (result RecordingRecoveryEntry, retryErr error) { phase := "load_recording_recovery" started := time.Now() digest := sha256.Sum256([]byte(bucket + "/" + objectKey)) var original RecordingRecoveryEntry defer func() { observed := result if observed.SourceEventID == "" { observed = original } attrs := append(recordingRecoveryAttrs(observed), "recovery_digest", fmt.Sprintf("%x", digest[:6]), "stage", phase, "duration_ms", time.Since(started).Milliseconds()) manualReview := observed.State == "expired" || observed.State == "put_in_flight" || observed.State == "outcome_unknown" if manualReview && (retryErr == nil || errors.Is(retryErr, ErrRecordingNotDue)) { reason := "original upload outcome unknown; files retained and PUT replay blocked" if observed.State == "expired" { reason = "48-hour retry window expired; files retained for inspection" } slog.WarnContext(ctx, "Recording recovery requires inspection", append(attrs, "stage", "recovery_manual_review", "reason", reason, "files_retained", true)...) } else if errors.Is(retryErr, ErrRecordingNotDue) { slog.InfoContext(ctx, "Recording retry waiting", append(attrs, "stage", "retry_wait", "reason", retryErr)...) } else if retryErr != nil { slog.ErrorContext(ctx, "Recording retry blocked", append(attrs, "error", retryErr, "manual_review", observed.State == "expired" || observed.State == "put_in_flight" || observed.State == "outcome_unknown")...) } else { slog.InfoContext(ctx, "Recording recovery transition complete", attrs...) } }() r.mu.Lock() defer r.mu.Unlock() entry, err := r.Load(bucket, objectKey) if err != nil { return RecordingRecoveryEntry{}, err } original = entry phase = "validate_recording_recovery" _, audioPath, infoPath, err := r.paths(bucket, objectKey) if err != nil { return entry, err } if err := ctx.Err(); err != nil { return entry, err } switch entry.State { case "delivered": phase = "recovery_already_delivered" return entry, nil case "expired": return entry, ErrRecordingRetryExpired case "outcome_unknown", "put_in_flight": return entry, ErrUploadOutcomeUnknown case "uploaded_unreported": phase = "report_original_result" return r.reportUploaded(ctx, infoPath, entry, report) case "retry_pending": default: return entry, ErrRecordingRecoveryIncomplete } phase = "check_retry_window" now := r.now().UTC() if !now.Before(entry.ExpiresAt) { entry.State = "expired" entry.NextAttemptAt = time.Time{} phase = "save_retry_expired" if err := r.persistState(infoPath, entry); err != nil { return entry, fmt.Errorf("persist expired recording recovery: %w", err) } return entry, ErrRecordingRetryExpired } if now.Before(entry.NextAttemptAt) { return entry, ErrRecordingNotDue } if grant == nil || report == nil { return entry, errors.New("Dispatcher recovery grant and final result reporter are required") } phase = "request_upload_grant" slog.InfoContext(ctx, "Request original recovery upload grant", append(recordingRecoveryAttrs(entry), "stage", phase, "direction", "agent->dispatcher")...) target, err := grant(ctx, entry) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return entry, ctxErr } entry.Attempts++ entry.NextAttemptAt = r.now().UTC().Add(recordingRetryDelay(entry.Attempts)) if saveErr := r.persistState(infoPath, entry); saveErr != nil { phase = "save_grant_backoff" return entry, errors.Join(fmt.Errorf("request original recovery grant failed: %w", err), fmt.Errorf("persist failed recovery grant backoff: %w", saveErr)) } slog.WarnContext(ctx, "Recording retry backoff saved", append(recordingRecoveryAttrs(entry), "stage", "retry_backoff_saved", "reason", err)...) return entry, fmt.Errorf("request original recovery grant failed: %w", err) } phase = "validate_original_upload_target" if err := r.validateTarget(ctx, entry, target); err != nil { return entry, err } inFlight := entry inFlight.State = "put_in_flight" inFlight.NextAttemptAt = time.Time{} phase = "save_put_in_flight" if err := r.persistState(infoPath, inFlight); err != nil { return entry, fmt.Errorf("persist PUT in-flight barrier: %w", err) } phase = "oss_put" slog.InfoContext(ctx, "Recording retry PUT started", append(recordingRecoveryAttrs(inFlight), "stage", phase, "direction", "agent->oss")...) uploaded, err := r.Upload.UploadFile(ctx, target.Grant, audioPath) if err != nil { var rejected *UploadHTTPError if errors.As(err, &rejected) { entry.Attempts++ entry.NextAttemptAt = r.now().UTC().Add(recordingRetryDelay(entry.Attempts)) if saveErr := r.persistState(infoPath, entry); saveErr != nil { phase = "save_put_backoff" return inFlight, fmt.Errorf("OSS rejected retry (status %d), recovery barrier retained: %w", rejected.StatusCode, saveErr) } slog.WarnContext(ctx, "Recording retry backoff saved", append(recordingRecoveryAttrs(entry), "stage", "retry_backoff_saved", "put_status", rejected.StatusCode, "reason", err)...) return entry, err } return inFlight, err } phase = "save_confirmed_put" inFlight.State = "uploaded_unreported" inFlight.Uploaded = &uploaded if err := r.persistState(infoPath, inFlight); err != nil { return inFlight, fmt.Errorf("persist confirmed OSS upload before reporting result: %w", err) } phase = "report_original_result" return r.reportUploaded(ctx, infoPath, inFlight, report) } func (r *RecordingRecovery) validateTarget(ctx context.Context, entry RecordingRecoveryEntry, target RecoveryTarget) error { if target.Grant == nil || target.Bucket != entry.Bucket || target.Grant.Bucket != entry.Bucket || target.Grant.UploadId != entry.UploadID || target.Grant.ObjectKey != entry.ObjectKey || target.Grant.MaxBytes < entry.SizeBytes || target.Grant.MaxBytes <= 0 || !strings.EqualFold(target.Grant.RequiredChecksumSha256, entry.SHA256) { return errors.New("recovery grant target does not match the original OSS asset") } if _, err := r.Upload.validateGrant(ctx, target.Grant); err != nil { return fmt.Errorf("recovery grant is invalid: %w", err) } return nil } func recordingRecoveryAttrs(entry RecordingRecoveryEntry) []any { return []any{"event_id", entry.SourceEventID, "dispatcher_id", entry.DispatcherID, "tenant_id", entry.TenantID, "recording_id", entry.RecordingID, "upload_id", entry.UploadID, "recovery_state", entry.State, "attempts", entry.Attempts, "next_attempt_at", entry.NextAttemptAt, "expires_at", entry.ExpiresAt} } func (r *RecordingRecovery) reportUploaded(ctx context.Context, infoPath string, entry RecordingRecoveryEntry, report RecordingResultReporter) (RecordingRecoveryEntry, error) { if report == nil { return entry, errors.New("Dispatcher final result reporter is required") } slog.InfoContext(ctx, "Report original uploaded result", append(recordingRecoveryAttrs(entry), "stage", "report_original_result", "direction", "agent->dispatcher")...) if err := report(ctx, entry); err != nil { slog.ErrorContext(ctx, "Original uploaded result reporting failed", append(recordingRecoveryAttrs(entry), "stage", "report_original_result", "error", err)...) return entry, fmt.Errorf("report confirmed upload result: %w", err) } entry.State = "delivered" if err := r.persistState(infoPath, entry); err != nil { slog.ErrorContext(ctx, "Delivered result receipt persistence failed", append(recordingRecoveryAttrs(entry), "stage", "save_delivered_receipt", "error", err)...) return entry, fmt.Errorf("persist delivered result receipt: %w", err) } slog.InfoContext(ctx, "Original result recovery acknowledged", append(recordingRecoveryAttrs(entry), "stage", "original_result_acknowledged")...) return entry, nil } func (r *RecordingRecovery) persistState(infoPath string, entry RecordingRecoveryEntry) error { if r.writeState != nil { return r.writeState(infoPath, entry) } return writeJSONAtomic(infoPath, entry) } func recordingRetryDelay(attempts int) time.Duration { delay := time.Minute for i := 1; i < attempts && delay < time.Hour; i++ { delay *= 2 if delay > time.Hour { return time.Hour } } return delay }