216 lines
9.2 KiB
Go
216 lines
9.2 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
|
)
|
|
|
|
var ErrRecordingNotDue = errors.New("recording recovery retry is not due")
|
|
var ErrRecordingRetryExpired = errors.New("recording recovery retry window expired")
|
|
|
|
// RecoveryTarget must come from the Dispatcher for the original upload. The
|
|
// fresh grant may change its token, but never its bucket, key, ID or checksum.
|
|
type RecoveryTarget struct {
|
|
Bucket string
|
|
Grant *agentpb.UploadGrant
|
|
}
|
|
|
|
type RecoveryGrant func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error)
|
|
type RecordingResultReporter func(context.Context, RecordingRecoveryEntry) error
|
|
|
|
// Retry runs one due recovery entry. The persisted in-flight barrier prevents
|
|
// a process restart from sending another PUT when its previous outcome is
|
|
// unknown. Once a PUT succeeds, only the original result is reported again.
|
|
func (r *RecordingRecovery) Retry(ctx context.Context, bucket, objectKey string, grant RecoveryGrant, report RecordingResultReporter) (result RecordingRecoveryEntry, retryErr error) {
|
|
phase := "load_recording_recovery"
|
|
started := time.Now()
|
|
digest := sha256.Sum256([]byte(bucket + "/" + objectKey))
|
|
var original RecordingRecoveryEntry
|
|
defer func() {
|
|
observed := result
|
|
if observed.SourceEventID == "" {
|
|
observed = original
|
|
}
|
|
attrs := append(recordingRecoveryAttrs(observed), "recovery_digest", fmt.Sprintf("%x", digest[:6]), "stage", phase, "duration_ms", time.Since(started).Milliseconds())
|
|
manualReview := observed.State == "expired" || observed.State == "put_in_flight" || observed.State == "outcome_unknown"
|
|
if manualReview && (retryErr == nil || errors.Is(retryErr, ErrRecordingNotDue)) {
|
|
reason := "original upload outcome unknown; files retained and PUT replay blocked"
|
|
if observed.State == "expired" {
|
|
reason = "48-hour retry window expired; files retained for inspection"
|
|
}
|
|
slog.WarnContext(ctx, "Recording recovery requires inspection", append(attrs, "stage", "recovery_manual_review", "reason", reason, "files_retained", true)...)
|
|
} else if errors.Is(retryErr, ErrRecordingNotDue) {
|
|
slog.InfoContext(ctx, "Recording retry waiting", append(attrs, "stage", "retry_wait", "reason", retryErr)...)
|
|
} else if retryErr != nil {
|
|
slog.ErrorContext(ctx, "Recording retry blocked", append(attrs, "error", retryErr, "manual_review", observed.State == "expired" || observed.State == "put_in_flight" || observed.State == "outcome_unknown")...)
|
|
} else {
|
|
slog.InfoContext(ctx, "Recording recovery transition complete", attrs...)
|
|
}
|
|
}()
|
|
dir, _, _, err := r.paths(bucket, objectKey)
|
|
if err != nil {
|
|
return RecordingRecoveryEntry{}, err
|
|
}
|
|
release, err := holdPrivateState(dir)
|
|
if err != nil {
|
|
return RecordingRecoveryEntry{}, err
|
|
}
|
|
defer release()
|
|
entry, err := r.load(bucket, objectKey)
|
|
if err != nil {
|
|
return RecordingRecoveryEntry{}, err
|
|
}
|
|
original = entry
|
|
phase = "validate_recording_recovery"
|
|
_, audioPath, infoPath, err := r.paths(bucket, objectKey)
|
|
if err != nil {
|
|
return entry, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return entry, err
|
|
}
|
|
switch entry.State {
|
|
case "delivered":
|
|
phase = "delivered_recording_cleanup"
|
|
_, err := r.cleanupDelivered(entry)
|
|
return entry, err
|
|
case "expired":
|
|
return entry, ErrRecordingRetryExpired
|
|
case "outcome_unknown", "put_in_flight":
|
|
return entry, ErrUploadOutcomeUnknown
|
|
case "uploaded_unreported":
|
|
phase = "report_original_result"
|
|
return r.reportUploaded(ctx, infoPath, entry, report)
|
|
case "retry_pending":
|
|
default:
|
|
return entry, ErrRecordingRecoveryIncomplete
|
|
}
|
|
phase = "check_retry_window"
|
|
now := r.now().UTC()
|
|
if !now.Before(entry.ExpiresAt) {
|
|
entry.State = "expired"
|
|
entry.NextAttemptAt = time.Time{}
|
|
phase = "save_retry_expired"
|
|
if err := r.persistState(infoPath, entry); err != nil {
|
|
return entry, fmt.Errorf("persist expired recording recovery: %w", err)
|
|
}
|
|
return entry, ErrRecordingRetryExpired
|
|
}
|
|
if now.Before(entry.NextAttemptAt) {
|
|
return entry, ErrRecordingNotDue
|
|
}
|
|
if grant == nil || report == nil {
|
|
return entry, errors.New("Dispatcher recovery grant and final result reporter are required")
|
|
}
|
|
phase = "request_upload_grant"
|
|
slog.InfoContext(ctx, "Request original recovery upload grant", append(recordingRecoveryAttrs(entry), "stage", phase, "direction", "agent->dispatcher")...)
|
|
target, err := grant(ctx, entry)
|
|
if err != nil {
|
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
|
return entry, ctxErr
|
|
}
|
|
entry.Attempts++
|
|
entry.NextAttemptAt = r.now().UTC().Add(recordingRetryDelay(entry.Attempts))
|
|
if saveErr := r.persistState(infoPath, entry); saveErr != nil {
|
|
phase = "save_grant_backoff"
|
|
return entry, errors.Join(fmt.Errorf("request original recovery grant failed: %w", err), fmt.Errorf("persist failed recovery grant backoff: %w", saveErr))
|
|
}
|
|
slog.WarnContext(ctx, "Recording retry backoff saved", append(recordingRecoveryAttrs(entry), "stage", "retry_backoff_saved", "reason", err)...)
|
|
return entry, fmt.Errorf("request original recovery grant failed: %w", err)
|
|
}
|
|
phase = "validate_original_upload_target"
|
|
if err := r.validateTarget(ctx, entry, target); err != nil {
|
|
return entry, err
|
|
}
|
|
inFlight := entry
|
|
inFlight.State = "put_in_flight"
|
|
inFlight.NextAttemptAt = time.Time{}
|
|
phase = "save_put_in_flight"
|
|
if err := r.persistState(infoPath, inFlight); err != nil {
|
|
return entry, fmt.Errorf("persist PUT in-flight barrier: %w", err)
|
|
}
|
|
phase = "oss_put"
|
|
slog.InfoContext(ctx, "Recording retry PUT started", append(recordingRecoveryAttrs(inFlight), "stage", phase, "direction", "agent->oss")...)
|
|
uploaded, err := r.Upload.UploadFile(ctx, target.Grant, audioPath)
|
|
if err != nil {
|
|
var rejected *UploadHTTPError
|
|
if errors.As(err, &rejected) {
|
|
entry.Attempts++
|
|
entry.NextAttemptAt = r.now().UTC().Add(recordingRetryDelay(entry.Attempts))
|
|
if saveErr := r.persistState(infoPath, entry); saveErr != nil {
|
|
phase = "save_put_backoff"
|
|
return inFlight, fmt.Errorf("OSS rejected retry (status %d), recovery barrier retained: %w", rejected.StatusCode, saveErr)
|
|
}
|
|
slog.WarnContext(ctx, "Recording retry backoff saved", append(recordingRecoveryAttrs(entry), "stage", "retry_backoff_saved", "put_status", rejected.StatusCode, "reason", err)...)
|
|
return entry, err
|
|
}
|
|
return inFlight, err
|
|
}
|
|
phase = "save_confirmed_put"
|
|
inFlight.State = "uploaded_unreported"
|
|
inFlight.Uploaded = &uploaded
|
|
if err := r.persistState(infoPath, inFlight); err != nil {
|
|
return inFlight, fmt.Errorf("persist confirmed OSS upload before reporting result: %w", err)
|
|
}
|
|
phase = "report_original_result"
|
|
return r.reportUploaded(ctx, infoPath, inFlight, report)
|
|
}
|
|
|
|
func (r *RecordingRecovery) validateTarget(ctx context.Context, entry RecordingRecoveryEntry, target RecoveryTarget) error {
|
|
if target.Grant == nil || target.Bucket != entry.Bucket || target.Grant.Bucket != entry.Bucket || target.Grant.UploadId != entry.UploadID || target.Grant.ObjectKey != entry.ObjectKey || target.Grant.MaxBytes < entry.SizeBytes || target.Grant.MaxBytes <= 0 || !strings.EqualFold(target.Grant.RequiredChecksumSha256, entry.SHA256) {
|
|
return errors.New("recovery grant target does not match the original OSS asset")
|
|
}
|
|
if _, err := r.Upload.validateGrant(ctx, target.Grant); err != nil {
|
|
return fmt.Errorf("recovery grant is invalid: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func recordingRecoveryAttrs(entry RecordingRecoveryEntry) []any {
|
|
return []any{"event_id", entry.SourceEventID, "dispatcher_id", entry.DispatcherID, "tenant_id", entry.TenantID, "recording_id", entry.RecordingID, "upload_id", entry.UploadID, "recovery_state", entry.State, "attempts", entry.Attempts, "next_attempt_at", entry.NextAttemptAt, "expires_at", entry.ExpiresAt}
|
|
}
|
|
|
|
func (r *RecordingRecovery) reportUploaded(ctx context.Context, infoPath string, entry RecordingRecoveryEntry, report RecordingResultReporter) (RecordingRecoveryEntry, error) {
|
|
if report == nil {
|
|
return entry, errors.New("Dispatcher final result reporter is required")
|
|
}
|
|
slog.InfoContext(ctx, "Report original uploaded result", append(recordingRecoveryAttrs(entry), "stage", "report_original_result", "direction", "agent->dispatcher")...)
|
|
if err := report(ctx, entry); err != nil {
|
|
slog.ErrorContext(ctx, "Original uploaded result reporting failed", append(recordingRecoveryAttrs(entry), "stage", "report_original_result", "error", err)...)
|
|
return entry, fmt.Errorf("report confirmed upload result: %w", err)
|
|
}
|
|
entry.State = "delivered"
|
|
if err := r.persistState(infoPath, entry); err != nil {
|
|
slog.ErrorContext(ctx, "Delivered result receipt persistence failed", append(recordingRecoveryAttrs(entry), "stage", "save_delivered_receipt", "error", err)...)
|
|
return entry, fmt.Errorf("persist delivered result receipt: %w", err)
|
|
}
|
|
slog.InfoContext(ctx, "Original result recovery acknowledged", append(recordingRecoveryAttrs(entry), "stage", "original_result_acknowledged")...)
|
|
_, err := r.cleanupDelivered(entry)
|
|
return entry, err
|
|
}
|
|
|
|
func (r *RecordingRecovery) persistState(infoPath string, entry RecordingRecoveryEntry) error {
|
|
if r.writeState != nil {
|
|
return r.writeState(infoPath, entry)
|
|
}
|
|
return writeJSONAtomic(infoPath, entry)
|
|
}
|
|
|
|
func recordingRetryDelay(attempts int) time.Duration {
|
|
delay := time.Minute
|
|
for i := 1; i < attempts && delay < time.Hour; i++ {
|
|
delay *= 2
|
|
if delay > time.Hour {
|
|
return time.Hour
|
|
}
|
|
}
|
|
return delay
|
|
}
|