go-sip/internal/agent/recording_retry.go

package agent

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
)

var ErrRecordingNotDue = errors.New("recording recovery retry is not due")
var ErrRecordingRetryExpired = errors.New("recording recovery retry window expired")

// RecoveryTarget must come from the Dispatcher for the original upload. The
// fresh grant may change its token, but never its bucket, key, ID or checksum.
type RecoveryTarget struct {
	Bucket string
	Grant  *agentpb.UploadGrant
}

type RecoveryGrant func(context.Context, RecordingRecoveryEntry) (RecoveryTarget, error)
type RecordingResultReporter func(context.Context, RecordingRecoveryEntry) error

// Retry runs one due recovery entry. The persisted in-flight barrier prevents
// a process restart from sending another PUT when its previous outcome is
// unknown. Once a PUT succeeds, only the original result is reported again.
func (r *RecordingRecovery) Retry(ctx context.Context, bucket, objectKey string, grant RecoveryGrant, report RecordingResultReporter) (RecordingRecoveryEntry, error) {
	r.mu.Lock()
	defer r.mu.Unlock()
	entry, err := r.Load(bucket, objectKey)
	if err != nil {
		return RecordingRecoveryEntry{}, err
	}
	_, audioPath, infoPath, err := r.paths(bucket, objectKey)
	if err != nil {
		return entry, err
	}
	if err := ctx.Err(); err != nil {
		return entry, err
	}
	switch entry.State {
	case "delivered":
		return entry, nil
	case "expired":
		return entry, ErrRecordingRetryExpired
	case "outcome_unknown", "put_in_flight":
		return entry, ErrUploadOutcomeUnknown
	case "uploaded_unreported":
		return r.reportUploaded(ctx, infoPath, entry, report)
	case "retry_pending":
	default:
		return entry, ErrRecordingRecoveryIncomplete
	}
	now := r.now().UTC()
	if !now.Before(entry.ExpiresAt) {
		entry.State = "expired"
		entry.NextAttemptAt = time.Time{}
		if err := r.persistState(infoPath, entry); err != nil {
			return entry, fmt.Errorf("persist expired recording recovery: %w", err)
		}
		return entry, ErrRecordingRetryExpired
	}
	if now.Before(entry.NextAttemptAt) {
		return entry, ErrRecordingNotDue
	}
	if grant == nil || report == nil {
		return entry, errors.New("Dispatcher recovery grant and final result reporter are required")
	}
	target, err := grant(ctx, entry)
	if err != nil {
		if ctxErr := ctx.Err(); ctxErr != nil {
			return entry, ctxErr
		}
		entry.Attempts++
		entry.NextAttemptAt = r.now().UTC().Add(recordingRetryDelay(entry.Attempts))
		if saveErr := r.persistState(infoPath, entry); saveErr != nil {
			return entry, fmt.Errorf("persist failed recovery grant backoff: %w", saveErr)
		}
		return entry, fmt.Errorf("request original recovery grant failed (%T)", err)
	}
	if err := r.validateTarget(ctx, entry, target); err != nil {
		return entry, err
	}
	inFlight := entry
	inFlight.State = "put_in_flight"
	inFlight.NextAttemptAt = time.Time{}
	if err := r.persistState(infoPath, inFlight); err != nil {
		return entry, fmt.Errorf("persist PUT in-flight barrier: %w", err)
	}
	uploaded, err := r.Upload.UploadFile(ctx, target.Grant, audioPath)
	if err != nil {
		var rejected *UploadHTTPError
		if errors.As(err, &rejected) {
			entry.Attempts++
			entry.NextAttemptAt = r.now().UTC().Add(recordingRetryDelay(entry.Attempts))
			if saveErr := r.persistState(infoPath, entry); saveErr != nil {
				return inFlight, fmt.Errorf("OSS rejected retry (status %d), recovery barrier retained: %w", rejected.StatusCode, saveErr)
			}
			return entry, err
		}
		return inFlight, err
	}
	inFlight.State = "uploaded_unreported"
	inFlight.Uploaded = &uploaded
	if err := r.persistState(infoPath, inFlight); err != nil {
		return inFlight, fmt.Errorf("persist confirmed OSS upload before reporting result: %w", err)
	}
	return r.reportUploaded(ctx, infoPath, inFlight, report)
}

func (r *RecordingRecovery) validateTarget(ctx context.Context, entry RecordingRecoveryEntry, target RecoveryTarget) error {
	if target.Grant == nil || target.Bucket != entry.Bucket || target.Grant.Bucket != entry.Bucket || target.Grant.UploadId != entry.UploadID || target.Grant.ObjectKey != entry.ObjectKey || target.Grant.MaxBytes < entry.SizeBytes || target.Grant.MaxBytes <= 0 || !strings.EqualFold(target.Grant.RequiredChecksumSha256, entry.SHA256) {
		return errors.New("recovery grant target does not match the original OSS asset")
	}
	if _, err := r.Upload.validateGrant(ctx, target.Grant); err != nil {
		return fmt.Errorf("recovery grant is invalid: %w", err)
	}
	return nil
}

func (r *RecordingRecovery) reportUploaded(ctx context.Context, infoPath string, entry RecordingRecoveryEntry, report RecordingResultReporter) (RecordingRecoveryEntry, error) {
	if report == nil {
		return entry, errors.New("Dispatcher final result reporter is required")
	}
	if err := report(ctx, entry); err != nil {
		return entry, fmt.Errorf("report confirmed upload result: %w", err)
	}
	entry.State = "delivered"
	if err := r.persistState(infoPath, entry); err != nil {
		return entry, fmt.Errorf("persist delivered result receipt: %w", err)
	}
	return entry, nil
}

func (r *RecordingRecovery) persistState(infoPath string, entry RecordingRecoveryEntry) error {
	if r.writeState != nil {
		return r.writeState(infoPath, entry)
	}
	return writeJSONAtomic(infoPath, entry)
}

func recordingRetryDelay(attempts int) time.Duration {
	delay := time.Minute
	for i := 1; i < attempts && delay < time.Hour; i++ {
		delay *= 2
		if delay > time.Hour {
			return time.Hour
		}
	}
	return delay
}