207 lines
8.1 KiB
Go
207 lines
8.1 KiB
Go
package agent
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const recordingRetryWindow = 48 * time.Hour
|
|
|
|
var ErrRecordingRecoveryIncomplete = errors.New("recording recovery files are incomplete or inconsistent")
|
|
|
|
// RecordingRecoveryEntry contains the original OSS target and the exact final
|
|
// result bytes. Temporary grants, credentials and signed URLs are never saved.
|
|
type RecordingRecoveryEntry struct {
|
|
CallID string `json:"call_id"`
|
|
SourceEventID string `json:"source_event_id"`
|
|
RecordingID string `json:"recording_id"`
|
|
UploadID string `json:"upload_id"`
|
|
Bucket string `json:"bucket"`
|
|
ObjectKey string `json:"object_key"`
|
|
SHA256 string `json:"sha256"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
ResultPayload []byte `json:"result_payload"`
|
|
Uploaded *UploadResult `json:"uploaded,omitempty"`
|
|
SavedAt time.Time `json:"saved_at"`
|
|
NextAttemptAt time.Time `json:"next_attempt_at,omitempty"`
|
|
ExpiresAt time.Time `json:"expires_at"`
|
|
Attempts int `json:"attempts"`
|
|
State string `json:"state"`
|
|
}
|
|
|
|
// RecordingRecovery is only used after a failed OSS PUT. A successful normal
|
|
// upload never creates either of its recovery files.
|
|
type RecordingRecovery struct {
|
|
Root string
|
|
Now func() time.Time
|
|
Upload UploadClient
|
|
|
|
mu sync.Mutex
|
|
writeState func(string, any) error // deterministic failure injection in package tests
|
|
}
|
|
|
|
func (r *RecordingRecovery) SaveFailure(audio []byte, entry RecordingRecoveryEntry, cause error) (RecordingRecoveryEntry, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
var rejected *UploadHTTPError
|
|
if !errors.Is(cause, ErrUploadOutcomeUnknown) && (!errors.As(cause, &rejected) || rejected.StatusCode < 300) {
|
|
return RecordingRecoveryEntry{}, errors.New("only a failed OSS PUT may create recording recovery files")
|
|
}
|
|
if len(audio) == 0 || entry.CallID == "" || entry.SourceEventID == "" || entry.RecordingID == "" || entry.UploadID == "" || len(entry.ResultPayload) == 0 || !json.Valid(entry.ResultPayload) {
|
|
return RecordingRecoveryEntry{}, errors.New("complete failed recording identity, audio and result are required")
|
|
}
|
|
if entry.State != "" || entry.Attempts != 0 || !entry.SavedAt.IsZero() || !entry.NextAttemptAt.IsZero() || !entry.ExpiresAt.IsZero() {
|
|
return RecordingRecoveryEntry{}, errors.New("new failed recording already has recovery state")
|
|
}
|
|
dir, audioPath, infoPath, err := r.paths(entry.Bucket, entry.ObjectKey)
|
|
if err != nil {
|
|
return RecordingRecoveryEntry{}, err
|
|
}
|
|
sha := sha256.Sum256(audio)
|
|
digest := hex.EncodeToString(sha[:])
|
|
if entry.SHA256 != "" && !strings.EqualFold(entry.SHA256, digest) {
|
|
return RecordingRecoveryEntry{}, ErrUploadChecksumMismatch
|
|
}
|
|
if entry.SizeBytes != 0 && entry.SizeBytes != int64(len(audio)) {
|
|
return RecordingRecoveryEntry{}, errors.New("failed recording size does not match the original grant")
|
|
}
|
|
if err := r.makePrivateDirs(entry.Bucket, entry.ObjectKey); err != nil {
|
|
return RecordingRecoveryEntry{}, err
|
|
}
|
|
file, err := os.OpenFile(audioPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
|
if err != nil {
|
|
return RecordingRecoveryEntry{}, fmt.Errorf("save failed recording: %w", err)
|
|
}
|
|
written, writeErr := file.Write(audio)
|
|
if writeErr == nil && written != len(audio) {
|
|
writeErr = io.ErrShortWrite
|
|
}
|
|
syncErr := file.Sync()
|
|
closeErr := file.Close()
|
|
if err := firstError(writeErr, syncErr, closeErr); err != nil {
|
|
return RecordingRecoveryEntry{}, fmt.Errorf("persist failed recording audio: %w", err)
|
|
}
|
|
if err := syncDirectory(dir); err != nil {
|
|
return RecordingRecoveryEntry{}, fmt.Errorf("persist failed recording path: %w", err)
|
|
}
|
|
entry.SHA256, entry.SizeBytes = digest, int64(len(audio))
|
|
entry.SavedAt = r.now().UTC()
|
|
entry.ExpiresAt = entry.SavedAt.Add(recordingRetryWindow)
|
|
entry.Attempts = 1
|
|
if errors.Is(cause, ErrUploadOutcomeUnknown) {
|
|
entry.State = "outcome_unknown"
|
|
} else {
|
|
entry.State = "retry_pending"
|
|
entry.NextAttemptAt = entry.SavedAt.Add(time.Minute)
|
|
}
|
|
if err := r.persistState(infoPath, entry); err != nil {
|
|
return RecordingRecoveryEntry{}, fmt.Errorf("persist failed recording info: %w", err)
|
|
}
|
|
return entry, nil
|
|
}
|
|
|
|
func (r *RecordingRecovery) Load(bucket, objectKey string) (RecordingRecoveryEntry, error) {
|
|
_, audioPath, infoPath, err := r.paths(bucket, objectKey)
|
|
if err != nil {
|
|
return RecordingRecoveryEntry{}, err
|
|
}
|
|
data, err := os.ReadFile(infoPath)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
if _, audioErr := os.Stat(audioPath); audioErr == nil {
|
|
return RecordingRecoveryEntry{}, ErrRecordingRecoveryIncomplete
|
|
}
|
|
}
|
|
if err != nil {
|
|
return RecordingRecoveryEntry{}, err
|
|
}
|
|
var entry RecordingRecoveryEntry
|
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&entry); err != nil {
|
|
return RecordingRecoveryEntry{}, fmt.Errorf("%w: decode info: %v", ErrRecordingRecoveryIncomplete, err)
|
|
}
|
|
var trailing any
|
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|
return RecordingRecoveryEntry{}, fmt.Errorf("%w: unexpected data after info object (%v)", ErrRecordingRecoveryIncomplete, err)
|
|
}
|
|
if entry.Bucket != bucket || entry.ObjectKey != objectKey || entry.CallID == "" || entry.SourceEventID == "" || entry.RecordingID == "" || entry.UploadID == "" || entry.SizeBytes <= 0 || entry.SHA256 == "" || !json.Valid(entry.ResultPayload) || entry.Attempts < 1 || entry.SavedAt.IsZero() || !entry.ExpiresAt.Equal(entry.SavedAt.Add(recordingRetryWindow)) {
|
|
return RecordingRecoveryEntry{}, ErrRecordingRecoveryIncomplete
|
|
}
|
|
switch entry.State {
|
|
case "retry_pending", "outcome_unknown", "put_in_flight", "uploaded_unreported", "delivered", "expired":
|
|
default:
|
|
return RecordingRecoveryEntry{}, ErrRecordingRecoveryIncomplete
|
|
}
|
|
if entry.State == "uploaded_unreported" || entry.State == "delivered" {
|
|
if entry.Uploaded == nil || entry.Uploaded.StatusCode < 200 || entry.Uploaded.StatusCode >= 300 || entry.Uploaded.SizeBytes != entry.SizeBytes || !strings.EqualFold(entry.Uploaded.SHA256, entry.SHA256) {
|
|
return RecordingRecoveryEntry{}, ErrRecordingRecoveryIncomplete
|
|
}
|
|
} else if entry.Uploaded != nil {
|
|
return RecordingRecoveryEntry{}, ErrRecordingRecoveryIncomplete
|
|
}
|
|
audio, err := os.ReadFile(audioPath)
|
|
if err != nil {
|
|
return RecordingRecoveryEntry{}, fmt.Errorf("%w: read original audio: %v", ErrRecordingRecoveryIncomplete, err)
|
|
}
|
|
sha := sha256.Sum256(audio)
|
|
if int64(len(audio)) != entry.SizeBytes || !strings.EqualFold(hex.EncodeToString(sha[:]), entry.SHA256) {
|
|
return RecordingRecoveryEntry{}, ErrRecordingRecoveryIncomplete
|
|
}
|
|
return entry, nil
|
|
}
|
|
|
|
func (r *RecordingRecovery) paths(bucket, objectKey string) (dir, audioPath, infoPath string, err error) {
|
|
if strings.TrimSpace(r.Root) == "" || validateName(bucket) != nil || objectKey == "" || strings.ContainsAny(objectKey, "\\\x00") {
|
|
return "", "", "", errors.New("recovery root, bucket or object key is invalid")
|
|
}
|
|
for _, segment := range strings.Split(objectKey, "/") {
|
|
if segment == "" || segment == "." || segment == ".." {
|
|
return "", "", "", errors.New("recovery object key cannot escape its bucket")
|
|
}
|
|
}
|
|
dir = filepath.Join(r.Root, bucket, filepath.FromSlash(objectKey))
|
|
return dir, filepath.Join(dir, "recording.wav"), filepath.Join(dir, "info.json"), nil
|
|
}
|
|
|
|
func (r *RecordingRecovery) makePrivateDirs(bucket, objectKey string) error {
|
|
if err := os.MkdirAll(r.Root, 0700); err != nil {
|
|
return err
|
|
}
|
|
current := r.Root
|
|
if err := checkRecoveryDirectory(current); err != nil {
|
|
return fmt.Errorf("recovery root: %w", err)
|
|
}
|
|
for _, segment := range append([]string{bucket}, strings.Split(objectKey, "/")...) {
|
|
child := filepath.Join(current, segment)
|
|
if err := os.Mkdir(child, 0700); err == nil {
|
|
if err := syncDirectory(current); err != nil {
|
|
return err
|
|
}
|
|
} else if !errors.Is(err, os.ErrExist) {
|
|
return err
|
|
}
|
|
if err := checkRecoveryDirectory(child); err != nil {
|
|
return fmt.Errorf("recovery target directory: %w", err)
|
|
}
|
|
current = child
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *RecordingRecovery) now() time.Time {
|
|
if r.Now != nil {
|
|
return r.Now()
|
|
}
|
|
return time.Now()
|
|
}
|