Deliver bounded recordings through OSS and recover failed PUTs
This commit is contained in:
@@ -66,7 +66,8 @@
|
||||
- Dispatcher 的终结与外呼回执顺序竞争隔离修复：原流程在 Agent 接受执行的 RPC 返回后才写入外呼回执，快速结束或 RPC 超时可能先到；现以一次 SQLite 事务在确认通话已结束后补齐原回执并释放占用，未知执行仅在确认结束后释放。迟到的执行响应、超时和重复结束不会产生第二份回执；注入 outbox 写入失败保留原占用。并发竞争及结束后立即生成唯一最终结果有单元测试；主入口真实 Agent 会话注入与通话执行仍未接线。
|
||||
- Dispatcher 录音事实 Unary RPC 隔离服务：`RequestRecordingUpload`、`ReportCallEnded`、`ReportCallResult` 均要求已配置本 D、核验 mTLS 指纹及当前 Agent 会话、数字租户和已保留的执行；复用官方 SDK 仅对原始录音签发固定 15 分钟授权，显式重申请仍用相同 bucket/object_key。结束事实可先于外呼响应而持久化原回执；录音结果核对 D 已存目标和 Agent 报告的成功 PUT，再与唯一结果 outbox 同事务提交。Mock 覆盖会话/租户拒绝、同资产重申请、上传前结果拒绝、坏 JSON、已上传与无录音结果及重复回报。隔离测试还通过本地双向 TLS 的 gRPC 实际传输：Agent `RecordingClient` 每次读取并克隆当前会话元数据，经受控 D 客户端领取授权、上报结束和唯一结果；未配置客户端明确拒绝。不包含实际 OSS PUT 或主入口批准执行的媒体录音接线。
|
||||
- 内存录音隔离组件：`RecordingSession` 仅复制共享通话流程实际读到和成功发送的 16-kHz PCM16，`EncodeMonoWAV` 直接在内存生成有界单声道 WAV；空音频、奇数字节、超过上限及未成功发送的音频都不能伪造成可上传录音。单元与 race 测试未产生业务文件。批准执行入口尚未接入该组件，且 Mock 中观测到的帧不等于真实 Asterisk 通话的全量媒体验收。
|
||||
- 已验证：`go test ./... -count=1`、`go test -race ./internal/media ./internal/callflow -count=1`、`go test -race ./internal/agent ./internal/rpc ./internal/store -count=1`（分批执行）、`go vet ./...`、`go build ./...`、`PATH=/tmp/sip-go-agent-tools/bin:$PATH bash scripts/check-current-contracts.sh`、`PATH=/tmp/sip-go-agent-tools/bin:$PATH bash scripts/check-proto.sh`、`git diff --check`。尚未完成实际录音到直传/失败恢复的接线、无录音与生成失败的真实调用、主入口 Agent↔Dispatcher 实际会话与录音执行接线（隔离 mTLS gRPC 已测）及上传事实/最终结果交付、MQ/端到端验收，不能宣称 P06 通过。
|
||||
- Agent 录音交付隔离组件：`RecordingDelivery` 先确认结束，再依照录音是否实际生成分别上报唯一空录音结果或请求原授权并直传内存 WAV；录音生成失败保留通话真实结果、空录音对象及明确原因，不虚构上传事实。隔离测试通过本地 HTTP PUT 和假 Dispatcher RPC 覆盖成功无业务文件、OSS 明确失败后私有文件保存、恢复写入失败、未知 PUT 隔离、重启重领原目标、上传已确认后只重发原结果。再次调用不会隐式重新 PUT；正常已确认上传但尚未被 D 持久收讫的跨进程间隙仍受 K16 边界约束。此处未连接真实 D gRPC、主入口批准执行媒体或 MQ。
|
||||
- 已验证：`go test ./... -count=1`、`go test -race ./internal/agent ./internal/rpc ./internal/store ./internal/callflow ./internal/media -count=1`、`go vet ./...`、`go build ./...`、`PATH=/tmp/sip-go-agent-tools/bin:$PATH bash scripts/check-current-contracts.sh`、`PATH=/tmp/sip-go-agent-tools/bin:$PATH bash scripts/check-proto.sh`、`git diff --check`。尚未完成批准执行媒体录音到 Agent 交付组件的主入口接线、主入口 Agent↔Dispatcher 实际会话与录音执行接线（隔离 mTLS gRPC 已测）、真实执行时上传事实/最终结果交付及 MQ/端到端验收，不能宣称 P06 通过。
|
||||
|
||||
## 验收台账
|
||||
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
||||
)
|
||||
|
||||
var ErrRecordingRetryManaged = errors.New("recording delivery already attempted; retry the original result or persisted recovery instead")
|
||||
|
||||
// CompletedRecording describes one confirmed call and, when expected, the
|
||||
// bounded in-memory WAV created from its actual media. Recording IDs are stable
|
||||
// across retries; ResultPayload has the approved call identity and recording:{}.
|
||||
type CompletedRecording struct {
|
||||
ResultPayload []byte
|
||||
Expected bool
|
||||
RecordingID string
|
||||
UploadID string
|
||||
WAV []byte
|
||||
DurationMS int64
|
||||
CaptureError error
|
||||
}
|
||||
|
||||
// RecordingDelivery owns one approved call. In-process retries may report a
|
||||
// confirmed PUT again, but can never repeat that PUT. Failed or uncertain PUTs
|
||||
// are handed to RecordingRecovery, not retried through Complete.
|
||||
type RecordingDelivery struct {
|
||||
Call RecordingClient
|
||||
Recovery *RecordingRecovery
|
||||
|
||||
mu sync.Mutex
|
||||
ended bool
|
||||
attempted bool
|
||||
pending []byte
|
||||
pendingProof *agentpb.UploadObservation
|
||||
}
|
||||
|
||||
func (d *RecordingDelivery) Complete(ctx context.Context, recording CompletedRecording) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.attempted {
|
||||
return ErrRecordingRetryManaged
|
||||
}
|
||||
// Release the confirmed call independently of OSS and final-result delivery.
|
||||
if !d.ended {
|
||||
if err := d.Call.ReportEnded(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
d.ended = true
|
||||
}
|
||||
result, err := emptyRecordingResult(recording.ResultPayload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !recording.Expected {
|
||||
if len(recording.WAV) != 0 || recording.CaptureError != nil || recording.RecordingID != "" || recording.UploadID != "" || recording.DurationMS != 0 {
|
||||
return errors.New("call without a recording cannot contain audio or upload identity")
|
||||
}
|
||||
return d.report(ctx, recording.ResultPayload, nil)
|
||||
}
|
||||
if recording.CaptureError != nil || len(recording.WAV) == 0 {
|
||||
if len(recording.WAV) != 0 {
|
||||
return errors.New("recording has both an error and apparently uploadable audio")
|
||||
}
|
||||
var reason string
|
||||
if err := json.Unmarshal(result["reason_message"], &reason); err != nil {
|
||||
return errors.New("failed recording needs the original call reason")
|
||||
}
|
||||
if reason != "" {
|
||||
reason += "; "
|
||||
}
|
||||
reason += "recording generation failed"
|
||||
result["reason_message"], err = json.Marshal(reason)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.report(ctx, payload, nil)
|
||||
}
|
||||
if recording.RecordingID == "" || recording.UploadID == "" || !validMonoWAV(recording.WAV, recording.DurationMS) {
|
||||
return errors.New("recording identity or 16-kHz mono WAV is invalid")
|
||||
}
|
||||
if d.Recovery == nil || strings.TrimSpace(d.Recovery.Root) == "" {
|
||||
return errors.New("private recording recovery directory and upload client are required before PUT")
|
||||
}
|
||||
sum := sha256.Sum256(recording.WAV)
|
||||
digest := hex.EncodeToString(sum[:])
|
||||
asset := &agentpb.AssetDescriptor{
|
||||
Kind: agentpb.AssetKind_ASSET_KIND_RECORDING, AssetId: recording.RecordingID,
|
||||
CallId: d.Call.SourceEventID, ExecutionId: d.Call.SourceEventID,
|
||||
Format: "wav", Channels: 1, SampleRateHz: 16000, DurationMs: recording.DurationMS,
|
||||
SizeBytes: int64(len(recording.WAV)), ChecksumSha256: digest,
|
||||
}
|
||||
grant, err := d.Call.RequestUpload(ctx, asset, recording.UploadID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result["recording"], err = json.Marshal(struct {
|
||||
Status string `json:"status"`
|
||||
Bucket string `json:"bucket"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
Format string `json:"format"`
|
||||
Channels int `json:"channels"`
|
||||
SampleRateHz int `json:"sample_rate_hz"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ChecksumSHA256 string `json:"checksum_sha256"`
|
||||
}{"uploaded", grant.Bucket, grant.ObjectKey, "wav", 1, 16000, recording.DurationMS, int64(len(recording.WAV)), digest})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A returned OSS failure needs the original exact result and bytes; no
|
||||
// failure in this path is allowed to fabricate an uploaded result.
|
||||
d.attempted = true
|
||||
uploaded, err := d.Recovery.Upload.UploadBytes(ctx, grant, recording.WAV)
|
||||
if err != nil {
|
||||
var rejected *UploadHTTPError
|
||||
if errors.As(err, &rejected) || errors.Is(err, ErrUploadOutcomeUnknown) {
|
||||
entry := RecordingRecoveryEntry{
|
||||
CallID: d.Call.SourceEventID, SourceEventID: d.Call.SourceEventID,
|
||||
RecordingID: recording.RecordingID, UploadID: recording.UploadID,
|
||||
Bucket: grant.Bucket, ObjectKey: grant.ObjectKey,
|
||||
SHA256: digest, SizeBytes: int64(len(recording.WAV)), ResultPayload: payload,
|
||||
}
|
||||
if _, saveErr := d.Recovery.SaveFailure(recording.WAV, entry, err); saveErr != nil {
|
||||
return errors.Join(err, fmt.Errorf("persist failed recording for recovery: %w", saveErr))
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
proof := &agentpb.UploadObservation{
|
||||
UploadId: recording.UploadID, RecordingId: recording.RecordingID,
|
||||
PutStatusCode: int32(uploaded.StatusCode), SizeBytes: uploaded.SizeBytes, ChecksumSha256: uploaded.SHA256,
|
||||
}
|
||||
return d.report(ctx, payload, proof)
|
||||
}
|
||||
|
||||
// RetryResult reports only the original result after a confirmed PUT (or a
|
||||
// no-recording result); it never requests another upload or sends audio.
|
||||
func (d *RecordingDelivery) RetryResult(ctx context.Context) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if len(d.pending) == 0 {
|
||||
return ErrRecordingRetryManaged
|
||||
}
|
||||
return d.report(ctx, d.pending, d.pendingProof)
|
||||
}
|
||||
|
||||
func (d *RecordingDelivery) report(ctx context.Context, payload []byte, proof *agentpb.UploadObservation) error {
|
||||
d.attempted = true
|
||||
d.pending = bytes.Clone(payload)
|
||||
d.pendingProof = proof
|
||||
if _, err := d.Call.ReportFinal(ctx, d.pending, d.pendingProof); err != nil {
|
||||
return err
|
||||
}
|
||||
d.pending = nil
|
||||
d.pendingProof = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func emptyRecordingResult(payload []byte) (map[string]json.RawMessage, error) {
|
||||
var result map[string]json.RawMessage
|
||||
if err := json.Unmarshal(payload, &result); err != nil || result == nil {
|
||||
return nil, errors.New("final call result must be a JSON object")
|
||||
}
|
||||
var recording map[string]json.RawMessage
|
||||
if err := json.Unmarshal(result["recording"], &recording); err != nil || recording == nil || len(recording) != 0 {
|
||||
return nil, errors.New("final call result must start with an empty recording object")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validMonoWAV(wav []byte, durationMS int64) bool {
|
||||
if len(wav) <= 44 || len(wav)%2 != 0 || uint64(len(wav)) > uint64(^uint32(0))+8 || durationMS < 0 {
|
||||
return false
|
||||
}
|
||||
return string(wav[:4]) == "RIFF" && binary.LittleEndian.Uint32(wav[4:8]) == uint32(len(wav)-8) &&
|
||||
string(wav[8:16]) == "WAVEfmt " && binary.LittleEndian.Uint32(wav[16:20]) == 16 &&
|
||||
binary.LittleEndian.Uint16(wav[20:22]) == 1 && binary.LittleEndian.Uint16(wav[22:24]) == 1 &&
|
||||
binary.LittleEndian.Uint32(wav[24:28]) == 16000 && binary.LittleEndian.Uint32(wav[28:32]) == 32000 &&
|
||||
binary.LittleEndian.Uint16(wav[32:34]) == 2 && binary.LittleEndian.Uint16(wav[34:36]) == 16 &&
|
||||
string(wav[36:40]) == "data" && binary.LittleEndian.Uint32(wav[40:44]) == uint32(len(wav)-44) &&
|
||||
int64(len(wav)-44)*1000/32000 == durationMS
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.ipao.vip/rogee/go-sip/internal/media"
|
||||
)
|
||||
|
||||
func TestRecordingDeliveryCaptureFailureDescribesFailureWithoutInventingUpload(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cause error
|
||||
}{
|
||||
{name: "bounded_capture_failed", cause: media.ErrWAVTooLarge},
|
||||
{name: "empty_expected_recording"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
delivery, stub, completed, puts, root := testDirectRecordingDelivery(t, http.StatusOK)
|
||||
completed.WAV = nil
|
||||
completed.CaptureError = tc.cause
|
||||
if err := delivery.Complete(context.Background(), completed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var result struct {
|
||||
Outcome string `json:"outcome"`
|
||||
ReasonMessage string `json:"reason_message"`
|
||||
Recording map[string]any `json:"recording"`
|
||||
}
|
||||
if err := json.Unmarshal(stub.resultBody, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(stub.calls, []string{"end", "result"}) || puts.Load() != 0 || stub.resultProof != nil || result.Outcome != "answered" || !strings.Contains(result.ReasonMessage, "recording generation failed") || len(result.Recording) != 0 {
|
||||
t.Fatalf("failed capture invented upload or changed the actual call outcome: calls=%v puts=%d result=%+v", stub.calls, puts.Load(), result)
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("no generated audio created a business file: entries=%d err=%v", len(entries), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryRejectsInvalidWAVBeforeGrantOrPUT(t *testing.T) {
|
||||
delivery, stub, completed, puts, _ := testDirectRecordingDelivery(t, http.StatusOK)
|
||||
completed.WAV = bytes.Clone(completed.WAV)
|
||||
completed.WAV[0] = 'X'
|
||||
if err := delivery.Complete(context.Background(), completed); err == nil {
|
||||
t.Fatal("corrupt WAV was reported as a valid upload")
|
||||
}
|
||||
if !reflect.DeepEqual(stub.calls, []string{"end"}) || stub.asset != nil || puts.Load() != 0 {
|
||||
t.Fatalf("invalid media reached upload: calls=%v asset=%v puts=%d", stub.calls, stub.asset, puts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryRecoveryDiskFailureNeverReportsUploaded(t *testing.T) {
|
||||
delivery, stub, completed, puts, root := testDirectRecordingDelivery(t, http.StatusServiceUnavailable)
|
||||
delivery.Recovery.writeState = func(string, any) error { return errors.New("injected disk full") }
|
||||
err := delivery.Complete(context.Background(), completed)
|
||||
var rejected *UploadHTTPError
|
||||
if !errors.As(err, &rejected) || !strings.Contains(err.Error(), "injected disk full") || puts.Load() != 1 {
|
||||
t.Fatalf("OSS and recovery-write failures were not both visible: err=%v puts=%d", err, puts.Load())
|
||||
}
|
||||
original, readErr := os.ReadFile(filepath.Join(root, stub.grant.Bucket, filepath.FromSlash(stub.grant.ObjectKey), "recording.wav"))
|
||||
if readErr != nil || !bytes.Equal(original, completed.WAV) || len(stub.resultBody) != 0 {
|
||||
t.Fatalf("disk failure lost the original audio or forged a result: read=%v result_size=%d", readErr, len(stub.resultBody))
|
||||
}
|
||||
if err := delivery.Complete(context.Background(), completed); !errors.Is(err, ErrRecordingRetryManaged) || puts.Load() != 1 {
|
||||
t.Fatalf("disk failure triggered an implicit second PUT: err=%v puts=%d", err, puts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
type recordingFailingTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f recordingFailingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return f(req)
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryUnknownPUTKeepsManualBarrierAcrossRestart(t *testing.T) {
|
||||
delivery, stub, completed, _, root := testDirectRecordingDelivery(t, http.StatusOK)
|
||||
var attempts atomic.Int32
|
||||
delivery.Recovery.Upload.HTTPClient = &http.Client{Transport: recordingFailingTransport(func(*http.Request) (*http.Response, error) {
|
||||
attempts.Add(1)
|
||||
return nil, errors.New("injected lost OSS response")
|
||||
})}
|
||||
if err := delivery.Complete(context.Background(), completed); !errors.Is(err, ErrUploadOutcomeUnknown) || attempts.Load() != 1 {
|
||||
t.Fatalf("unknown PUT was not retained as unknown: err=%v attempts=%d", err, attempts.Load())
|
||||
}
|
||||
entry, err := delivery.Recovery.Load(stub.grant.Bucket, stub.grant.ObjectKey)
|
||||
if err != nil || entry.State != "outcome_unknown" || len(stub.resultBody) != 0 {
|
||||
t.Fatalf("unknown PUT fabricated an uploaded result: state=%s err=%v", entry.State, err)
|
||||
}
|
||||
restarted := testRecordingDelivery(stub)
|
||||
restarted.Recovery = &RecordingRecovery{Root: root, Upload: delivery.Recovery.Upload}
|
||||
_, err = restarted.RetryRecovered(context.Background(), entry.Bucket, entry.ObjectKey)
|
||||
if !errors.Is(err, ErrUploadOutcomeUnknown) || attempts.Load() != 1 || !reflect.DeepEqual(stub.calls, []string{"end", "grant"}) {
|
||||
t.Fatalf("restart retried unknown PUT or reported a final result: err=%v attempts=%d calls=%v", err, attempts.Load(), stub.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryRestartRetriesOriginalAndReportsConfirmedPUTOnly(t *testing.T) {
|
||||
delivery, stub, completed, originalPUTs, root := testDirectRecordingDelivery(t, http.StatusServiceUnavailable)
|
||||
now := time.Now()
|
||||
delivery.Recovery.Now = func() time.Time { return now }
|
||||
delivery.Recovery.Upload.Now = func() time.Time { return now }
|
||||
if err := delivery.Complete(context.Background(), completed); err == nil || originalPUTs.Load() != 1 {
|
||||
t.Fatalf("original failed PUT not detected: err=%v puts=%d", err, originalPUTs.Load())
|
||||
}
|
||||
original, err := delivery.Recovery.Load(stub.grant.Bucket, stub.grant.ObjectKey)
|
||||
if err != nil || original.State != "retry_pending" {
|
||||
t.Fatalf("original upload identity not persisted: state=%s err=%v", original.State, err)
|
||||
}
|
||||
now = now.Add(time.Minute)
|
||||
var recoveredPUTs atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
recoveredPUTs.Add(1)
|
||||
if r.Method != http.MethodPut || r.URL.Path != "/fresh-token" {
|
||||
http.Error(w, "changed request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
stub.grant.TargetUrl = server.URL + "/fresh-token"
|
||||
stub.grant.ExpiresAtUnixMs = now.Add(15 * time.Minute).UnixMilli()
|
||||
stub.resultError = errors.New("injected Dispatcher result failure after confirmed PUT")
|
||||
freshRecovery := func() *RecordingRecovery {
|
||||
return &RecordingRecovery{Root: root, Now: func() time.Time { return now }, Upload: UploadClient{AllowInsecureHTTP: true, Now: func() time.Time { return now }}}
|
||||
}
|
||||
restarted := testRecordingDelivery(stub)
|
||||
restarted.Recovery = freshRecovery()
|
||||
_, err = restarted.RetryRecovered(context.Background(), original.Bucket, original.ObjectKey)
|
||||
if err == nil || recoveredPUTs.Load() != 1 || originalPUTs.Load() != 1 {
|
||||
t.Fatalf("first restarted retry did not preserve original grant/result boundary: err=%v recovered=%d initial=%d", err, recoveredPUTs.Load(), originalPUTs.Load())
|
||||
}
|
||||
pending, err := restarted.Recovery.Load(original.Bucket, original.ObjectKey)
|
||||
if err != nil || pending.State != "uploaded_unreported" || stub.asset == nil || stub.asset.CallId != original.CallID || stub.asset.ExecutionId != original.SourceEventID || stub.asset.ChecksumSha256 != original.SHA256 || stub.asset.SizeBytes != original.SizeBytes || stub.asset.DurationMs != completed.DurationMS || !bytes.Equal(stub.resultBody, original.ResultPayload) || stub.resultProof == nil {
|
||||
t.Fatalf("confirmed PUT was not preserved before result delivery: state=%s asset=%+v proof=%+v err=%v", pending.State, stub.asset, stub.resultProof, err)
|
||||
}
|
||||
stub.resultError = nil
|
||||
again := testRecordingDelivery(stub)
|
||||
again.Recovery = freshRecovery()
|
||||
finished, err := again.RetryRecovered(context.Background(), original.Bucket, original.ObjectKey)
|
||||
if err != nil || finished.State != "delivered" || recoveredPUTs.Load() != 1 || !reflect.DeepEqual(stub.calls, []string{"end", "grant", "grant", "result", "result"}) {
|
||||
t.Fatalf("restart repeated confirmed PUT or lost result: state=%s calls=%v puts=%d err=%v", finished.State, stub.calls, recoveredPUTs.Load(), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
||||
)
|
||||
|
||||
// RetryRecovered resumes one privately persisted failed upload. The original
|
||||
// destination, asset digest, upload identity and final result remain fixed;
|
||||
// only the short-lived grant may be reissued. Unknown PUTs stay blocked.
|
||||
func (d *RecordingDelivery) RetryRecovered(ctx context.Context, bucket, objectKey string) (RecordingRecoveryEntry, error) {
|
||||
if d == nil || d.Recovery == nil {
|
||||
return RecordingRecoveryEntry{}, ErrRecordingRecoveryIncomplete
|
||||
}
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.attempted = true
|
||||
d.ended = true // SaveFailure follows an acknowledged ReportCallEnded.
|
||||
return d.Recovery.Retry(ctx, bucket, objectKey, d.reissueRecordingGrant, d.reportRecoveredResult)
|
||||
}
|
||||
|
||||
func (d *RecordingDelivery) sameOriginalCall(entry RecordingRecoveryEntry) bool {
|
||||
return d.Call.SourceEventID != "" && entry.SourceEventID == d.Call.SourceEventID && entry.CallID == entry.SourceEventID
|
||||
}
|
||||
|
||||
func (d *RecordingDelivery) reissueRecordingGrant(ctx context.Context, entry RecordingRecoveryEntry) (RecoveryTarget, error) {
|
||||
if !d.sameOriginalCall(entry) {
|
||||
return RecoveryTarget{}, ErrRecordingRecoveryIncomplete
|
||||
}
|
||||
var original struct {
|
||||
Recording struct {
|
||||
Status string `json:"status"`
|
||||
Bucket string `json:"bucket"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
Format string `json:"format"`
|
||||
Channels int `json:"channels"`
|
||||
SampleRateHz int `json:"sample_rate_hz"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ChecksumSHA256 string `json:"checksum_sha256"`
|
||||
} `json:"recording"`
|
||||
}
|
||||
if err := json.Unmarshal(entry.ResultPayload, &original); err != nil {
|
||||
return RecoveryTarget{}, errors.Join(ErrRecordingRecoveryIncomplete, err)
|
||||
}
|
||||
r := original.Recording
|
||||
if r.Status != "uploaded" || r.Bucket != entry.Bucket || r.ObjectKey != entry.ObjectKey || r.Format != "wav" || r.Channels != 1 || r.SampleRateHz != 16000 || r.DurationMS < 0 || r.SizeBytes != entry.SizeBytes || r.ChecksumSHA256 != entry.SHA256 {
|
||||
return RecoveryTarget{}, ErrRecordingRecoveryIncomplete
|
||||
}
|
||||
asset := &agentpb.AssetDescriptor{
|
||||
Kind: agentpb.AssetKind_ASSET_KIND_RECORDING, AssetId: entry.RecordingID,
|
||||
CallId: entry.CallID, ExecutionId: entry.SourceEventID,
|
||||
Format: r.Format, Channels: int32(r.Channels), SampleRateHz: int32(r.SampleRateHz),
|
||||
DurationMs: r.DurationMS, SizeBytes: r.SizeBytes, ChecksumSha256: r.ChecksumSHA256,
|
||||
}
|
||||
grant, err := d.Call.RequestUpload(ctx, asset, entry.UploadID)
|
||||
if err != nil {
|
||||
return RecoveryTarget{}, err
|
||||
}
|
||||
return RecoveryTarget{Bucket: grant.Bucket, Grant: grant}, nil
|
||||
}
|
||||
|
||||
func (d *RecordingDelivery) reportRecoveredResult(ctx context.Context, entry RecordingRecoveryEntry) error {
|
||||
if !d.sameOriginalCall(entry) || entry.Uploaded == nil || entry.Uploaded.StatusCode < 200 || entry.Uploaded.StatusCode >= 300 || entry.Uploaded.SizeBytes != entry.SizeBytes || entry.Uploaded.SHA256 != entry.SHA256 {
|
||||
return ErrRecordingRecoveryIncomplete
|
||||
}
|
||||
_, err := d.Call.ReportFinal(ctx, entry.ResultPayload, &agentpb.UploadObservation{
|
||||
UploadId: entry.UploadID, RecordingId: entry.RecordingID,
|
||||
PutStatusCode: int32(entry.Uploaded.StatusCode), SizeBytes: entry.Uploaded.SizeBytes,
|
||||
ChecksumSha256: entry.Uploaded.SHA256,
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
||||
"git.ipao.vip/rogee/go-sip/internal/media"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type recordingDeliveryRPC struct {
|
||||
agentpb.AgentControlServiceClient
|
||||
calls []string
|
||||
endError error
|
||||
resultError error
|
||||
grant *agentpb.UploadGrant
|
||||
asset *agentpb.AssetDescriptor
|
||||
resultBody []byte
|
||||
resultProof *agentpb.UploadObservation
|
||||
}
|
||||
|
||||
func (s *recordingDeliveryRPC) RequestRecordingUpload(_ context.Context, req *agentpb.RequestRecordingUploadRequest, _ ...grpc.CallOption) (*agentpb.RequestRecordingUploadResponse, error) {
|
||||
s.calls = append(s.calls, "grant")
|
||||
s.asset = req.Asset
|
||||
if s.grant == nil {
|
||||
return nil, errors.New("injected missing grant")
|
||||
}
|
||||
return &agentpb.RequestRecordingUploadResponse{Grant: s.grant}, nil
|
||||
}
|
||||
|
||||
func (s *recordingDeliveryRPC) ReportCallEnded(_ context.Context, req *agentpb.ReportCallEndedRequest, _ ...grpc.CallOption) (*agentpb.ReportCallEndedResponse, error) {
|
||||
s.calls = append(s.calls, "end")
|
||||
if s.endError != nil {
|
||||
return nil, s.endError
|
||||
}
|
||||
return &agentpb.ReportCallEndedResponse{Receipt: &agentpb.OperationReceipt{FactId: req.SourceEventId, Result: agentpb.ResultCode_RESULT_CODE_APPLIED}}, nil
|
||||
}
|
||||
|
||||
func (s *recordingDeliveryRPC) ReportCallResult(_ context.Context, req *agentpb.ReportCallResultRequest, _ ...grpc.CallOption) (*agentpb.ReportCallResultResponse, error) {
|
||||
s.calls = append(s.calls, "result")
|
||||
s.resultBody = append([]byte(nil), req.ResultPayloadJson...)
|
||||
s.resultProof = req.Upload
|
||||
if s.resultError != nil {
|
||||
return nil, s.resultError
|
||||
}
|
||||
return &agentpb.ReportCallResultResponse{Receipt: &agentpb.OperationReceipt{FactId: "result-original", Result: agentpb.ResultCode_RESULT_CODE_ACCEPTED}}, nil
|
||||
}
|
||||
|
||||
func testRecordingDelivery(stub *recordingDeliveryRPC) *RecordingDelivery {
|
||||
return &RecordingDelivery{Call: RecordingClient{
|
||||
Client: stub, DispatcherID: "dispatcher-mock", TenantID: 1001, SourceEventID: "call-mock",
|
||||
Session: func(context.Context) (*agentpb.RequestMeta, error) {
|
||||
return &agentpb.RequestMeta{AgentId: "agent-mock", CellId: "cell-mock", BootId: "boot-mock", DispatcherEpoch: "epoch-mock", SessionGeneration: 1}, nil
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryNoRecordingReportsOnlyAfterConfirmedEnd(t *testing.T) {
|
||||
stub := &recordingDeliveryRPC{}
|
||||
delivery := testRecordingDelivery(stub)
|
||||
payload := []byte(`{"task_id":"task-asr","caller_profile_id":"caller-mock","callee":"15003164745","trunk_id":"trunk-mock","started_at":"2026-09-21T01:30:00Z","ended_at":"2026-09-21T01:30:05Z","duration_ms":5000,"outcome":"no_answer","reason_code":480,"reason_message":"no answer","transcript":[],"opt_out":false,"recording":{}}`)
|
||||
if err := delivery.Complete(context.Background(), CompletedRecording{ResultPayload: payload}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(stub.calls, []string{"end", "result"}) || stub.resultProof != nil {
|
||||
t.Fatalf("no-recording path requested upload or reported a fake PUT: calls=%v proof=%v", stub.calls, stub.resultProof)
|
||||
}
|
||||
var result struct {
|
||||
Recording map[string]any `json:"recording"`
|
||||
}
|
||||
if err := json.Unmarshal(stub.resultBody, &result); err != nil || len(result.Recording) != 0 {
|
||||
t.Fatalf("no-answer result invented a recording state: recording=%v err=%v", result.Recording, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryCannotReturnResultWhenEndNotConfirmed(t *testing.T) {
|
||||
stub := &recordingDeliveryRPC{endError: errors.New("injected Dispatcher persistence failure")}
|
||||
delivery := testRecordingDelivery(stub)
|
||||
if err := delivery.Complete(context.Background(), CompletedRecording{ResultPayload: []byte(`{"recording":{}}`)}); err == nil {
|
||||
t.Fatal("failed call-end acknowledgment was treated as a completed result")
|
||||
}
|
||||
if !reflect.DeepEqual(stub.calls, []string{"end"}) || len(stub.resultBody) != 0 {
|
||||
t.Fatalf("unconfirmed call end escaped into a final result: calls=%v", stub.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func testDirectRecordingDelivery(t *testing.T, putStatus int) (*RecordingDelivery, *recordingDeliveryRPC, CompletedRecording, *atomic.Int32, string) {
|
||||
t.Helper()
|
||||
wav, durationMS, err := media.EncodeMonoWAV(bytes.Repeat([]byte{1, 0}, 320), 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var puts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
puts.Add(1)
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, int64(len(wav)+1)))
|
||||
if err != nil || r.Method != http.MethodPut || !bytes.Equal(body, wav) {
|
||||
t.Errorf("OSS received a different or incomplete recording: method=%s size=%d err=%v", r.Method, len(body), err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(putStatus)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
sum := sha256.Sum256(wav)
|
||||
stub := &recordingDeliveryRPC{grant: &agentpb.UploadGrant{
|
||||
UploadId: "upload-mock", Bucket: "bucket-mock", ObjectKey: "mock/recording.wav",
|
||||
TargetUrl: server.URL + "/upload", ExpiresAtUnixMs: time.Now().Add(time.Minute).UnixMilli(),
|
||||
MaxBytes: int64(len(wav)), RequiredChecksumSha256: hex.EncodeToString(sum[:]),
|
||||
}}
|
||||
delivery := testRecordingDelivery(stub)
|
||||
root := t.TempDir()
|
||||
if err := os.Chmod(root, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivery.Recovery = &RecordingRecovery{Root: root, Upload: UploadClient{AllowInsecureHTTP: true}}
|
||||
completed := CompletedRecording{
|
||||
ResultPayload: []byte(`{"task_id":"task-asr","caller_profile_id":"caller-mock","callee":"15003164745","trunk_id":"trunk-mock","started_at":"2026-09-21T01:30:00Z","ended_at":"2026-09-21T01:30:20Z","duration_ms":20000,"outcome":"answered","reason_code":null,"reason_message":"answered","transcript":[],"opt_out":false,"recording":{}}`),
|
||||
Expected: true, RecordingID: "recording-mock", UploadID: "upload-mock", WAV: wav, DurationMS: durationMS,
|
||||
}
|
||||
return delivery, stub, completed, &puts, root
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryDirectPUTThenOneResultWithoutBusinessFiles(t *testing.T) {
|
||||
delivery, stub, completed, puts, root := testDirectRecordingDelivery(t, http.StatusCreated)
|
||||
if err := delivery.Complete(context.Background(), completed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(stub.calls, []string{"end", "grant", "result"}) || puts.Load() != 1 || stub.resultProof == nil || stub.resultProof.PutStatusCode != http.StatusCreated {
|
||||
t.Fatalf("wrong upload/result sequence: calls=%v puts=%d proof=%+v", stub.calls, puts.Load(), stub.resultProof)
|
||||
}
|
||||
var result struct {
|
||||
Recording struct {
|
||||
Status string `json:"status"`
|
||||
Bucket string `json:"bucket"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
Format string `json:"format"`
|
||||
ChecksumSHA256 string `json:"checksum_sha256"`
|
||||
Channels int `json:"channels"`
|
||||
SampleRateHz int `json:"sample_rate_hz"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
} `json:"recording"`
|
||||
}
|
||||
if err := json.Unmarshal(stub.resultBody, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(completed.WAV)
|
||||
if result.Recording.Status != "uploaded" || result.Recording.Bucket != stub.grant.Bucket || result.Recording.ObjectKey != stub.grant.ObjectKey || result.Recording.Format != "wav" || result.Recording.Channels != 1 || result.Recording.SampleRateHz != 16000 || result.Recording.DurationMS != completed.DurationMS || result.Recording.SizeBytes != int64(len(completed.WAV)) || result.Recording.ChecksumSHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("final result does not describe actual OSS upload: %+v", result.Recording)
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("normal successful upload created business files: entries=%d err=%v", len(entries), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryFailedPUTPersistsOriginalAndNeverRetriesImplicitly(t *testing.T) {
|
||||
delivery, stub, completed, puts, _ := testDirectRecordingDelivery(t, http.StatusServiceUnavailable)
|
||||
var rejected *UploadHTTPError
|
||||
uploadErr := delivery.Complete(context.Background(), completed)
|
||||
if !errors.As(uploadErr, &rejected) || rejected.StatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("OSS explicit failure not preserved: %v", uploadErr)
|
||||
}
|
||||
entry, err := delivery.Recovery.Load(stub.grant.Bucket, stub.grant.ObjectKey)
|
||||
var saved struct {
|
||||
Recording struct {
|
||||
Status string `json:"status"`
|
||||
} `json:"recording"`
|
||||
}
|
||||
if err != nil || entry.State != "retry_pending" || json.Unmarshal(entry.ResultPayload, &saved) != nil || saved.Recording.Status != "uploaded" || len(stub.resultBody) != 0 {
|
||||
t.Fatalf("failed PUT was not privately persisted without a false report: state=%s err=%v initial=%v", entry.State, err, uploadErr)
|
||||
}
|
||||
if !reflect.DeepEqual(stub.calls, []string{"end", "grant"}) || puts.Load() != 1 {
|
||||
t.Fatalf("failed upload was treated as a final result: calls=%v puts=%d", stub.calls, puts.Load())
|
||||
}
|
||||
if err := delivery.Complete(context.Background(), completed); !errors.Is(err, ErrRecordingRetryManaged) || puts.Load() != 1 {
|
||||
t.Fatalf("failed PUT was automatically repeated: err=%v puts=%d", err, puts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingDeliveryConfirmedPUTReportRetryDoesNotRepeatPUT(t *testing.T) {
|
||||
delivery, stub, completed, puts, root := testDirectRecordingDelivery(t, http.StatusOK)
|
||||
stub.resultError = errors.New("injected MQ/Dispatcher result failure")
|
||||
if err := delivery.Complete(context.Background(), completed); err == nil || puts.Load() != 1 {
|
||||
t.Fatalf("failed result report should retain confirmed PUT in memory: err=%v puts=%d", err, puts.Load())
|
||||
}
|
||||
stub.resultError = nil
|
||||
if err := delivery.Complete(context.Background(), completed); !errors.Is(err, ErrRecordingRetryManaged) || puts.Load() != 1 {
|
||||
t.Fatalf("confirmed PUT was implicitly repeated: err=%v puts=%d", err, puts.Load())
|
||||
}
|
||||
if err := delivery.RetryResult(context.Background()); err != nil || puts.Load() != 1 {
|
||||
t.Fatalf("result-only retry repeated OSS PUT: err=%v puts=%d", err, puts.Load())
|
||||
}
|
||||
if !reflect.DeepEqual(stub.calls, []string{"end", "grant", "result", "result"}) {
|
||||
t.Fatalf("confirmed PUT was not retried at the original result identity: %v", stub.calls)
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("result-only retry wrote successful upload to business files: entries=%d err=%v", len(entries), err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user