package agent import ( "context" "errors" "net/http" "os" "path/filepath" "testing" "time" agentpb "git.ipao.vip/rogee/go-sip/gen/agent" "github.com/google/uuid" "google.golang.org/protobuf/proto" ) type mockFailureRPC struct { requests []*agentpb.ReportExecutionEventRequest unavailable bool wrongReceipt bool } func (m *mockFailureRPC) ReportExecutionEvent(_ context.Context, request *agentpb.ReportExecutionEventRequest) (*agentpb.ReportExecutionEventResponse, error) { m.requests = append(m.requests, proto.Clone(request).(*agentpb.ReportExecutionEventRequest)) if m.unavailable { return nil, errors.New("Dispatcher unavailable before receipt") } factID := request.Fact.FactId if m.wrongReceipt { factID = uuid.NewString() } return &agentpb.ReportExecutionEventResponse{Receipt: &agentpb.OperationReceipt{ Result: agentpb.ResultCode_RESULT_CODE_ACCEPTED, FactId: factID, ContentSha256: request.Fact.ContentSha256, }}, nil } func TestMockUploadFailureReportReusesDurableFactAcrossActiveBoots(t *testing.T) { root := t.TempDir() observed := time.Date(2026, 9, 21, 2, 0, 0, 0, time.UTC) spool, err := NewSpool(root, func() time.Time { return observed }) if err != nil { t.Fatal(err) } binding := &agentpb.ExecutionBinding{TenantId: "tenant-a", TenantKey: "tenant-a", ExecutionId: "execution-a"} if err := spool.ClaimUpload(UploadAttempt{ UploadID: "upload-a", Identity: "identity-a", State: "attempted", RequestID: uuid.NewString(), Binding: binding, Asset: &agentpb.AssetDescriptor{AssetId: "recording-a"}, }); err != nil { t.Fatal(err) } active := &agentpb.RequestMeta{ ProtocolVersion: "agent.v1", AgentId: "agent-a", CellId: "cell-a", BootId: "boot-old", DispatcherEpoch: "epoch-a", SessionGeneration: 1, } remote := &mockFailureRPC{unavailable: true} known, err := spool.ReportMockUploadFailure(context.Background(), remote, active, "upload-a", &UploadHTTPError{StatusCode: http.StatusForbidden}) if !known || err == nil || len(remote.requests) != 1 { t.Fatalf("explicit 403 failure was not durably sent: known=%t requests=%d err=%v", known, len(remote.requests), err) } record, err := spool.LoadUploadAttempt("upload-a") if err != nil || record.FailureFact == nil || record.FailureDelivered { t.Fatalf("fact lost before receipt: record=%+v err=%v", record, err) } factID, digest, sourceBoot := record.FailureFact.FactId, record.FailureFact.ContentSha256, record.FailureFact.SourceBootId if sourceBoot != active.BootId || record.FailureFact.ObservedAtUnixMs != observed.UnixMilli() { t.Fatalf("fact lost source boot or observation clock: boot=%q observed=%d", sourceBoot, record.FailureFact.ObservedAtUnixMs) } restarted, err := NewSpool(root, nil) if err != nil { t.Fatal(err) } activeNew := proto.Clone(active).(*agentpb.RequestMeta) activeNew.BootId = "boot-new" activeNew.SessionGeneration = 2 remote.unavailable, remote.wrongReceipt = false, true if err := restarted.RecoverMockUploadFailures(context.Background(), remote, activeNew); err == nil { t.Fatal("unrelated accepted receipt closed a durable failure fact") } record, err = restarted.LoadUploadAttempt("upload-a") if err != nil || record.FailureDelivered { t.Fatalf("false receipt advanced failure state: record=%+v err=%v", record, err) } remote.wrongReceipt = false if err := restarted.RecoverMockUploadFailures(context.Background(), remote, activeNew); err != nil { t.Fatalf("failure fact not recovered after boot change: %v", err) } record, err = restarted.LoadUploadAttempt("upload-a") if err != nil || !record.FailureDelivered || record.State != "attempted" { t.Fatalf("accepted receipt did not close only the fact: record=%+v err=%v", record, err) } if len(remote.requests) != 3 { t.Fatalf("expected original, false receipt and recovery; got %d reports", len(remote.requests)) } for _, request := range remote.requests { if request.Fact.FactId != factID || request.Fact.ContentSha256 != digest || request.Fact.SourceBootId != sourceBoot { t.Fatalf("recovery generated a new failure identity: %+v", request.Fact) } } if remote.requests[2].Meta.BootId != activeNew.BootId || remote.requests[2].Meta.SessionGeneration != activeNew.SessionGeneration { t.Fatalf("recovery did not use the current active session: %+v", remote.requests[2].Meta) } if err := restarted.RecoverMockUploadFailures(context.Background(), remote, activeNew); err != nil || len(remote.requests) != 3 { t.Fatalf("acknowledged failure re-reported: requests=%d err=%v", len(remote.requests), err) } } func TestMockUploadFailureClassificationKeepsUncertainOutcomesUnknown(t *testing.T) { for _, tc := range []struct { name string err error code string }{ {"expired grant", ErrUploadGrantExpired, "upload_authorization_expired"}, {"invalid grant", ErrUploadGrantInvalid, "upload_authorization_failed"}, {"checksum mismatch", ErrUploadChecksumMismatch, "checksum_mismatch"}, {"explicit 400", &UploadHTTPError{StatusCode: http.StatusBadRequest}, "upload_failed"}, {"explicit 403", &UploadHTTPError{StatusCode: http.StatusForbidden}, "upload_authorization_failed"}, {"missing recording", os.ErrNotExist, "upload_failed"}, {"request timeout", &UploadHTTPError{StatusCode: http.StatusRequestTimeout}, ""}, {"throttled", &UploadHTTPError{StatusCode: http.StatusTooManyRequests}, ""}, {"server uncertain", &UploadHTTPError{StatusCode: http.StatusInternalServerError}, ""}, {"transport uncertain", errors.New("connection reset"), ""}, } { t.Run(tc.name, func(t *testing.T) { if got := MockUploadFailureCode(tc.err); got != tc.code { t.Fatalf("classification %q, want %q", got, tc.code) } }) } spool, err := NewSpool(t.TempDir(), nil) if err != nil { t.Fatal(err) } if err := spool.ClaimUpload(UploadAttempt{ UploadID: "unknown-a", Identity: "identity-a", State: "attempted", RequestID: uuid.NewString(), Binding: &agentpb.ExecutionBinding{TenantId: "tenant-a", TenantKey: "tenant-a", ExecutionId: "execution-a"}, Asset: &agentpb.AssetDescriptor{AssetId: "recording-a"}, }); err != nil { t.Fatal(err) } active := &agentpb.RequestMeta{ProtocolVersion: "agent.v1", AgentId: "agent-a", CellId: "cell-a", BootId: "boot-a", DispatcherEpoch: "epoch-a", SessionGeneration: 1} remote := &mockFailureRPC{} known, err := spool.ReportMockUploadFailure(context.Background(), remote, active, "unknown-a", &UploadHTTPError{StatusCode: 500}) if err != nil || known || len(remote.requests) != 0 { t.Fatalf("unknown PUT was reported as failed: known=%t reports=%d err=%v", known, len(remote.requests), err) } pending, err := spool.PendingUploadFailures() if err != nil || len(pending) != 0 { t.Fatalf("unknown PUT produced a terminal fact: pending=%d err=%v", len(pending), err) } if _, err := os.Stat(filepath.Join(spool.root, ".uploads", "unknown-a", "state.json")); err != nil { t.Fatal(err) } }