package agent import ( "bytes" "context" "errors" "testing" "time" agentpb "git.ipao.vip/rogee/go-sip/gen/agent" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) const verifiedSessionCutover = "verified Agent session generation changed" type cutoverRecordingRPC struct { agentpb.AgentControlServiceClient failure error lateFailure error alwaysReject bool sameGeneration bool changeIdentity string generation uint64 agentID string cellID string boot string epoch string metas []*agentpb.RequestMeta assets []*agentpb.AssetDescriptor uploadIDs []string payloads [][]byte } func (s *cutoverRecordingRPC) receive(meta *agentpb.RequestMeta) error { s.metas = append(s.metas, proto.Clone(meta).(*agentpb.RequestMeta)) if len(s.metas) == 1 { if !s.sameGeneration { s.generation++ } switch s.changeIdentity { case "agent": s.agentID = "different-agent" case "cell": s.cellID = "different-cell" case "boot": s.boot = "different-boot" case "epoch": s.epoch = "different-epoch" } return s.failure } if s.lateFailure != nil { return s.lateFailure } if s.alwaysReject { return s.failure } return nil } func (s *cutoverRecordingRPC) RequestRecordingUpload(_ context.Context, req *agentpb.RequestRecordingUploadRequest, _ ...grpc.CallOption) (*agentpb.RequestRecordingUploadResponse, error) { s.uploadIDs = append(s.uploadIDs, req.UploadId) s.assets = append(s.assets, proto.Clone(req.Asset).(*agentpb.AssetDescriptor)) if err := s.receive(req.Meta); err != nil { return nil, err } return &agentpb.RequestRecordingUploadResponse{Grant: &agentpb.UploadGrant{ UploadId: req.UploadId, Bucket: "mock-bucket", ObjectKey: "mock-key", MaxBytes: req.Asset.SizeBytes, RequiredChecksumSha256: req.Asset.ChecksumSha256, }}, nil } func (s *cutoverRecordingRPC) ReportCallEnded(_ context.Context, req *agentpb.ReportCallEndedRequest, _ ...grpc.CallOption) (*agentpb.ReportCallEndedResponse, error) { if err := s.receive(req.Meta); err != nil { return nil, err } return &agentpb.ReportCallEndedResponse{Receipt: &agentpb.OperationReceipt{FactId: req.SourceEventId, Result: agentpb.ResultCode_RESULT_CODE_APPLIED}}, nil } func (s *cutoverRecordingRPC) ReportCallResult(_ context.Context, req *agentpb.ReportCallResultRequest, _ ...grpc.CallOption) (*agentpb.ReportCallResultResponse, error) { s.payloads = append(s.payloads, bytes.Clone(req.ResultPayloadJson)) if err := s.receive(req.Meta); err != nil { return nil, err } return &agentpb.ReportCallResultResponse{Receipt: &agentpb.OperationReceipt{FactId: "result-fact", Result: agentpb.ResultCode_RESULT_CODE_ACCEPTED}}, nil } func newCutoverRecordingClient(s *cutoverRecordingRPC) RecordingClient { s.generation = 1 s.agentID = "agent-1" s.cellID = "cell-1" s.boot = "same-boot" s.epoch = "epoch-1" return RecordingClient{ Client: s, DispatcherID: "dispatcher-1", TenantID: 1, SourceEventID: "call-1", Session: func(context.Context) (*agentpb.RequestMeta, error) { return &agentpb.RequestMeta{AgentId: s.agentID, CellId: s.cellID, BootId: s.boot, DispatcherEpoch: s.epoch, SessionGeneration: s.generation}, nil }, } } func TestRecordingClientReReportsOnlyOriginalFactAfterVerifiedSessionCutover(t *testing.T) { for _, tc := range []struct { name string operation string report func(RecordingClient) error }{ {"upload grant", "call-1/upload/up-1", func(c RecordingClient) error { _, err := c.RequestUpload(context.Background(), &agentpb.AssetDescriptor{ Kind: agentpb.AssetKind_ASSET_KIND_RECORDING, ExecutionId: "call-1", CallId: "call-1", AssetId: "recording-1", SizeBytes: 4, ChecksumSha256: "digest", }, "up-1") return err }}, {"confirmed end", "call-1/ended", func(c RecordingClient) error { return c.ReportEnded(context.Background()) }}, {"final result", "call-1/result", func(c RecordingClient) error { _, err := c.ReportFinal(context.Background(), []byte(`{"call_id":"call-1"}`), nil) return err }}, } { t.Run(tc.name, func(t *testing.T) { transport := &cutoverRecordingRPC{failure: status.Error(codes.Unauthenticated, verifiedSessionCutover)} if err := tc.report(newCutoverRecordingClient(transport)); err != nil { t.Fatalf("original fact not delivered after session cutover: %v", err) } if len(transport.metas) != 2 { t.Fatalf("want exactly one bounded re-report, got %d requests", len(transport.metas)) } old, current := transport.metas[0], transport.metas[1] if old.SessionGeneration != 1 || current.SessionGeneration != 2 || old.AgentId != current.AgentId || old.CellId != current.CellId || old.BootId != current.BootId || old.DispatcherEpoch != current.DispatcherEpoch || old.OperationId != tc.operation || current.OperationId != tc.operation || old.IdempotencyKey != tc.operation || current.IdempotencyKey != tc.operation { t.Fatalf("session cutover changed fact identity or accepted an unbound Agent: first=%v second=%v", old, current) } if tc.name == "upload grant" && (len(transport.assets) != 2 || !proto.Equal(transport.assets[0], transport.assets[1]) || transport.uploadIDs[0] != "up-1" || transport.uploadIDs[1] != "up-1") { t.Fatal("upload grant request changed original asset/upload identity") } if tc.name == "final result" && (len(transport.payloads) != 2 || !bytes.Equal(transport.payloads[0], transport.payloads[1])) { t.Fatal("re-report changed original final result payload") } }) } } func TestRecordingClientDoesNotRetryUnverifiedOrUnknownOutcomes(t *testing.T) { for _, tc := range []struct { name string failure error identity string }{ {"permission denied", status.Error(codes.PermissionDenied, "inactive Agent"), ""}, {"timeout with unknown result", status.Error(codes.DeadlineExceeded, "result unknown"), ""}, {"unavailable with unknown result", status.Error(codes.Unavailable, "result unknown"), ""}, {"unverified unauthenticated", status.Error(codes.Unauthenticated, "mTLS peer missing"), ""}, {"different Agent after cutover", status.Error(codes.Unauthenticated, verifiedSessionCutover), "agent"}, {"different Cell after cutover", status.Error(codes.Unauthenticated, verifiedSessionCutover), "cell"}, {"different boot after cutover", status.Error(codes.Unauthenticated, verifiedSessionCutover), "boot"}, {"different epoch after cutover", status.Error(codes.Unauthenticated, verifiedSessionCutover), "epoch"}, } { t.Run(tc.name, func(t *testing.T) { transport := &cutoverRecordingRPC{failure: tc.failure, changeIdentity: tc.identity} err := newCutoverRecordingClient(transport).ReportEnded(context.Background()) if err == nil || len(transport.metas) != 1 { t.Fatalf("unauthorized or uncertain call end was re-reported: requests=%d error=%v", len(transport.metas), err) } }) } } func TestRecordingClientStopsWhenSecondOutcomeIsUnknown(t *testing.T) { transport := &cutoverRecordingRPC{ failure: status.Error(codes.Unauthenticated, verifiedSessionCutover), lateFailure: status.Error(codes.DeadlineExceeded, "result unknown after re-report"), } err := newCutoverRecordingClient(transport).ReportEnded(context.Background()) if status.Code(err) != codes.DeadlineExceeded || len(transport.metas) != 2 { t.Fatalf("unknown second outcome caused an unsafe retry: requests=%d error=%v", len(transport.metas), err) } } func TestRecordingClientAllowsOriginalGenerationUntilDispatcherCatchesUp(t *testing.T) { transport := &cutoverRecordingRPC{failure: status.Error(codes.Unauthenticated, verifiedSessionCutover), sameGeneration: true} if err := newCutoverRecordingClient(transport).ReportEnded(context.Background()); err != nil { t.Fatalf("original fact was not delivered after Dispatcher session caught up: %v", err) } if len(transport.metas) != 2 || transport.metas[0].SessionGeneration != transport.metas[1].SessionGeneration { t.Fatalf("re-report changed an otherwise still-active Agent generation: %d requests", len(transport.metas)) } } func TestRecordingClientSessionCutoverReReportIsBounded(t *testing.T) { transport := &cutoverRecordingRPC{failure: status.Error(codes.Unauthenticated, verifiedSessionCutover), alwaysReject: true} ctx, cancel := context.WithTimeout(context.Background(), 280*time.Millisecond) defer cancel() err := newCutoverRecordingClient(transport).ReportEnded(ctx) if !errors.Is(err, context.DeadlineExceeded) || len(transport.metas) < 2 || len(transport.metas) > 4 { t.Fatalf("re-report did not stop at caller deadline: requests=%d error=%v", len(transport.metas), err) } }