203 lines
8.5 KiB
Go
203 lines
8.5 KiB
Go
package agent
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
|
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
const verifiedSessionCutover = "verified Agent session generation changed"
|
|
|
|
type cutoverRecordingRPC struct {
|
|
agentpb.AgentControlServiceClient
|
|
failure error
|
|
lateFailure error
|
|
alwaysReject bool
|
|
sameGeneration bool
|
|
changeIdentity string
|
|
generation uint64
|
|
agentID string
|
|
cellID string
|
|
boot string
|
|
epoch string
|
|
metas []*agentpb.RequestMeta
|
|
assets []*agentpb.AssetDescriptor
|
|
uploadIDs []string
|
|
payloads [][]byte
|
|
}
|
|
|
|
func (s *cutoverRecordingRPC) receive(meta *agentpb.RequestMeta) error {
|
|
s.metas = append(s.metas, proto.Clone(meta).(*agentpb.RequestMeta))
|
|
if len(s.metas) == 1 {
|
|
if !s.sameGeneration {
|
|
s.generation++
|
|
}
|
|
switch s.changeIdentity {
|
|
case "agent":
|
|
s.agentID = "different-agent"
|
|
case "cell":
|
|
s.cellID = "different-cell"
|
|
case "boot":
|
|
s.boot = "different-boot"
|
|
case "epoch":
|
|
s.epoch = "different-epoch"
|
|
}
|
|
return s.failure
|
|
}
|
|
if s.lateFailure != nil {
|
|
return s.lateFailure
|
|
}
|
|
if s.alwaysReject {
|
|
return s.failure
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *cutoverRecordingRPC) RequestRecordingUpload(_ context.Context, req *agentpb.RequestRecordingUploadRequest, _ ...grpc.CallOption) (*agentpb.RequestRecordingUploadResponse, error) {
|
|
s.uploadIDs = append(s.uploadIDs, req.UploadId)
|
|
s.assets = append(s.assets, proto.Clone(req.Asset).(*agentpb.AssetDescriptor))
|
|
if err := s.receive(req.Meta); err != nil {
|
|
return nil, err
|
|
}
|
|
return &agentpb.RequestRecordingUploadResponse{Grant: &agentpb.UploadGrant{
|
|
UploadId: req.UploadId, Bucket: "mock-bucket", ObjectKey: "mock-key",
|
|
MaxBytes: req.Asset.SizeBytes, RequiredChecksumSha256: req.Asset.ChecksumSha256,
|
|
}}, nil
|
|
}
|
|
|
|
func (s *cutoverRecordingRPC) ReportCallEnded(_ context.Context, req *agentpb.ReportCallEndedRequest, _ ...grpc.CallOption) (*agentpb.ReportCallEndedResponse, error) {
|
|
if err := s.receive(req.Meta); err != nil {
|
|
return nil, err
|
|
}
|
|
return &agentpb.ReportCallEndedResponse{Receipt: &agentpb.OperationReceipt{FactId: req.SourceEventId, Result: agentpb.ResultCode_RESULT_CODE_APPLIED}}, nil
|
|
}
|
|
|
|
func (s *cutoverRecordingRPC) ReportCallResult(_ context.Context, req *agentpb.ReportCallResultRequest, _ ...grpc.CallOption) (*agentpb.ReportCallResultResponse, error) {
|
|
s.payloads = append(s.payloads, bytes.Clone(req.ResultPayloadJson))
|
|
if err := s.receive(req.Meta); err != nil {
|
|
return nil, err
|
|
}
|
|
return &agentpb.ReportCallResultResponse{Receipt: &agentpb.OperationReceipt{FactId: "result-fact", Result: agentpb.ResultCode_RESULT_CODE_ACCEPTED}}, nil
|
|
}
|
|
|
|
func newCutoverRecordingClient(s *cutoverRecordingRPC) RecordingClient {
|
|
s.generation = 1
|
|
s.agentID = "agent-1"
|
|
s.cellID = "cell-1"
|
|
s.boot = "same-boot"
|
|
s.epoch = "epoch-1"
|
|
return RecordingClient{
|
|
Client: s, DispatcherID: "dispatcher-1", TenantID: 1, SourceEventID: "call-1",
|
|
Session: func(context.Context) (*agentpb.RequestMeta, error) {
|
|
return &agentpb.RequestMeta{AgentId: s.agentID, CellId: s.cellID, BootId: s.boot, DispatcherEpoch: s.epoch, SessionGeneration: s.generation}, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestRecordingClientReReportsOnlyOriginalFactAfterVerifiedSessionCutover(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
operation string
|
|
report func(RecordingClient) error
|
|
}{
|
|
{"upload grant", "call-1/upload/up-1", func(c RecordingClient) error {
|
|
_, err := c.RequestUpload(context.Background(), &agentpb.AssetDescriptor{
|
|
Kind: agentpb.AssetKind_ASSET_KIND_RECORDING, ExecutionId: "call-1", CallId: "call-1", AssetId: "recording-1", SizeBytes: 4, ChecksumSha256: "digest",
|
|
}, "up-1")
|
|
return err
|
|
}},
|
|
{"confirmed end", "call-1/ended", func(c RecordingClient) error { return c.ReportEnded(context.Background()) }},
|
|
{"final result", "call-1/result", func(c RecordingClient) error {
|
|
_, err := c.ReportFinal(context.Background(), []byte(`{"call_id":"call-1"}`), nil)
|
|
return err
|
|
}},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
transport := &cutoverRecordingRPC{failure: status.Error(codes.Unauthenticated, verifiedSessionCutover)}
|
|
if err := tc.report(newCutoverRecordingClient(transport)); err != nil {
|
|
t.Fatalf("original fact not delivered after session cutover: %v", err)
|
|
}
|
|
if len(transport.metas) != 2 {
|
|
t.Fatalf("want exactly one bounded re-report, got %d requests", len(transport.metas))
|
|
}
|
|
old, current := transport.metas[0], transport.metas[1]
|
|
if old.SessionGeneration != 1 || current.SessionGeneration != 2 ||
|
|
old.AgentId != current.AgentId || old.CellId != current.CellId || old.BootId != current.BootId || old.DispatcherEpoch != current.DispatcherEpoch ||
|
|
old.OperationId != tc.operation || current.OperationId != tc.operation || old.IdempotencyKey != tc.operation || current.IdempotencyKey != tc.operation {
|
|
t.Fatalf("session cutover changed fact identity or accepted an unbound Agent: first=%v second=%v", old, current)
|
|
}
|
|
if tc.name == "upload grant" && (len(transport.assets) != 2 || !proto.Equal(transport.assets[0], transport.assets[1]) || transport.uploadIDs[0] != "up-1" || transport.uploadIDs[1] != "up-1") {
|
|
t.Fatal("upload grant request changed original asset/upload identity")
|
|
}
|
|
if tc.name == "final result" && (len(transport.payloads) != 2 || !bytes.Equal(transport.payloads[0], transport.payloads[1])) {
|
|
t.Fatal("re-report changed original final result payload")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRecordingClientDoesNotRetryUnverifiedOrUnknownOutcomes(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
failure error
|
|
identity string
|
|
}{
|
|
{"permission denied", status.Error(codes.PermissionDenied, "inactive Agent"), ""},
|
|
{"timeout with unknown result", status.Error(codes.DeadlineExceeded, "result unknown"), ""},
|
|
{"unavailable with unknown result", status.Error(codes.Unavailable, "result unknown"), ""},
|
|
{"unverified unauthenticated", status.Error(codes.Unauthenticated, "mTLS peer missing"), ""},
|
|
{"different Agent after cutover", status.Error(codes.Unauthenticated, verifiedSessionCutover), "agent"},
|
|
{"different Cell after cutover", status.Error(codes.Unauthenticated, verifiedSessionCutover), "cell"},
|
|
{"different boot after cutover", status.Error(codes.Unauthenticated, verifiedSessionCutover), "boot"},
|
|
{"different epoch after cutover", status.Error(codes.Unauthenticated, verifiedSessionCutover), "epoch"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
transport := &cutoverRecordingRPC{failure: tc.failure, changeIdentity: tc.identity}
|
|
err := newCutoverRecordingClient(transport).ReportEnded(context.Background())
|
|
if err == nil || len(transport.metas) != 1 {
|
|
t.Fatalf("unauthorized or uncertain call end was re-reported: requests=%d error=%v", len(transport.metas), err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRecordingClientStopsWhenSecondOutcomeIsUnknown(t *testing.T) {
|
|
transport := &cutoverRecordingRPC{
|
|
failure: status.Error(codes.Unauthenticated, verifiedSessionCutover),
|
|
lateFailure: status.Error(codes.DeadlineExceeded, "result unknown after re-report"),
|
|
}
|
|
err := newCutoverRecordingClient(transport).ReportEnded(context.Background())
|
|
if status.Code(err) != codes.DeadlineExceeded || len(transport.metas) != 2 {
|
|
t.Fatalf("unknown second outcome caused an unsafe retry: requests=%d error=%v", len(transport.metas), err)
|
|
}
|
|
}
|
|
|
|
func TestRecordingClientAllowsOriginalGenerationUntilDispatcherCatchesUp(t *testing.T) {
|
|
transport := &cutoverRecordingRPC{failure: status.Error(codes.Unauthenticated, verifiedSessionCutover), sameGeneration: true}
|
|
if err := newCutoverRecordingClient(transport).ReportEnded(context.Background()); err != nil {
|
|
t.Fatalf("original fact was not delivered after Dispatcher session caught up: %v", err)
|
|
}
|
|
if len(transport.metas) != 2 || transport.metas[0].SessionGeneration != transport.metas[1].SessionGeneration {
|
|
t.Fatalf("re-report changed an otherwise still-active Agent generation: %d requests", len(transport.metas))
|
|
}
|
|
}
|
|
|
|
func TestRecordingClientSessionCutoverReReportIsBounded(t *testing.T) {
|
|
transport := &cutoverRecordingRPC{failure: status.Error(codes.Unauthenticated, verifiedSessionCutover), alwaysReject: true}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 280*time.Millisecond)
|
|
defer cancel()
|
|
err := newCutoverRecordingClient(transport).ReportEnded(ctx)
|
|
if !errors.Is(err, context.DeadlineExceeded) || len(transport.metas) < 2 || len(transport.metas) > 4 {
|
|
t.Fatalf("re-report did not stop at caller deadline: requests=%d error=%v", len(transport.metas), err)
|
|
}
|
|
}
|