333 lines
17 KiB
Go
333 lines
17 KiB
Go
package rpc
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/google/uuid"
|
|
"strings"
|
|
"time"
|
|
|
|
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
|
"git.ipao.vip/rogee/go-sip/internal/contract"
|
|
ossclient "git.ipao.vip/rogee/go-sip/internal/oss"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
"git.ipao.vip/rogee/go-sip/internal/tenant"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
// DispatcherUploadServer is the Dispatcher-owned upload boundary. It keeps
|
|
// credentials and durable upload state on the Dispatcher and gives Agents only
|
|
// short-lived presigned PUT grants.
|
|
type DispatcherUploadServer struct {
|
|
agentpb.UnimplementedAgentControlServiceServer
|
|
|
|
store *store.Store
|
|
oss *ossclient.Client
|
|
now func() time.Time
|
|
localV3Authorize func(context.Context, *agentpb.RequestUploadRequest) error
|
|
localV3Complete func(context.Context, *agentpb.CompleteUploadRequest, store.UploadRecord) (bool, error)
|
|
requirePeer bool
|
|
maxAssetBytes int64
|
|
peerCertificateFingerprints map[string]struct{}
|
|
allowedAgentIDs map[string]struct{}
|
|
}
|
|
|
|
type DispatcherUploadOptions struct {
|
|
RequirePeer bool
|
|
PeerCertificateFingerprints map[string]struct{}
|
|
AllowedAgentIDs map[string]struct{}
|
|
// LocalV3Authorize binds an OSS grant to a confirmed Mock recording.
|
|
LocalV3Authorize func(context.Context, *agentpb.RequestUploadRequest) error
|
|
// LocalV3Complete is the sole Mock-mode upload outcome path. A successful
|
|
// local write is not delivery: true requires the final call.result to
|
|
// have reached the durable MQ queue with publisher confirmation.
|
|
LocalV3Complete func(context.Context, *agentpb.CompleteUploadRequest, store.UploadRecord) (bool, error)
|
|
}
|
|
|
|
func NewDispatcherUploadServer(st *store.Store, client *ossclient.Client, now func() time.Time, requirePeer bool) (*DispatcherUploadServer, error) {
|
|
return NewDispatcherUploadServerWithOptions(st, client, now, DispatcherUploadOptions{RequirePeer: requirePeer})
|
|
}
|
|
|
|
func NewDispatcherUploadServerWithOptions(st *store.Store, client *ossclient.Client, now func() time.Time, options DispatcherUploadOptions) (*DispatcherUploadServer, error) {
|
|
if st == nil {
|
|
return nil, errors.New("upload server requires Dispatcher store")
|
|
}
|
|
if client == nil {
|
|
return nil, errors.New("upload server requires OSS client")
|
|
}
|
|
if (options.LocalV3Authorize == nil) != (options.LocalV3Complete == nil) {
|
|
return nil, errors.New("local V3 upload authorization and completion must be configured together")
|
|
}
|
|
if now == nil {
|
|
now = time.Now
|
|
}
|
|
return &DispatcherUploadServer{
|
|
store: st, oss: client, now: now, requirePeer: options.RequirePeer,
|
|
localV3Authorize: options.LocalV3Authorize,
|
|
localV3Complete: options.LocalV3Complete,
|
|
maxAssetBytes: client.Config().MaxAssetBytes,
|
|
peerCertificateFingerprints: cloneStringSet(options.PeerCertificateFingerprints),
|
|
allowedAgentIDs: cloneStringSet(options.AllowedAgentIDs),
|
|
}, nil
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) RequestUpload(ctx context.Context, req *agentpb.RequestUploadRequest) (*agentpb.RequestUploadResponse, error) {
|
|
if err := s.validateRequest(ctx, req.GetMeta(), req.GetBinding(), req.GetAsset(), req.GetUploadId()); err != nil {
|
|
return nil, err
|
|
}
|
|
if req.Asset.SizeBytes <= 0 || req.Asset.ChecksumSha256 == "" {
|
|
return s.requestUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "asset size and SHA-256 are required", false), nil
|
|
}
|
|
if s.maxAssetBytes > 0 && req.Asset.SizeBytes > s.maxAssetBytes {
|
|
return s.requestUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED, "asset exceeds Dispatcher OSS limit", false), nil
|
|
}
|
|
if s.localV3Authorize != nil {
|
|
if err := s.localV3Authorize(ctx, req); err != nil {
|
|
return nil, status.Errorf(codes.FailedPrecondition, "local V3 recording grant refused: %v", err)
|
|
}
|
|
}
|
|
|
|
record, err := s.store.LoadUpload(req.UploadId)
|
|
if err == nil {
|
|
binding, asset, _, decodeErr := decodeUploadRecord(record)
|
|
if decodeErr != nil {
|
|
return nil, status.Errorf(codes.Internal, "decode durable upload %q: %v", req.UploadId, decodeErr)
|
|
}
|
|
if !proto.Equal(binding, req.Binding) || !proto.Equal(asset, req.Asset) {
|
|
return s.requestUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_ABORTED, "upload ID is bound to a different execution or asset", false), nil
|
|
}
|
|
if record.State == "uploaded" {
|
|
detail := "upload already reported; retry completion notification, not PUT"
|
|
if s.localV3Complete != nil {
|
|
detail = "upload already reported; retry final call.result completion, not PUT"
|
|
}
|
|
return nil, status.Error(codes.Unavailable, detail)
|
|
}
|
|
if record.State == "completed" {
|
|
detail := "upload notification delivered to MQ"
|
|
if s.localV3Complete != nil {
|
|
detail = "final call.result delivered to MQ"
|
|
}
|
|
return &agentpb.RequestUploadResponse{Receipt: s.receipt(req.Meta, agentpb.ResultCode_RESULT_CODE_ACCEPTED, detail, false), State: agentpb.UploadState_UPLOAD_STATE_COMPLETED}, nil
|
|
}
|
|
} else if !errors.Is(err, sql.ErrNoRows) {
|
|
return nil, status.Errorf(codes.Internal, "load upload: %v", err)
|
|
}
|
|
if req.Meta.OperationId == "" || req.Meta.IdempotencyKey == "" {
|
|
return nil, status.Error(codes.InvalidArgument, "upload request operation and idempotency identities are required")
|
|
}
|
|
requestRaw, err := (proto.MarshalOptions{Deterministic: true}).Marshal(req)
|
|
if err != nil {
|
|
return nil, status.Error(codes.InvalidArgument, "encode upload request")
|
|
}
|
|
requestSum := sha256.Sum256(requestRaw)
|
|
requestHash := hex.EncodeToString(requestSum[:])
|
|
previous, err := s.store.LoadUploadGrantRequest(req.UploadId, req.Meta.OperationId, requestHash)
|
|
if err == nil {
|
|
var original agentpb.UploadGrant
|
|
if err := proto.Unmarshal(previous, &original); err != nil {
|
|
return nil, status.Errorf(codes.Internal, "decode original grant: %v", err)
|
|
}
|
|
return &agentpb.RequestUploadResponse{Receipt: s.receipt(req.Meta, agentpb.ResultCode_RESULT_CODE_ACCEPTED, "original upload grant", false), Grant: &original, State: agentpb.UploadState_UPLOAD_STATE_REQUESTED}, nil
|
|
}
|
|
if errors.Is(err, store.ErrIdempotencyConflict) {
|
|
return nil, status.Error(codes.AlreadyExists, "upload request identity conflicts")
|
|
}
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return nil, status.Errorf(codes.Internal, "load original grant: %v", err)
|
|
}
|
|
objectKey := s.objectKey(req.Binding, req.Asset)
|
|
grant, err := s.oss.Grant(ctx, req.UploadId, objectKey, req.Asset.ChecksumSha256, req.Asset.SizeBytes, s.now())
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "create OSS upload grant: %v", err)
|
|
}
|
|
bindingRaw, err := (proto.MarshalOptions{Deterministic: true}).Marshal(req.Binding)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "encode upload binding: %v", err)
|
|
}
|
|
assetRaw, err := (proto.MarshalOptions{Deterministic: true}).Marshal(req.Asset)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "encode upload asset: %v", err)
|
|
}
|
|
grantRaw, err := proto.Marshal(grant)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "encode upload grant: %v", err)
|
|
}
|
|
persisted, err := s.store.IssueUploadGrant(store.UploadRecord{
|
|
UploadID: req.UploadId,
|
|
Binding: bindingRaw,
|
|
Asset: assetRaw,
|
|
Grant: grantRaw,
|
|
ObjectKey: objectKey,
|
|
Bucket: s.oss.Config().Bucket,
|
|
State: "granted",
|
|
CreatedAt: s.now().UTC(),
|
|
}, req.Meta.OperationId, requestHash)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.FailedPrecondition, "persist upload grant: %v", err)
|
|
}
|
|
if err := proto.Unmarshal(persisted, grant); err != nil {
|
|
return nil, status.Errorf(codes.Internal, "decode persisted grant: %v", err)
|
|
}
|
|
return &agentpb.RequestUploadResponse{
|
|
Receipt: s.receipt(req.Meta, agentpb.ResultCode_RESULT_CODE_ACCEPTED, "upload grant issued", false),
|
|
Grant: grant,
|
|
State: agentpb.UploadState_UPLOAD_STATE_REQUESTED,
|
|
}, nil
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) CompleteUpload(ctx context.Context, req *agentpb.CompleteUploadRequest) (*agentpb.CompleteUploadResponse, error) {
|
|
if err := s.validateRequest(ctx, req.GetMeta(), req.GetBinding(), req.GetAsset(), req.GetUploadId()); err != nil {
|
|
return nil, err
|
|
}
|
|
if req.UploadedSizeBytes <= 0 || req.UploadedChecksumSha256 == "" {
|
|
return s.completeUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "uploaded size and SHA-256 are required", false), nil
|
|
}
|
|
record, err := s.store.LoadUpload(req.UploadId)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return s.completeUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_NOT_FOUND, "upload not found", false), nil
|
|
}
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "load upload: %v", err)
|
|
}
|
|
binding, asset, grant, err := decodeUploadRecord(record)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "decode durable upload %q: %v", req.UploadId, err)
|
|
}
|
|
if !proto.Equal(binding, req.Binding) || !proto.Equal(asset, req.Asset) {
|
|
return s.completeUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_ABORTED, "upload completion binding does not match request", false), nil
|
|
}
|
|
if asset.SizeBytes != req.UploadedSizeBytes || !strings.EqualFold(asset.ChecksumSha256, req.UploadedChecksumSha256) || req.UploadedSizeBytes > grant.MaxBytes {
|
|
return s.completeUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "uploaded asset does not match grant", false), nil
|
|
}
|
|
if record.State == "completed" && s.localV3Complete == nil {
|
|
return &agentpb.CompleteUploadResponse{Receipt: s.receipt(req.Meta, agentpb.ResultCode_RESULT_CODE_ACCEPTED, "upload notification delivered to MQ", false), State: agentpb.UploadState_UPLOAD_STATE_COMPLETED}, nil
|
|
}
|
|
if record.State == "uploaded" && s.localV3Complete == nil {
|
|
return nil, status.Error(codes.Unavailable, "upload fact retained; original notification awaits MQ delivery")
|
|
}
|
|
if asset.Kind != agentpb.AssetKind_ASSET_KIND_RECORDING {
|
|
return s.completeUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "only recording upload facts are supported", false), nil
|
|
}
|
|
if s.localV3Complete != nil {
|
|
if record.State != "granted" && record.State != "uploaded" && record.State != "completed" {
|
|
return s.completeUploadFailure(req.Meta, agentpb.FailureCode_FAILURE_CODE_ABORTED, "upload already closed without a completed asset", false), nil
|
|
}
|
|
delivered, err := s.localV3Complete(ctx, req, record)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.FailedPrecondition, "persist local V3 upload result: %v", err)
|
|
}
|
|
if !delivered {
|
|
return nil, status.Error(codes.Unavailable, "upload fact retained; final call.result awaits MQ delivery")
|
|
}
|
|
return &agentpb.CompleteUploadResponse{Receipt: s.receipt(req.Meta, agentpb.ResultCode_RESULT_CODE_ACCEPTED, "final call.result delivered to MQ", false), State: agentpb.UploadState_UPLOAD_STATE_COMPLETED}, nil
|
|
}
|
|
dispatcherID, err := s.store.DispatcherID()
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.FailedPrecondition, "upload requires bound Dispatcher identity: %v", err)
|
|
}
|
|
route, err := tenant.NewDispatcherRoute(dispatcherID, binding.TenantKey)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.InvalidArgument, "upload notification route: %v", err)
|
|
}
|
|
uploadedAt := s.now().UTC()
|
|
eventID := uuid.NewSHA1(uuid.NameSpaceURL, []byte("recording.uploaded:"+dispatcherID+":"+req.UploadId)).String()
|
|
event, err := (contract.EventBuilder{
|
|
EventType: "recording.uploaded", TenantID: binding.TenantId, TenantKey: binding.TenantKey, TraceID: req.Meta.TraceId,
|
|
Aggregate: "recording", AggregateID: asset.AssetId, Version: 1,
|
|
Payload: map[string]any{"call_id": asset.CallId, "recording_id": asset.AssetId, "upload_id": req.UploadId,
|
|
"bucket": record.Bucket, "object_key": record.ObjectKey, "format": asset.Format, "channels": asset.Channels,
|
|
"sample_rate_hz": asset.SampleRateHz, "duration_ms": asset.DurationMs, "size_bytes": asset.SizeBytes, "checksum_sha256": asset.ChecksumSha256},
|
|
}).MarshalMQ(dispatcherID, uploadedAt, eventID)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "encode upload notification: %v", err)
|
|
}
|
|
if err := s.store.RecordUploadNotification(req.UploadId, eventID, binding.TenantKey, route.OutboundKey, event, uploadedAt); err != nil {
|
|
return nil, status.Errorf(codes.Internal, "persist upload fact and notification: %v", err)
|
|
}
|
|
return nil, status.Error(codes.Unavailable, "upload fact retained; original notification awaits MQ delivery")
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) validateRequest(ctx context.Context, meta *agentpb.RequestMeta, binding *agentpb.ExecutionBinding, asset *agentpb.AssetDescriptor, uploadID string) error {
|
|
if meta == nil || binding == nil || asset == nil {
|
|
return status.Error(codes.InvalidArgument, "request metadata, binding and asset are required")
|
|
}
|
|
if meta.AgentId == "" || meta.CellId == "" || meta.OperationId == "" || meta.IdempotencyKey == "" {
|
|
return status.Error(codes.InvalidArgument, "agent, Cell, operation and idempotency metadata are required")
|
|
}
|
|
if binding.ExecutionId == "" || binding.TaskId == "" || binding.TenantId == "" || binding.TenantKey == "" {
|
|
return status.Error(codes.InvalidArgument, "complete execution binding is required")
|
|
}
|
|
if err := contract.ValidateTenantKey(binding.TenantKey); err != nil {
|
|
return status.Errorf(codes.InvalidArgument, "invalid tenant key: %v", err)
|
|
}
|
|
if asset.AssetId == "" || uploadID == "" {
|
|
return status.Error(codes.InvalidArgument, "asset ID and upload ID are required")
|
|
}
|
|
return validateDispatcherPeer(ctx, meta, s.requirePeer, s.peerCertificateFingerprints, s.allowedAgentIDs)
|
|
}
|
|
|
|
func cloneStringSet(source map[string]struct{}) map[string]struct{} {
|
|
if len(source) == 0 {
|
|
return nil
|
|
}
|
|
result := make(map[string]struct{}, len(source))
|
|
for value := range source {
|
|
result[value] = struct{}{}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) objectKey(binding *agentpb.ExecutionBinding, asset *agentpb.AssetDescriptor) string {
|
|
prefix := strings.Trim(s.oss.Config().KeyPrefix, "/")
|
|
input := binding.TenantKey + "\x00" + binding.ExecutionId + "\x00" + asset.AssetId
|
|
digest := sha256.Sum256([]byte(input))
|
|
name := hex.EncodeToString(digest[:])
|
|
if prefix == "" {
|
|
return name
|
|
}
|
|
return prefix + "/" + name
|
|
}
|
|
|
|
func decodeUploadRecord(record store.UploadRecord) (*agentpb.ExecutionBinding, *agentpb.AssetDescriptor, *agentpb.UploadGrant, error) {
|
|
binding := &agentpb.ExecutionBinding{}
|
|
asset := &agentpb.AssetDescriptor{}
|
|
grant := &agentpb.UploadGrant{}
|
|
if err := proto.Unmarshal(record.Binding, binding); err != nil {
|
|
return nil, nil, nil, fmt.Errorf("binding: %w", err)
|
|
}
|
|
if err := proto.Unmarshal(record.Asset, asset); err != nil {
|
|
return nil, nil, nil, fmt.Errorf("asset: %w", err)
|
|
}
|
|
if err := proto.Unmarshal(record.Grant, grant); err != nil {
|
|
return nil, nil, nil, fmt.Errorf("grant: %w", err)
|
|
}
|
|
return binding, asset, grant, nil
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) responseMeta(meta *agentpb.RequestMeta) *agentpb.ResponseMeta {
|
|
return &agentpb.ResponseMeta{ProtocolVersion: meta.ProtocolVersion, RequestId: meta.RequestId, TraceId: meta.TraceId, OperationId: meta.OperationId, ObservedAtUnixMs: s.now().UnixMilli(), DispatcherEpoch: meta.DispatcherEpoch, AgentId: meta.AgentId, CellId: meta.CellId, BootId: meta.BootId, SessionGeneration: meta.SessionGeneration}
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) receipt(meta *agentpb.RequestMeta, result agentpb.ResultCode, detail string, retryable bool) *agentpb.OperationReceipt {
|
|
receipt := &agentpb.OperationReceipt{Meta: s.responseMeta(meta), Result: result, AcceptedAtUnixMs: s.now().UnixMilli()}
|
|
if detail != "" {
|
|
receipt.Failure = &agentpb.Failure{Code: agentpb.FailureCode_FAILURE_CODE_UNSPECIFIED, Detail: detail, Retryable: retryable}
|
|
}
|
|
return receipt
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) requestUploadFailure(meta *agentpb.RequestMeta, code agentpb.FailureCode, detail string, retryable bool) *agentpb.RequestUploadResponse {
|
|
return &agentpb.RequestUploadResponse{Receipt: &agentpb.OperationReceipt{Meta: s.responseMeta(meta), Result: agentpb.ResultCode_RESULT_CODE_REJECTED, Failure: &agentpb.Failure{Code: code, Detail: detail, Retryable: retryable}, AcceptedAtUnixMs: s.now().UnixMilli()}, State: agentpb.UploadState_UPLOAD_STATE_FAILED}
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) completeUploadFailure(meta *agentpb.RequestMeta, code agentpb.FailureCode, detail string, retryable bool) *agentpb.CompleteUploadResponse {
|
|
return &agentpb.CompleteUploadResponse{Receipt: &agentpb.OperationReceipt{Meta: s.responseMeta(meta), Result: agentpb.ResultCode_RESULT_CODE_REJECTED, Failure: &agentpb.Failure{Code: code, Detail: detail, Retryable: retryable}, AcceptedAtUnixMs: s.now().UnixMilli()}, State: agentpb.UploadState_UPLOAD_STATE_FAILED}
|
|
}
|