297 lines
14 KiB
Go
297 lines
14 KiB
Go
package rpc
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
agentv1 "git.ipao.vip/rogee/go-sip/gen/agent/v1"
|
|
"git.ipao.vip/rogee/go-sip/internal/agent"
|
|
"git.ipao.vip/rogee/go-sip/internal/contract"
|
|
ossclient "git.ipao.vip/rogee/go-sip/internal/oss"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
"git.ipao.vip/rogee/go-sip/internal/tenant"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
// DispatcherUploadServer is the Dispatcher-owned upload boundary. It keeps
|
|
// credentials and durable upload state on the Dispatcher and gives Agents only
|
|
// short-lived presigned PUT grants.
|
|
type DispatcherUploadServer struct {
|
|
agentv1.UnimplementedAgentControlServiceServer
|
|
|
|
store *store.Store
|
|
oss *ossclient.Client
|
|
now func() time.Time
|
|
requirePeer bool
|
|
maxAssetBytes int64
|
|
peerCertificateFingerprints map[string]struct{}
|
|
allowedAgentIDs map[string]struct{}
|
|
}
|
|
|
|
type DispatcherUploadOptions struct {
|
|
RequirePeer bool
|
|
PeerCertificateFingerprints map[string]struct{}
|
|
AllowedAgentIDs map[string]struct{}
|
|
}
|
|
|
|
func NewDispatcherUploadServer(st *store.Store, client *ossclient.Client, now func() time.Time, requirePeer bool) (*DispatcherUploadServer, error) {
|
|
return NewDispatcherUploadServerWithOptions(st, client, now, DispatcherUploadOptions{RequirePeer: requirePeer})
|
|
}
|
|
|
|
func NewDispatcherUploadServerWithOptions(st *store.Store, client *ossclient.Client, now func() time.Time, options DispatcherUploadOptions) (*DispatcherUploadServer, error) {
|
|
if st == nil {
|
|
return nil, errors.New("upload server requires Dispatcher store")
|
|
}
|
|
if client == nil {
|
|
return nil, errors.New("upload server requires OSS client")
|
|
}
|
|
if now == nil {
|
|
now = time.Now
|
|
}
|
|
return &DispatcherUploadServer{
|
|
store: st, oss: client, now: now, requirePeer: options.RequirePeer,
|
|
maxAssetBytes: client.Config().MaxAssetBytes,
|
|
peerCertificateFingerprints: cloneStringSet(options.PeerCertificateFingerprints),
|
|
allowedAgentIDs: cloneStringSet(options.AllowedAgentIDs),
|
|
}, nil
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) RequestUpload(ctx context.Context, req *agentv1.RequestUploadRequest) (*agentv1.RequestUploadResponse, error) {
|
|
if err := s.validateRequest(ctx, req.GetMeta(), req.GetBinding(), req.GetAsset(), req.GetUploadId()); err != nil {
|
|
return nil, err
|
|
}
|
|
if req.Asset.SizeBytes <= 0 || req.Asset.ChecksumSha256 == "" {
|
|
return s.requestUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "asset size and SHA-256 are required", false), nil
|
|
}
|
|
if s.maxAssetBytes > 0 && req.Asset.SizeBytes > s.maxAssetBytes {
|
|
return s.requestUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED, "asset exceeds Dispatcher OSS limit", false), nil
|
|
}
|
|
|
|
record, err := s.store.LoadUpload(req.UploadId)
|
|
if err == nil {
|
|
binding, asset, grant, decodeErr := decodeUploadRecord(record)
|
|
if decodeErr != nil {
|
|
return nil, status.Errorf(codes.Internal, "decode durable upload %q: %v", req.UploadId, decodeErr)
|
|
}
|
|
if !proto.Equal(binding, req.Binding) || !proto.Equal(asset, req.Asset) {
|
|
return s.requestUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_ABORTED, "upload ID is bound to a different execution or asset", false), nil
|
|
}
|
|
detail := "duplicate upload request"
|
|
if record.State == "granted" && grant.ExpiresAtUnixMs <= s.now().UnixMilli() {
|
|
// This path is reached only when the caller explicitly requests a new
|
|
// token after the previous one expired. Agent upload flow does not
|
|
// renew or retry automatically.
|
|
replacement, grantErr := s.oss.Grant(ctx, req.UploadId, record.ObjectKey, asset.ChecksumSha256, asset.SizeBytes, s.now())
|
|
if grantErr != nil {
|
|
return nil, status.Errorf(codes.Internal, "issue replacement OSS upload grant: %v", grantErr)
|
|
}
|
|
replacementRaw, marshalErr := proto.Marshal(replacement)
|
|
if marshalErr != nil {
|
|
return nil, status.Errorf(codes.Internal, "encode replacement upload grant: %v", marshalErr)
|
|
}
|
|
if replaceErr := s.store.ReplaceUploadGrant(req.UploadId, record.ObjectKey, replacementRaw); replaceErr != nil {
|
|
return nil, status.Errorf(codes.Internal, "persist replacement upload grant: %v", replaceErr)
|
|
}
|
|
grant = replacement
|
|
detail = "expired upload grant replaced after explicit request"
|
|
}
|
|
return &agentv1.RequestUploadResponse{
|
|
Receipt: s.receipt(req.Meta, agentv1.ResultCode_RESULT_CODE_ACCEPTED, detail, false),
|
|
Grant: proto.Clone(grant).(*agentv1.UploadGrant),
|
|
State: uploadStateForRecord(record.State),
|
|
}, nil
|
|
} else if !errors.Is(err, sql.ErrNoRows) {
|
|
return nil, status.Errorf(codes.Internal, "load upload: %v", err)
|
|
}
|
|
|
|
objectKey := s.objectKey(req.Binding, req.Asset)
|
|
grant, err := s.oss.Grant(ctx, req.UploadId, objectKey, req.Asset.ChecksumSha256, req.Asset.SizeBytes, s.now())
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "create OSS upload grant: %v", err)
|
|
}
|
|
bindingRaw, err := proto.Marshal(req.Binding)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "encode upload binding: %v", err)
|
|
}
|
|
assetRaw, err := proto.Marshal(req.Asset)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "encode upload asset: %v", err)
|
|
}
|
|
grantRaw, err := proto.Marshal(grant)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "encode upload grant: %v", err)
|
|
}
|
|
if err := s.store.InsertUpload(store.UploadRecord{
|
|
UploadID: req.UploadId,
|
|
Binding: bindingRaw,
|
|
Asset: assetRaw,
|
|
Grant: grantRaw,
|
|
ObjectKey: objectKey,
|
|
State: "granted",
|
|
CreatedAt: s.now().UTC(),
|
|
}); err != nil {
|
|
// A concurrent duplicate is safe to reconcile by reading the durable row.
|
|
if existing, loadErr := s.store.LoadUpload(req.UploadId); loadErr == nil {
|
|
binding, asset, grant, decodeErr := decodeUploadRecord(existing)
|
|
if decodeErr == nil && proto.Equal(binding, req.Binding) && proto.Equal(asset, req.Asset) {
|
|
return &agentv1.RequestUploadResponse{Receipt: s.receipt(req.Meta, agentv1.ResultCode_RESULT_CODE_ACCEPTED, "duplicate upload request", false), Grant: grant, State: uploadStateForRecord(existing.State)}, nil
|
|
}
|
|
}
|
|
return nil, status.Errorf(codes.Internal, "persist upload grant: %v", err)
|
|
}
|
|
return &agentv1.RequestUploadResponse{
|
|
Receipt: s.receipt(req.Meta, agentv1.ResultCode_RESULT_CODE_ACCEPTED, "upload grant issued", false),
|
|
Grant: grant,
|
|
State: agentv1.UploadState_UPLOAD_STATE_REQUESTED,
|
|
}, nil
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) CompleteUpload(ctx context.Context, req *agentv1.CompleteUploadRequest) (*agentv1.CompleteUploadResponse, error) {
|
|
if err := s.validateRequest(ctx, req.GetMeta(), req.GetBinding(), req.GetAsset(), req.GetUploadId()); err != nil {
|
|
return nil, err
|
|
}
|
|
if req.UploadedSizeBytes <= 0 || req.UploadedChecksumSha256 == "" {
|
|
return s.completeUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "uploaded size and SHA-256 are required", false), nil
|
|
}
|
|
record, err := s.store.LoadUpload(req.UploadId)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return s.completeUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_NOT_FOUND, "upload not found", false), nil
|
|
}
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "load upload: %v", err)
|
|
}
|
|
binding, asset, grant, err := decodeUploadRecord(record)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "decode durable upload %q: %v", req.UploadId, err)
|
|
}
|
|
if !proto.Equal(binding, req.Binding) || !proto.Equal(asset, req.Asset) {
|
|
return s.completeUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_ABORTED, "upload completion binding does not match request", false), nil
|
|
}
|
|
if asset.SizeBytes != req.UploadedSizeBytes || !strings.EqualFold(asset.ChecksumSha256, req.UploadedChecksumSha256) || req.UploadedSizeBytes > grant.MaxBytes {
|
|
return s.completeUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "uploaded asset does not match grant", false), nil
|
|
}
|
|
if record.State == "completed" {
|
|
if record.OSSID == "" {
|
|
return nil, status.Error(codes.Internal, "completed upload has no OSS ID")
|
|
}
|
|
return &agentv1.CompleteUploadResponse{Receipt: s.receipt(req.Meta, agentv1.ResultCode_RESULT_CODE_ACCEPTED, "duplicate upload completion", false), State: agentv1.UploadState_UPLOAD_STATE_COMPLETED, OssId: record.OSSID}, nil
|
|
}
|
|
if grant.ExpiresAtUnixMs <= s.now().UnixMilli() {
|
|
return s.completeUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_FAILED_PRECONDITION, "upload grant has expired", true), nil
|
|
}
|
|
if err := s.oss.Verify(ctx, record.ObjectKey, req.UploadedChecksumSha256, req.UploadedSizeBytes); err != nil {
|
|
return s.completeUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_FAILED_PRECONDITION, fmt.Sprintf("verify OSS object: %v", err), true), nil
|
|
}
|
|
if asset.Kind != agentv1.AssetKind_ASSET_KIND_RECORDING {
|
|
return s.completeUploadFailure(req.Meta, agentv1.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "only recording assets can be marked recording.ready", false), nil
|
|
}
|
|
ossID := s.oss.ObjectID(record.ObjectKey)
|
|
completedAt := s.now().UTC()
|
|
eventID := "recording-ready-" + req.UploadId
|
|
event, err := (agent.EventWriter{
|
|
TenantID: binding.TenantId, TenantKey: binding.TenantKey, TraceID: req.Meta.TraceId,
|
|
}).RecordingReady(completedAt, eventID, asset.CallId, asset.AssetId, ossID, asset.Format, asset.Channels, asset.SampleRateHz, asset.DurationMs, asset.SizeBytes, asset.ChecksumSha256)
|
|
if err != nil {
|
|
return nil, status.Errorf(codes.Internal, "build verified recording event: %v", err)
|
|
}
|
|
if err := s.store.CompleteUploadAndOutbox(req.UploadId, ossID, completedAt, eventID, binding.TenantKey, tenant.EventExchange, "agent-call.recording.ready", event); err != nil {
|
|
return nil, status.Errorf(codes.Internal, "persist completed upload and event: %v", err)
|
|
}
|
|
return &agentv1.CompleteUploadResponse{Receipt: s.receipt(req.Meta, agentv1.ResultCode_RESULT_CODE_ACCEPTED, "OSS upload verified and recording.ready persisted", false), State: agentv1.UploadState_UPLOAD_STATE_COMPLETED, OssId: ossID}, nil
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) validateRequest(ctx context.Context, meta *agentv1.RequestMeta, binding *agentv1.ExecutionBinding, asset *agentv1.AssetDescriptor, uploadID string) error {
|
|
if meta == nil || binding == nil || asset == nil {
|
|
return status.Error(codes.InvalidArgument, "request metadata, binding and asset are required")
|
|
}
|
|
if meta.AgentId == "" || meta.CellId == "" || meta.OperationId == "" || meta.IdempotencyKey == "" {
|
|
return status.Error(codes.InvalidArgument, "agent, Cell, operation and idempotency metadata are required")
|
|
}
|
|
if binding.ExecutionId == "" || binding.TaskId == "" || binding.TenantId == "" || binding.TenantKey == "" {
|
|
return status.Error(codes.InvalidArgument, "complete execution binding is required")
|
|
}
|
|
if err := contract.ValidateTenantKey(binding.TenantKey); err != nil {
|
|
return status.Errorf(codes.InvalidArgument, "invalid tenant key: %v", err)
|
|
}
|
|
if asset.AssetId == "" || uploadID == "" {
|
|
return status.Error(codes.InvalidArgument, "asset ID and upload ID are required")
|
|
}
|
|
return validateDispatcherPeer(ctx, meta, s.requirePeer, s.peerCertificateFingerprints, s.allowedAgentIDs)
|
|
}
|
|
|
|
func cloneStringSet(source map[string]struct{}) map[string]struct{} {
|
|
if len(source) == 0 {
|
|
return nil
|
|
}
|
|
result := make(map[string]struct{}, len(source))
|
|
for value := range source {
|
|
result[value] = struct{}{}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) objectKey(binding *agentv1.ExecutionBinding, asset *agentv1.AssetDescriptor) string {
|
|
prefix := strings.Trim(s.oss.Config().KeyPrefix, "/")
|
|
input := binding.TenantKey + "\x00" + binding.ExecutionId + "\x00" + asset.AssetId
|
|
digest := sha256.Sum256([]byte(input))
|
|
name := hex.EncodeToString(digest[:])
|
|
if prefix == "" {
|
|
return name
|
|
}
|
|
return prefix + "/" + name
|
|
}
|
|
|
|
func decodeUploadRecord(record store.UploadRecord) (*agentv1.ExecutionBinding, *agentv1.AssetDescriptor, *agentv1.UploadGrant, error) {
|
|
binding := &agentv1.ExecutionBinding{}
|
|
asset := &agentv1.AssetDescriptor{}
|
|
grant := &agentv1.UploadGrant{}
|
|
if err := proto.Unmarshal(record.Binding, binding); err != nil {
|
|
return nil, nil, nil, fmt.Errorf("binding: %w", err)
|
|
}
|
|
if err := proto.Unmarshal(record.Asset, asset); err != nil {
|
|
return nil, nil, nil, fmt.Errorf("asset: %w", err)
|
|
}
|
|
if err := proto.Unmarshal(record.Grant, grant); err != nil {
|
|
return nil, nil, nil, fmt.Errorf("grant: %w", err)
|
|
}
|
|
return binding, asset, grant, nil
|
|
}
|
|
|
|
func uploadStateForRecord(state string) agentv1.UploadState {
|
|
if state == "completed" {
|
|
return agentv1.UploadState_UPLOAD_STATE_COMPLETED
|
|
}
|
|
if state == "failed" {
|
|
return agentv1.UploadState_UPLOAD_STATE_FAILED
|
|
}
|
|
return agentv1.UploadState_UPLOAD_STATE_REQUESTED
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) responseMeta(meta *agentv1.RequestMeta) *agentv1.ResponseMeta {
|
|
return &agentv1.ResponseMeta{ProtocolVersion: meta.ProtocolVersion, RequestId: meta.RequestId, TraceId: meta.TraceId, OperationId: meta.OperationId, ObservedAtUnixMs: s.now().UnixMilli(), DispatcherEpoch: meta.DispatcherEpoch, AgentId: meta.AgentId, CellId: meta.CellId, BootId: meta.BootId, SessionGeneration: meta.SessionGeneration}
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) receipt(meta *agentv1.RequestMeta, result agentv1.ResultCode, detail string, retryable bool) *agentv1.OperationReceipt {
|
|
receipt := &agentv1.OperationReceipt{Meta: s.responseMeta(meta), Result: result, AcceptedAtUnixMs: s.now().UnixMilli()}
|
|
if detail != "" {
|
|
receipt.Failure = &agentv1.Failure{Code: agentv1.FailureCode_FAILURE_CODE_UNSPECIFIED, Detail: detail, Retryable: retryable}
|
|
}
|
|
return receipt
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) requestUploadFailure(meta *agentv1.RequestMeta, code agentv1.FailureCode, detail string, retryable bool) *agentv1.RequestUploadResponse {
|
|
return &agentv1.RequestUploadResponse{Receipt: &agentv1.OperationReceipt{Meta: s.responseMeta(meta), Result: agentv1.ResultCode_RESULT_CODE_REJECTED, Failure: &agentv1.Failure{Code: code, Detail: detail, Retryable: retryable}, AcceptedAtUnixMs: s.now().UnixMilli()}, State: agentv1.UploadState_UPLOAD_STATE_FAILED}
|
|
}
|
|
|
|
func (s *DispatcherUploadServer) completeUploadFailure(meta *agentv1.RequestMeta, code agentv1.FailureCode, detail string, retryable bool) *agentv1.CompleteUploadResponse {
|
|
return &agentv1.CompleteUploadResponse{Receipt: &agentv1.OperationReceipt{Meta: s.responseMeta(meta), Result: agentv1.ResultCode_RESULT_CODE_REJECTED, Failure: &agentv1.Failure{Code: code, Detail: detail, Retryable: retryable}, AcceptedAtUnixMs: s.now().UnixMilli()}, State: agentv1.UploadState_UPLOAD_STATE_FAILED}
|
|
}
|