165 lines
7.4 KiB
Go
165 lines
7.4 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
|
"git.ipao.vip/rogee/go-sip/internal/agent"
|
|
"git.ipao.vip/rogee/go-sip/internal/config"
|
|
"git.ipao.vip/rogee/go-sip/internal/rpc"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
// Runtime recovery never repeats an OSS PUT with an unknown outcome. The
|
|
// recording state machine alone decides whether another attempt is due.
|
|
func startRealAgentRecovery(ctx context.Context, settings config.AgentEnvironment, dispatcher agentpb.AgentControlServiceClient, handler *rpc.Server) {
|
|
go func() {
|
|
ticker := time.NewTicker(10 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
if err := recoverRealAgentOnce(ctx, settings, dispatcher, handler); err != nil && ctx.Err() == nil {
|
|
slog.ErrorContext(ctx, "Agent private recovery needs inspection", "stage", "recovery_cycle_failed", "error", err)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func recoverRealAgentOnce(ctx context.Context, settings config.AgentEnvironment, dispatcher agentpb.AgentControlServiceClient, handler *rpc.Server) error {
|
|
if handler == nil || dispatcher == nil || strings.TrimSpace(settings.RecoveryRoot) == "" {
|
|
return errors.New("real Agent recovery needs a Dispatcher and private directory")
|
|
}
|
|
journal := agent.ResultJournal{Root: settings.RecoveryRoot}
|
|
call := func(id, dispatcherID string, tenantID int64) (agent.RecordingClient, error) {
|
|
if dispatcherID != settings.DispatcherID || tenantID <= 0 || id == "" {
|
|
return agent.RecordingClient{}, errors.New("original result has no approved Dispatcher or tenant binding")
|
|
}
|
|
return agent.RecordingClient{Client: dispatcher, DispatcherID: dispatcherID, TenantID: tenantID, SourceEventID: id,
|
|
Session: func(context.Context) (*agentpb.RequestMeta, error) { return handler.ActiveSessionMeta() }}, nil
|
|
}
|
|
resultErr := journal.RecoverWithEnd(ctx, func(ctx context.Context, entry agent.ResultEntry) error {
|
|
client, err := call(entry.SourceEventID, entry.DispatcherID, entry.TenantID)
|
|
if err == nil {
|
|
err = client.ReportEnded(ctx)
|
|
}
|
|
if err != nil {
|
|
logRecoveryError("call_end", entry.SourceEventID, err, "dispatcher_id", entry.DispatcherID, "tenant_id", entry.TenantID, "result_state", entry.State)
|
|
}
|
|
return err
|
|
}, func(ctx context.Context, entry agent.ResultEntry) error {
|
|
client, err := call(entry.SourceEventID, entry.DispatcherID, entry.TenantID)
|
|
if err == nil {
|
|
err = validateRestoredResult(entry.Payload, entry.Upload)
|
|
}
|
|
if err == nil {
|
|
_, err = client.ReportFinal(ctx, entry.Payload, entry.Upload)
|
|
}
|
|
if err != nil {
|
|
logRecoveryError("result", entry.SourceEventID, err, "dispatcher_id", entry.DispatcherID, "tenant_id", entry.TenantID, "result_state", entry.State)
|
|
}
|
|
return err
|
|
})
|
|
recovery := agent.RecordingRecovery{Root: settings.RecoveryRoot, Upload: agent.UploadClient{AllowedHosts: map[string]struct{}{strings.ToLower(settings.OSSAllowedHost): {}}}}
|
|
_, uploadErr := recovery.RecoverDue(ctx, func(ctx context.Context, entry agent.RecordingRecoveryEntry) (target agent.RecoveryTarget, targetErr error) {
|
|
defer func() {
|
|
if targetErr != nil {
|
|
logRecoveryError("upload_grant", entry.SourceEventID, targetErr, "dispatcher_id", entry.DispatcherID, "tenant_id", entry.TenantID, "recording_id", entry.RecordingID, "upload_id", entry.UploadID, "attempts", entry.Attempts, "next_attempt_at", entry.NextAttemptAt)
|
|
}
|
|
}()
|
|
client, err := call(entry.SourceEventID, entry.DispatcherID, entry.TenantID)
|
|
if err != nil {
|
|
return agent.RecoveryTarget{}, err
|
|
}
|
|
asset, err := restoredAsset(entry)
|
|
if err != nil {
|
|
return agent.RecoveryTarget{}, err
|
|
}
|
|
grant, err := client.RequestUpload(ctx, asset, entry.UploadID)
|
|
if err != nil {
|
|
return agent.RecoveryTarget{}, err
|
|
}
|
|
return agent.RecoveryTarget{Bucket: grant.Bucket, Grant: grant}, nil
|
|
}, func(ctx context.Context, entry agent.RecordingRecoveryEntry) (reportErr error) {
|
|
defer func() {
|
|
if reportErr != nil {
|
|
logRecoveryError("recording_result", entry.SourceEventID, reportErr, "dispatcher_id", entry.DispatcherID, "tenant_id", entry.TenantID, "recording_id", entry.RecordingID, "upload_id", entry.UploadID, "attempts", entry.Attempts)
|
|
}
|
|
}()
|
|
client, err := call(entry.SourceEventID, entry.DispatcherID, entry.TenantID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if entry.Uploaded == nil {
|
|
return errors.New("original recording has no observed PUT success")
|
|
}
|
|
observation := &agentpb.UploadObservation{UploadId: entry.UploadID, RecordingId: entry.RecordingID,
|
|
PutStatusCode: int32(entry.Uploaded.StatusCode), SizeBytes: entry.Uploaded.SizeBytes, ChecksumSha256: entry.Uploaded.SHA256}
|
|
if err := validateRestoredResult(entry.ResultPayload, observation); err != nil {
|
|
return err
|
|
}
|
|
_, err = client.ReportFinal(ctx, entry.ResultPayload, observation)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return journal.DiscardPrepared(entry.SourceEventID)
|
|
})
|
|
return errors.Join(resultErr, uploadErr)
|
|
}
|
|
|
|
func logRecoveryError(stage, id string, err error, identity ...any) {
|
|
digest := sha256.Sum256([]byte(id))
|
|
attrs := append([]any{"stage", stage, "event_id", id, "event_digest", fmt.Sprintf("%x", digest[:6]), "grpc_code", status.Code(err), "error", err}, identity...)
|
|
slog.Error("Agent recovery blocked", attrs...)
|
|
}
|
|
|
|
func restoredAsset(entry agent.RecordingRecoveryEntry) (*agentpb.AssetDescriptor, error) {
|
|
var payload struct {
|
|
Recording struct {
|
|
Status string `json:"status"`
|
|
Bucket string `json:"bucket"`
|
|
ObjectKey string `json:"object_key"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
ChecksumSHA256 string `json:"checksum_sha256"`
|
|
} `json:"recording"`
|
|
}
|
|
if err := json.Unmarshal(entry.ResultPayload, &payload); err != nil || payload.Recording.Status != "uploaded" || payload.Recording.Bucket != entry.Bucket || payload.Recording.ObjectKey != entry.ObjectKey || payload.Recording.SizeBytes != entry.SizeBytes || payload.Recording.ChecksumSHA256 != entry.SHA256 || payload.Recording.DurationMS <= 0 {
|
|
return nil, errors.New("original recording asset does not match its persisted result")
|
|
}
|
|
return &agentpb.AssetDescriptor{Kind: agentpb.AssetKind_ASSET_KIND_RECORDING, AssetId: entry.RecordingID,
|
|
CallId: entry.SourceEventID, ExecutionId: entry.SourceEventID, Format: "wav", Channels: 1, SampleRateHz: 16000,
|
|
DurationMs: payload.Recording.DurationMS, SizeBytes: entry.SizeBytes, ChecksumSha256: entry.SHA256}, nil
|
|
}
|
|
|
|
func validateRestoredResult(payload []byte, upload *agentpb.UploadObservation) error {
|
|
var result struct {
|
|
Recording struct {
|
|
Status string `json:"status"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
ChecksumSHA256 string `json:"checksum_sha256"`
|
|
} `json:"recording"`
|
|
}
|
|
if err := json.Unmarshal(payload, &result); err != nil {
|
|
return errors.New("original result is invalid")
|
|
}
|
|
if result.Recording.Status == "uploaded" {
|
|
if upload == nil || upload.GetPutStatusCode() < 200 || upload.GetPutStatusCode() >= 300 || result.Recording.SizeBytes != upload.GetSizeBytes() || result.Recording.ChecksumSHA256 != upload.GetChecksumSha256() || upload.GetUploadId() == "" || upload.GetRecordingId() == "" {
|
|
return errors.New("original recording has no matching confirmed upload")
|
|
}
|
|
} else if result.Recording.Status != "" || upload != nil {
|
|
return fmt.Errorf("original result has inconsistent recording status")
|
|
}
|
|
return nil
|
|
}
|