343 lines
16 KiB
Go
343 lines
16 KiB
Go
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/configread"
|
|
"git.ipao.vip/rogee/go-sip/internal/contract"
|
|
"git.ipao.vip/rogee/go-sip/internal/mq"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
"git.ipao.vip/rogee/go-sip/internal/tenant"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type taskControlStatusReader interface {
|
|
ReadTaskStatus(context.Context, string, string, string) (configread.TaskStatus, error)
|
|
}
|
|
|
|
type taskQueueController interface {
|
|
StartTask(context.Context, store.LocalTaskAssignment) error
|
|
StopTask(context.Context, store.LocalTaskAssignment) error
|
|
DrainTask(context.Context, store.LocalTaskAssignment) error
|
|
ApplyActiveCallPolicy(context.Context, store.LocalTaskAssignment, string, string) error
|
|
}
|
|
|
|
type localTaskControlProcessor struct {
|
|
dispatcher *Dispatcher
|
|
statusReader taskControlStatusReader
|
|
queueControl taskQueueController
|
|
controlMu sync.Mutex
|
|
}
|
|
|
|
type localTaskControlCommand struct {
|
|
SchemaVersion string `json:"schema_version"`
|
|
DispatcherID string `json:"dispatcher_id"`
|
|
TenantID string `json:"tenant_id"`
|
|
TenantKey string `json:"tenant_key"`
|
|
TraceID string `json:"trace_id"`
|
|
IssuedAt string `json:"issued_at"`
|
|
CommandType string `json:"command_type"`
|
|
Payload struct {
|
|
TaskID string `json:"task_id"`
|
|
Action string `json:"action"`
|
|
Reason string `json:"reason"`
|
|
} `json:"payload"`
|
|
}
|
|
|
|
type localTaskControlResultEvent struct {
|
|
SchemaVersion string `json:"schema_version"`
|
|
EventID string `json:"event_id"`
|
|
EventType string `json:"event_type"`
|
|
DispatcherID string `json:"dispatcher_id"`
|
|
TenantID string `json:"tenant_id"`
|
|
TenantKey string `json:"tenant_key"`
|
|
TraceID string `json:"trace_id"`
|
|
OccurredAt string `json:"occurred_at"`
|
|
AggregateType string `json:"aggregate_type"`
|
|
AggregateID string `json:"aggregate_id"`
|
|
AggregateVersion int64 `json:"aggregate_version"`
|
|
Payload localTaskControlResultPayload `json:"payload"`
|
|
}
|
|
|
|
type localTaskControlResultPayload struct {
|
|
CommandType string `json:"command_type"`
|
|
TaskID string `json:"task_id"`
|
|
Action string `json:"action"`
|
|
Status string `json:"status"`
|
|
ReasonCode string `json:"reason_code"`
|
|
TaskState string `json:"task_state"`
|
|
}
|
|
|
|
func newLocalTaskControlProcessor(d *Dispatcher, reader taskControlStatusReader, queues taskQueueController) *localTaskControlProcessor {
|
|
return &localTaskControlProcessor{dispatcher: d, statusReader: reader, queueControl: queues}
|
|
}
|
|
|
|
// Handle processes one SaaS-owned control delivery. State barriers and receipts
|
|
// are durable before the consumer ACKs the message.
|
|
func (p *localTaskControlProcessor) Handle(ctx context.Context, routingKey string, body []byte) error {
|
|
if p == nil || p.dispatcher == nil || p.statusReader == nil || p.queueControl == nil {
|
|
return errors.New("task control processor dependencies are required")
|
|
}
|
|
if err := contract.ValidateLocalTaskControlV04(body); err != nil {
|
|
return mq.Permanent(fmt.Errorf("invalid v0.4 task-control message: %w", err))
|
|
}
|
|
var command localTaskControlCommand
|
|
if err := json.Unmarshal(body, &command); err != nil {
|
|
return mq.Permanent(fmt.Errorf("decode local task-control message: %w", err))
|
|
}
|
|
if command.SchemaVersion != "task-control.v0.4-proposal" || command.CommandType != "task.control" {
|
|
return mq.Permanent(errors.New("message is not a local task-control command"))
|
|
}
|
|
if command.DispatcherID != p.dispatcher.dispatcherID || tenant.ValidateDispatcherID(command.DispatcherID) != nil {
|
|
return mq.Permanent(errors.New("task-control Dispatcher identity mismatch"))
|
|
}
|
|
if routingKey != "d."+p.dispatcher.dispatcherID+".control.in" {
|
|
return mq.Permanent(fmt.Errorf("task-control routing key mismatch: got %q", routingKey))
|
|
}
|
|
if !validLocalID(command.TraceID) || command.TenantID == "" || command.TenantKey == "" || !localTaskIDPattern.MatchString(command.Payload.TaskID) {
|
|
return mq.Permanent(errors.New("task-control identity fields are invalid"))
|
|
}
|
|
issuedAt, err := time.Parse(time.RFC3339Nano, command.IssuedAt)
|
|
if err != nil {
|
|
return mq.Permanent(fmt.Errorf("invalid task-control issued_at: %w", err))
|
|
}
|
|
|
|
// The single control consumer already serializes deliveries; this mutex also
|
|
// protects direct/test callers and preserves task-transition ordering.
|
|
p.controlMu.Lock()
|
|
defer p.controlMu.Unlock()
|
|
|
|
assignment, err := p.dispatcher.store.LocalTaskAssignment(p.dispatcher.dispatcherID, command.Payload.TaskID)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
ready, readinessErr := p.dispatcher.store.LocalTaskDiscoveryReadyV04(p.dispatcher.dispatcherID)
|
|
if readinessErr != nil {
|
|
return fmt.Errorf("read task ownership readiness: %w", readinessErr)
|
|
}
|
|
if !ready {
|
|
return fmt.Errorf("task ownership awaits complete snapshot: %w", store.ErrLocalDiscoveryUnavailable)
|
|
}
|
|
return p.persistUnknownTaskResult(command, "task_unavailable", "stopped")
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if assignment.TenantID != command.TenantID || assignment.TenantKey != command.TenantKey {
|
|
return mq.Permanent(errors.New("task-control tenant binding mismatch"))
|
|
}
|
|
|
|
now := p.dispatcher.now().UTC()
|
|
if now.Before(issuedAt) {
|
|
return p.persistControlResult(ctx, command, assignment, nil, "rejected", "command_not_yet_valid", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
|
|
switch command.Payload.Action {
|
|
case "pause":
|
|
return p.pause(ctx, command, assignment)
|
|
case "resume":
|
|
return p.resume(ctx, command, assignment)
|
|
case "stop":
|
|
return p.stop(ctx, command, assignment)
|
|
default:
|
|
return mq.Permanent(fmt.Errorf("unsupported task-control action %q", command.Payload.Action))
|
|
}
|
|
}
|
|
|
|
func (p *localTaskControlProcessor) pause(ctx context.Context, command localTaskControlCommand, assignment store.LocalTaskAssignment) error {
|
|
assignment, err := p.dispatcher.store.SetLocalTaskAdmissionBarrier(assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, "paused")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := p.queueControl.StopTask(ctx, assignment); err != nil {
|
|
return p.persistControlResult(ctx, command, assignment, nil, "rejected", "queue_unavailable", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
// An authenticated pause interrupts in-flight calls even if the SaaS
|
|
// status lookup subsequently fails; the durable barrier stays closed.
|
|
if err := p.queueControl.ApplyActiveCallPolicy(ctx, assignment, "pause", "hangup"); err != nil {
|
|
return p.persistControlResult(ctx, command, assignment, nil, "rejected", "active_call_policy_failed", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
status, statusErr := p.readAuthoritativeStatus(ctx, command)
|
|
if statusErr != nil {
|
|
return p.persistControlResult(ctx, command, assignment, nil, "rejected", "task_unavailable", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
if assignment.AdmissionState == "stopped" || assignment.AdmissionState == "finished" || assignment.AdmissionState == "removed" {
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "task_stopped", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
if status.Status != "paused" {
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "state_mismatch", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
return p.persistControlResult(ctx, command, assignment, &status, "applied", "applied", "paused")
|
|
}
|
|
|
|
func (p *localTaskControlProcessor) resume(ctx context.Context, command localTaskControlCommand, assignment store.LocalTaskAssignment) error {
|
|
status, statusErr := p.readAuthoritativeStatus(ctx, command)
|
|
if statusErr != nil {
|
|
return p.persistControlResult(ctx, command, assignment, nil, "rejected", "task_unavailable", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
if assignment.Removed || assignment.AdmissionState == "stopped" || assignment.AdmissionState == "finished" || assignment.AdmissionState == "removed" {
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "task_stopped", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
if status.Status != "running" {
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "state_mismatch", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
assignment, err := p.dispatcher.store.ResumeLocalTaskAdmission(assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, status.Status, status.TaskRevision)
|
|
if err != nil {
|
|
if errors.Is(err, store.ErrLocalTaskStopped) {
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "task_stopped", "stopped")
|
|
}
|
|
if errors.Is(err, store.ErrConfigRevisionConflict) || errors.Is(err, store.ErrConfigRevisionRollback) {
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "state_mismatch", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
return err
|
|
}
|
|
if err := p.queueControl.StartTask(ctx, assignment); err != nil {
|
|
assignment, barrierErr := p.dispatcher.store.SetLocalTaskAdmissionBarrier(assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, "paused")
|
|
if barrierErr != nil {
|
|
return errors.Join(err, barrierErr)
|
|
}
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "queue_unavailable", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
return p.persistControlResult(ctx, command, assignment, &status, "applied", "applied", "running")
|
|
}
|
|
|
|
func (p *localTaskControlProcessor) stop(ctx context.Context, command localTaskControlCommand, assignment store.LocalTaskAssignment) error {
|
|
assignment, err := p.dispatcher.store.SetLocalTaskAdmissionBarrier(assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, "stopped")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := p.queueControl.StopTask(ctx, assignment); err != nil {
|
|
return p.persistControlResult(ctx, command, assignment, nil, "rejected", "queue_unavailable", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
if err := p.queueControl.ApplyActiveCallPolicy(ctx, assignment, "stop", "hangup"); err != nil {
|
|
return p.persistControlResult(ctx, command, assignment, nil, "rejected", "active_call_policy_failed", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
status, statusErr := p.readAuthoritativeStatus(ctx, command)
|
|
if statusErr != nil {
|
|
return p.persistControlResult(ctx, command, assignment, nil, "rejected", "task_unavailable", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
if status.Status != "stopped" {
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "state_mismatch", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
if assignment.AdmissionState == "finished" || assignment.AdmissionState == "removed" {
|
|
return p.persistControlResult(ctx, command, assignment, &status, "rejected", "task_stopped", publicTaskAdmissionState(assignment.AdmissionState))
|
|
}
|
|
if err := p.dispatcher.store.RecordLocalTaskStatusObservation(
|
|
assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, status.Status, status.TaskRevision,
|
|
); err != nil {
|
|
return fmt.Errorf("persist authoritative stopped task status before queue drain: %w", err)
|
|
}
|
|
if err := p.queueControl.DrainTask(ctx, assignment); err != nil {
|
|
return err
|
|
}
|
|
return p.persistControlResult(ctx, command, assignment, &status, "applied", "applied", "stopped")
|
|
}
|
|
|
|
func (p *localTaskControlProcessor) readAuthoritativeStatus(ctx context.Context, command localTaskControlCommand) (configread.TaskStatus, error) {
|
|
status, err := p.statusReader.ReadTaskStatus(ctx, command.Payload.TaskID, command.TenantID, command.TenantKey)
|
|
if err != nil {
|
|
return configread.TaskStatus{}, err
|
|
}
|
|
if status.DispatcherID != command.DispatcherID || status.TaskID != command.Payload.TaskID || status.TenantID != command.TenantID ||
|
|
status.TenantKey != command.TenantKey || status.TaskRevision <= 0 {
|
|
return configread.TaskStatus{}, errors.New("authoritative task status identity mismatch")
|
|
}
|
|
return status, nil
|
|
}
|
|
|
|
func (p *localTaskControlProcessor) persistControlResult(_ context.Context, command localTaskControlCommand, assignment store.LocalTaskAssignment, observed *configread.TaskStatus, status, reason, admissionState string) error {
|
|
revision := assignment.TaskRevision
|
|
observedStatus := ""
|
|
if observed != nil {
|
|
observedStatus = observed.Status
|
|
revision = observed.TaskRevision
|
|
}
|
|
if revision <= 0 {
|
|
revision = 1
|
|
}
|
|
state := publicTaskAdmissionState(admissionState)
|
|
now := p.dispatcher.now().UTC()
|
|
event := localTaskControlResultEvent{
|
|
SchemaVersion: localCommandSchemaVersion,
|
|
EventID: uuid.NewString(),
|
|
EventType: "command.result",
|
|
DispatcherID: command.DispatcherID,
|
|
TenantID: command.TenantID,
|
|
TenantKey: command.TenantKey,
|
|
TraceID: command.TraceID,
|
|
OccurredAt: now.Format(time.RFC3339Nano),
|
|
AggregateType: "task",
|
|
AggregateID: command.Payload.TaskID,
|
|
AggregateVersion: revision,
|
|
Payload: localTaskControlResultPayload{
|
|
CommandType: "task.control", TaskID: command.Payload.TaskID,
|
|
Action: command.Payload.Action, Status: status, ReasonCode: reason, TaskState: state,
|
|
},
|
|
}
|
|
body, err := json.Marshal(event)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal task-control result: %w", err)
|
|
}
|
|
if err := contract.ValidateLocalCommandNext(body); err != nil {
|
|
return fmt.Errorf("validate task-control result: %w", err)
|
|
}
|
|
outbox := store.LocalEventRecord{
|
|
EventID: event.EventID, TenantKey: command.TenantKey, Exchange: mq.ResultsExchangeV3,
|
|
RoutingKey: "d." + command.DispatcherID + ".out", Body: body,
|
|
}
|
|
_, _, err = p.dispatcher.store.PersistLocalTaskControlResult(store.LocalTaskControlResultRecord{
|
|
DispatcherID: command.DispatcherID, TaskID: command.Payload.TaskID, TenantID: command.TenantID,
|
|
TenantKey: command.TenantKey, Action: command.Payload.Action, ResultStatus: status,
|
|
ReasonCode: reason, ObservedStatus: observedStatus, TaskRevision: func() int64 {
|
|
if observed != nil {
|
|
return observed.TaskRevision
|
|
}
|
|
return 0
|
|
}(), AdmissionState: state, Event: outbox,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("persist task-control result: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (p *localTaskControlProcessor) persistUnknownTaskResult(command localTaskControlCommand, reason, taskState string) error {
|
|
revision := int64(1)
|
|
now := p.dispatcher.now().UTC()
|
|
event := localTaskControlResultEvent{
|
|
SchemaVersion: localCommandSchemaVersion, EventID: uuid.NewString(), EventType: "command.result",
|
|
DispatcherID: command.DispatcherID, TenantID: command.TenantID, TenantKey: command.TenantKey,
|
|
TraceID: command.TraceID, OccurredAt: now.Format(time.RFC3339Nano), AggregateType: "task",
|
|
AggregateID: command.Payload.TaskID, AggregateVersion: revision,
|
|
Payload: localTaskControlResultPayload{CommandType: "task.control", TaskID: command.Payload.TaskID,
|
|
Action: command.Payload.Action, Status: "rejected", ReasonCode: reason, TaskState: taskState},
|
|
}
|
|
body, err := json.Marshal(event)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal unassigned task-control result: %w", err)
|
|
}
|
|
if err := contract.ValidateLocalCommandNext(body); err != nil {
|
|
return fmt.Errorf("validate unassigned task-control result: %w", err)
|
|
}
|
|
_, err = p.dispatcher.store.EnqueueLocalEvent(store.LocalEventRecord{
|
|
EventID: event.EventID, TenantKey: command.TenantKey, Exchange: mq.ResultsExchangeV3,
|
|
RoutingKey: "d." + command.DispatcherID + ".out", Body: body,
|
|
})
|
|
return err
|
|
}
|
|
|
|
func publicTaskAdmissionState(state string) string {
|
|
switch state {
|
|
case "running", "paused", "stopped", "finished":
|
|
return state
|
|
case "removed":
|
|
return "stopped"
|
|
default:
|
|
return "stopped"
|
|
}
|
|
}
|