package dispatcher import ( "context" "database/sql" "errors" "fmt" "log/slog" "sync" agentpb "git.ipao.vip/rogee/go-sip/gen/agent" "git.ipao.vip/rogee/go-sip/internal/mq" "git.ipao.vip/rogee/go-sip/internal/store" ) type taskQueueConsumer interface { Wait(context.Context) error Stop(context.Context) error } type taskQueueBroker interface { StartPredeclaredConsumer(context.Context, string, mq.MessageHandler) (taskQueueConsumer, error) DrainPredeclared(context.Context, string) (int, error) DrainControlPredeclared(context.Context, string, mq.MessageHandler) (int, error) } type v3TaskBrokerAdapter struct{ broker *mq.V3Broker } func (a v3TaskBrokerAdapter) StartPredeclaredConsumer(ctx context.Context, queue string, handler mq.MessageHandler) (taskQueueConsumer, error) { return a.broker.StartPredeclaredConsumer(ctx, queue, handler) } func (a v3TaskBrokerAdapter) DrainPredeclared(ctx context.Context, queue string) (int, error) { return a.broker.DrainPredeclared(ctx, queue) } func (a v3TaskBrokerAdapter) DrainControlPredeclared(ctx context.Context, queue string, handler mq.MessageHandler) (int, error) { return a.broker.DrainControlPredeclared(ctx, queue, handler) } type v3TaskQueueController struct { dispatcher *Dispatcher broker taskQueueBroker taskController TaskController mockAuthorizedAgentID string mockAuthorizedAgents *AgentCoordinator mu sync.Mutex consumers map[string]taskQueueConsumer errors chan error } func newV3TaskQueueController(d *Dispatcher, broker *mq.V3Broker, taskController TaskController) *v3TaskQueueController { return newTaskQueueController(d, v3TaskBrokerAdapter{broker: broker}, taskController) } func newTaskQueueController(d *Dispatcher, broker taskQueueBroker, taskController TaskController) *v3TaskQueueController { return &v3TaskQueueController{ dispatcher: d, broker: broker, taskController: taskController, consumers: make(map[string]taskQueueConsumer), errors: make(chan error, 64), } } // EnableMockAuthorizedOrigination must run before any task consumer starts. // Non-mock modes never call this setter and cannot originate by this path. func (q *v3TaskQueueController) EnableMockAuthorizedOrigination(agentID string, agents *AgentCoordinator) error { if q == nil || agentID == "" || agents == nil { return errors.New("mock authorized origination requires one activated Agent") } q.mu.Lock() defer q.mu.Unlock() if len(q.consumers) != 0 || q.mockAuthorizedAgentID != "" { return errors.New("mock authorized origination must be configured once before consumption") } q.mockAuthorizedAgentID, q.mockAuthorizedAgents = agentID, agents return nil } func (q *v3TaskQueueController) handleTaskCommand(ctx context.Context, routingKey string, body []byte) error { if err := q.dispatcher.AcceptLocalV01Command(ctx, routingKey, body); err != nil { return err } if q.mockAuthorizedAgentID == "" { return nil } executionID, err := q.dispatcher.store.LocalPendingOriginationForCommand(body) if errors.Is(err, sql.ErrNoRows) { // Rejected command or already-issued/refused execution. return nil } if err != nil { return fmt.Errorf("read pending local execution after durable admission: %w", err) } result, dispatchErr := q.dispatcher.DispatchAndFinalizeAuthorizedMock(ctx, executionID, q.mockAuthorizedAgentID, q.mockAuthorizedAgents) decision, err := q.dispatcher.store.LocalOriginationDecision(executionID) if err != nil { return errors.Join(dispatchErr, fmt.Errorf("verify durable final dial decision before ACK: %w", err)) } if dispatchErr != nil { slog.Error("Mock Agent instruction refused or uncertain; no automatic retry", "execution_id", executionID, "decision", decision, "unknown", result.Unknown, "error", dispatchErr) } else if decision != "issued" { return fmt.Errorf("Agent applied execution %s without an issued dial decision", executionID) } // The command was durably admitted and its final decision was persisted. // An Agent error is never passed to MQ as a requeue/re-originate signal. return nil } func (q *v3TaskQueueController) StartTask(ctx context.Context, assignment store.LocalTaskAssignment) error { if q == nil || q.dispatcher == nil || q.broker == nil { return errors.New("v3 task queue controller is not configured") } q.mu.Lock() defer q.mu.Unlock() if _, exists := q.consumers[assignment.TaskID]; exists { return nil } current, err := q.dispatcher.store.LocalTaskAssignment(assignment.DispatcherID, assignment.TaskID) if err != nil { return err } if current.TenantID != assignment.TenantID || current.TenantKey != assignment.TenantKey { return store.ErrTenantBindingConflict } if current.Removed || current.AdmissionState != "running" || current.Status != "running" { return fmt.Errorf("task %s is not eligible for queue consumption", assignment.TaskID) } if current.Queue.QueueName == "" { return fmt.Errorf("task %s has no SaaS-owned queue name", assignment.TaskID) } consumer, err := q.broker.StartPredeclaredConsumer(ctx, current.Queue.QueueName, q.handleTaskCommand) if err != nil { return err } q.consumers[current.TaskID] = consumer go q.watchConsumer(current.TaskID, consumer) return nil } func (q *v3TaskQueueController) StopTask(ctx context.Context, assignment store.LocalTaskAssignment) error { if q == nil || q.dispatcher == nil { return errors.New("v3 task queue controller is not configured") } q.mu.Lock() defer q.mu.Unlock() current, err := q.dispatcher.store.LocalTaskAssignment(assignment.DispatcherID, assignment.TaskID) if err != nil { return err } if current.TenantID != assignment.TenantID || current.TenantKey != assignment.TenantKey { return store.ErrTenantBindingConflict } if current.AdmissionState == "running" { return errors.New("refuse to stop task consumer before durable admission barrier") } consumer, exists := q.consumers[assignment.TaskID] if !exists { return nil } if err := consumer.Stop(ctx); err != nil { return err } delete(q.consumers, assignment.TaskID) return nil } // StopConsumer cancels consumption without changing the durable task state. It // is used during process shutdown and when a fresh execution snapshot is absent. func (q *v3TaskQueueController) StopConsumer(ctx context.Context, taskID string) error { if q == nil { return errors.New("v3 task queue controller is not configured") } q.mu.Lock() defer q.mu.Unlock() consumer, exists := q.consumers[taskID] if !exists { return nil } if err := consumer.Stop(ctx); err != nil { return err } delete(q.consumers, taskID) return nil } func (q *v3TaskQueueController) StopAll(ctx context.Context) error { q.mu.Lock() taskIDs := make([]string, 0, len(q.consumers)) for taskID := range q.consumers { taskIDs = append(taskIDs, taskID) } q.mu.Unlock() var failures []error for _, taskID := range taskIDs { if err := q.StopConsumer(ctx, taskID); err != nil { failures = append(failures, fmt.Errorf("stop task %s consumer: %w", taskID, err)) } } return errors.Join(failures...) } func (q *v3TaskQueueController) DrainTask(ctx context.Context, assignment store.LocalTaskAssignment) error { if q == nil || q.dispatcher == nil || q.broker == nil { return errors.New("v3 task queue controller is not configured") } current, err := q.dispatcher.store.LocalTaskAssignment(assignment.DispatcherID, assignment.TaskID) if err != nil { return err } if current.TenantID != assignment.TenantID || current.TenantKey != assignment.TenantKey { return store.ErrTenantBindingConflict } if current.Removed || current.AdmissionState != "stopped" || current.Status != "stopped" { return errors.New("refuse to drain task queue before authoritative stop") } if current.Queue.QueueName == "" { return errors.New("stopped task has no SaaS-owned queue name") } _, err = q.broker.DrainPredeclared(ctx, current.Queue.QueueName) return err } func (q *v3TaskQueueController) ApplyActiveCallPolicy(ctx context.Context, assignment store.LocalTaskAssignment, action, policy string) error { if policy == "drain" { return nil } if policy != "hangup" { return fmt.Errorf("unsupported active-call policy %q", policy) } var controlAction agentpb.ControlAction switch action { case "pause": controlAction = agentpb.ControlAction_CONTROL_ACTION_PAUSE case "stop": controlAction = agentpb.ControlAction_CONTROL_ACTION_STOP default: return fmt.Errorf("active-call hangup is not valid for %q", action) } q.dispatcher.executionMu.Lock() defer q.dispatcher.executionMu.Unlock() targets, err := q.dispatcher.store.ActiveTaskExecutionControls(assignment.TenantID, assignment.TenantKey, assignment.TaskID) if err != nil { return err } if len(targets) == 0 { return nil } if q.taskController == nil { return errors.New("active-call hangup requires the configured Agent task controller") } for _, target := range targets { if target.Binding == nil || target.AgentID == "" { return fmt.Errorf("execution %s has no durable Agent control binding", target.ExecutionID) } response, err := q.taskController.Control(ctx, target.AgentID, target.Binding, controlAction, agentpb.ActiveCallPolicy_ACTIVE_CALL_POLICY_HANGUP) if err != nil { return fmt.Errorf("hang up active execution %s: %w", target.ExecutionID, err) } if response == nil || response.GetReceipt().GetResult() != agentpb.ResultCode_RESULT_CODE_APPLIED || response.GetAppliedTaskRevision() != target.Binding.TaskRevision+1 || response.GetState() != agentpb.ExecutionState_EXECUTION_STATE_TERMINAL { return fmt.Errorf("Agent did not confirm hangup for execution %s", target.ExecutionID) } } return nil } func (q *v3TaskQueueController) Errors() <-chan error { return q.errors } func (q *v3TaskQueueController) watchConsumer(taskID string, consumer taskQueueConsumer) { if err := consumer.Wait(context.Background()); err != nil { q.mu.Lock() if q.consumers[taskID] == consumer { delete(q.consumers, taskID) } q.mu.Unlock() q.errors <- fmt.Errorf("task queue consumer %s: %w", taskID, err) } }