280 lines
9.9 KiB
Go
280 lines
9.9 KiB
Go
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"sync"
|
|
|
|
agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
|
|
"git.ipao.vip/rogee/go-sip/internal/mq"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
)
|
|
|
|
type taskQueueConsumer interface {
|
|
Wait(context.Context) error
|
|
Stop(context.Context) error
|
|
}
|
|
|
|
type taskQueueBroker interface {
|
|
StartPredeclaredConsumer(context.Context, string, mq.MessageHandler) (taskQueueConsumer, error)
|
|
DrainPredeclared(context.Context, string) (int, error)
|
|
DrainControlPredeclared(context.Context, string, mq.MessageHandler) (int, error)
|
|
}
|
|
|
|
type v3TaskBrokerAdapter struct{ broker *mq.V3Broker }
|
|
|
|
func (a v3TaskBrokerAdapter) StartPredeclaredConsumer(ctx context.Context, queue string, handler mq.MessageHandler) (taskQueueConsumer, error) {
|
|
return a.broker.StartPredeclaredConsumer(ctx, queue, handler)
|
|
}
|
|
|
|
func (a v3TaskBrokerAdapter) DrainPredeclared(ctx context.Context, queue string) (int, error) {
|
|
return a.broker.DrainPredeclared(ctx, queue)
|
|
}
|
|
|
|
func (a v3TaskBrokerAdapter) DrainControlPredeclared(ctx context.Context, queue string, handler mq.MessageHandler) (int, error) {
|
|
return a.broker.DrainControlPredeclared(ctx, queue, handler)
|
|
}
|
|
|
|
type v3TaskQueueController struct {
|
|
dispatcher *Dispatcher
|
|
broker taskQueueBroker
|
|
taskController TaskController
|
|
mockAuthorizedAgentID string
|
|
mockAuthorizedAgents *AgentCoordinator
|
|
mu sync.Mutex
|
|
consumers map[string]taskQueueConsumer
|
|
errors chan error
|
|
}
|
|
|
|
func newV3TaskQueueController(d *Dispatcher, broker *mq.V3Broker, taskController TaskController) *v3TaskQueueController {
|
|
return newTaskQueueController(d, v3TaskBrokerAdapter{broker: broker}, taskController)
|
|
}
|
|
|
|
func newTaskQueueController(d *Dispatcher, broker taskQueueBroker, taskController TaskController) *v3TaskQueueController {
|
|
return &v3TaskQueueController{
|
|
dispatcher: d, broker: broker, taskController: taskController,
|
|
consumers: make(map[string]taskQueueConsumer), errors: make(chan error, 64),
|
|
}
|
|
}
|
|
|
|
// EnableMockAuthorizedOrigination must run before any task consumer starts.
|
|
// Non-mock modes never call this setter and cannot originate by this path.
|
|
func (q *v3TaskQueueController) EnableMockAuthorizedOrigination(agentID string, agents *AgentCoordinator) error {
|
|
if q == nil || agentID == "" || agents == nil {
|
|
return errors.New("mock authorized origination requires one activated Agent")
|
|
}
|
|
q.mu.Lock()
|
|
defer q.mu.Unlock()
|
|
if len(q.consumers) != 0 || q.mockAuthorizedAgentID != "" {
|
|
return errors.New("mock authorized origination must be configured once before consumption")
|
|
}
|
|
q.mockAuthorizedAgentID, q.mockAuthorizedAgents = agentID, agents
|
|
return nil
|
|
}
|
|
|
|
func (q *v3TaskQueueController) handleTaskCommand(ctx context.Context, routingKey string, body []byte) error {
|
|
if err := q.dispatcher.AcceptLocalV01Command(ctx, routingKey, body); err != nil {
|
|
return err
|
|
}
|
|
if q.mockAuthorizedAgentID == "" {
|
|
return nil
|
|
}
|
|
executionID, err := q.dispatcher.store.LocalPendingOriginationForCommand(body)
|
|
if errors.Is(err, sql.ErrNoRows) { // Rejected command or already-issued/refused execution.
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("read pending local execution after durable admission: %w", err)
|
|
}
|
|
result, dispatchErr := q.dispatcher.DispatchAndFinalizeAuthorizedMock(ctx, executionID, q.mockAuthorizedAgentID, q.mockAuthorizedAgents)
|
|
decision, err := q.dispatcher.store.LocalOriginationDecision(executionID)
|
|
if err != nil {
|
|
return errors.Join(dispatchErr, fmt.Errorf("verify durable final dial decision before ACK: %w", err))
|
|
}
|
|
if dispatchErr != nil {
|
|
slog.Error("Mock Agent instruction refused or uncertain; no automatic retry", "execution_id", executionID,
|
|
"decision", decision, "unknown", result.Unknown, "error", dispatchErr)
|
|
} else if decision != "issued" {
|
|
return fmt.Errorf("Agent applied execution %s without an issued dial decision", executionID)
|
|
}
|
|
// The command was durably admitted and its final decision was persisted.
|
|
// An Agent error is never passed to MQ as a requeue/re-originate signal.
|
|
return nil
|
|
}
|
|
|
|
func (q *v3TaskQueueController) StartTask(ctx context.Context, assignment store.LocalTaskAssignment) error {
|
|
if q == nil || q.dispatcher == nil || q.broker == nil {
|
|
return errors.New("v3 task queue controller is not configured")
|
|
}
|
|
q.mu.Lock()
|
|
defer q.mu.Unlock()
|
|
if _, exists := q.consumers[assignment.TaskID]; exists {
|
|
return nil
|
|
}
|
|
current, err := q.dispatcher.store.LocalTaskAssignment(assignment.DispatcherID, assignment.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if current.TenantID != assignment.TenantID || current.TenantKey != assignment.TenantKey {
|
|
return store.ErrTenantBindingConflict
|
|
}
|
|
if current.Removed || current.AdmissionState != "running" || current.Status != "running" {
|
|
return fmt.Errorf("task %s is not eligible for queue consumption", assignment.TaskID)
|
|
}
|
|
if current.Queue.QueueName == "" {
|
|
return fmt.Errorf("task %s has no SaaS-owned queue name", assignment.TaskID)
|
|
}
|
|
consumer, err := q.broker.StartPredeclaredConsumer(ctx, current.Queue.QueueName, q.handleTaskCommand)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
q.consumers[current.TaskID] = consumer
|
|
go q.watchConsumer(current.TaskID, consumer)
|
|
return nil
|
|
}
|
|
|
|
func (q *v3TaskQueueController) StopTask(ctx context.Context, assignment store.LocalTaskAssignment) error {
|
|
if q == nil || q.dispatcher == nil {
|
|
return errors.New("v3 task queue controller is not configured")
|
|
}
|
|
q.mu.Lock()
|
|
defer q.mu.Unlock()
|
|
current, err := q.dispatcher.store.LocalTaskAssignment(assignment.DispatcherID, assignment.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if current.TenantID != assignment.TenantID || current.TenantKey != assignment.TenantKey {
|
|
return store.ErrTenantBindingConflict
|
|
}
|
|
if current.AdmissionState == "running" {
|
|
return errors.New("refuse to stop task consumer before durable admission barrier")
|
|
}
|
|
consumer, exists := q.consumers[assignment.TaskID]
|
|
if !exists {
|
|
return nil
|
|
}
|
|
if err := consumer.Stop(ctx); err != nil {
|
|
return err
|
|
}
|
|
delete(q.consumers, assignment.TaskID)
|
|
return nil
|
|
}
|
|
|
|
// StopConsumer cancels consumption without changing the durable task state. It
|
|
// is used during process shutdown and when a fresh execution snapshot is absent.
|
|
func (q *v3TaskQueueController) StopConsumer(ctx context.Context, taskID string) error {
|
|
if q == nil {
|
|
return errors.New("v3 task queue controller is not configured")
|
|
}
|
|
q.mu.Lock()
|
|
defer q.mu.Unlock()
|
|
consumer, exists := q.consumers[taskID]
|
|
if !exists {
|
|
return nil
|
|
}
|
|
if err := consumer.Stop(ctx); err != nil {
|
|
return err
|
|
}
|
|
delete(q.consumers, taskID)
|
|
return nil
|
|
}
|
|
|
|
func (q *v3TaskQueueController) StopAll(ctx context.Context) error {
|
|
q.mu.Lock()
|
|
taskIDs := make([]string, 0, len(q.consumers))
|
|
for taskID := range q.consumers {
|
|
taskIDs = append(taskIDs, taskID)
|
|
}
|
|
q.mu.Unlock()
|
|
var failures []error
|
|
for _, taskID := range taskIDs {
|
|
if err := q.StopConsumer(ctx, taskID); err != nil {
|
|
failures = append(failures, fmt.Errorf("stop task %s consumer: %w", taskID, err))
|
|
}
|
|
}
|
|
return errors.Join(failures...)
|
|
}
|
|
|
|
func (q *v3TaskQueueController) DrainTask(ctx context.Context, assignment store.LocalTaskAssignment) error {
|
|
if q == nil || q.dispatcher == nil || q.broker == nil {
|
|
return errors.New("v3 task queue controller is not configured")
|
|
}
|
|
current, err := q.dispatcher.store.LocalTaskAssignment(assignment.DispatcherID, assignment.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if current.TenantID != assignment.TenantID || current.TenantKey != assignment.TenantKey {
|
|
return store.ErrTenantBindingConflict
|
|
}
|
|
if current.Removed || current.AdmissionState != "stopped" || current.Status != "stopped" {
|
|
return errors.New("refuse to drain task queue before authoritative stop")
|
|
}
|
|
if current.Queue.QueueName == "" {
|
|
return errors.New("stopped task has no SaaS-owned queue name")
|
|
}
|
|
_, err = q.broker.DrainPredeclared(ctx, current.Queue.QueueName)
|
|
return err
|
|
}
|
|
|
|
func (q *v3TaskQueueController) ApplyActiveCallPolicy(ctx context.Context, assignment store.LocalTaskAssignment, action, policy string) error {
|
|
if policy == "drain" {
|
|
return nil
|
|
}
|
|
if policy != "hangup" {
|
|
return fmt.Errorf("unsupported active-call policy %q", policy)
|
|
}
|
|
var controlAction agentpb.ControlAction
|
|
switch action {
|
|
case "pause":
|
|
controlAction = agentpb.ControlAction_CONTROL_ACTION_PAUSE
|
|
case "stop":
|
|
controlAction = agentpb.ControlAction_CONTROL_ACTION_STOP
|
|
default:
|
|
return fmt.Errorf("active-call hangup is not valid for %q", action)
|
|
}
|
|
|
|
q.dispatcher.executionMu.Lock()
|
|
defer q.dispatcher.executionMu.Unlock()
|
|
targets, err := q.dispatcher.store.ActiveTaskExecutionControls(assignment.TenantID, assignment.TenantKey, assignment.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(targets) == 0 {
|
|
return nil
|
|
}
|
|
if q.taskController == nil {
|
|
return errors.New("active-call hangup requires the configured Agent task controller")
|
|
}
|
|
for _, target := range targets {
|
|
if target.Binding == nil || target.AgentID == "" {
|
|
return fmt.Errorf("execution %s has no durable Agent control binding", target.ExecutionID)
|
|
}
|
|
response, err := q.taskController.Control(ctx, target.AgentID, target.Binding, controlAction, agentpb.ActiveCallPolicy_ACTIVE_CALL_POLICY_HANGUP)
|
|
if err != nil {
|
|
return fmt.Errorf("hang up active execution %s: %w", target.ExecutionID, err)
|
|
}
|
|
if response == nil || response.GetReceipt().GetResult() != agentpb.ResultCode_RESULT_CODE_APPLIED ||
|
|
response.GetAppliedTaskRevision() != target.Binding.TaskRevision+1 ||
|
|
response.GetState() != agentpb.ExecutionState_EXECUTION_STATE_TERMINAL {
|
|
return fmt.Errorf("Agent did not confirm hangup for execution %s", target.ExecutionID)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (q *v3TaskQueueController) Errors() <-chan error { return q.errors }
|
|
|
|
func (q *v3TaskQueueController) watchConsumer(taskID string, consumer taskQueueConsumer) {
|
|
if err := consumer.Wait(context.Background()); err != nil {
|
|
q.mu.Lock()
|
|
if q.consumers[taskID] == consumer {
|
|
delete(q.consumers, taskID)
|
|
}
|
|
q.mu.Unlock()
|
|
q.errors <- fmt.Errorf("task queue consumer %s: %w", taskID, err)
|
|
}
|
|
}
|