go-sip/internal/dispatcher/task_control_v3_test.go

package dispatcher

import (
	"context"
	"encoding/json"
	"errors"
	"path/filepath"
	"reflect"
	"testing"
	"time"

	"git.ipao.vip/rogee/go-sip/internal/configread"
	"git.ipao.vip/rogee/go-sip/internal/mq"
	"git.ipao.vip/rogee/go-sip/internal/store"
)

type fakeTaskControlStatusReader struct {
	status configread.TaskStatus
	err    error
	calls  int
}

func (f *fakeTaskControlStatusReader) ReadTaskStatus(context.Context, string, string, string) (configread.TaskStatus, error) {
	f.calls++
	return f.status, f.err
}

type fakeTaskQueueController struct {
	events   []string
	err      error
	store    *store.Store
	drainErr error
}

func (f *fakeTaskQueueController) StartTask(context.Context, store.LocalTaskAssignment) error {
	f.events = append(f.events, "start")
	return f.err
}

func (f *fakeTaskQueueController) StopTask(context.Context, store.LocalTaskAssignment) error {
	f.events = append(f.events, "stop")
	return f.err
}

func (f *fakeTaskQueueController) DrainTask(_ context.Context, assignment store.LocalTaskAssignment) error {
	if assignment.AdmissionState != "stopped" {
		return errUnexpectedTaskState
	}
	if f.store != nil {
		current, err := f.store.LocalTaskAssignment(assignment.DispatcherID, assignment.TaskID)
		if err != nil {
			return err
		}
		if current.Status != "stopped" {
			return errors.New("authoritative stopped status was not persisted before queue drain")
		}
		var outboxCount int
		if err := f.store.DB().QueryRow(`SELECT COUNT(*) FROM outbox`).Scan(&outboxCount); err != nil {
			return err
		}
		if outboxCount != 0 {
			return errors.New("task-control receipt was persisted before queue drain")
		}
	}
	f.events = append(f.events, "drain")
	return f.drainErr
}

func (f *fakeTaskQueueController) ApplyActiveCallPolicy(_ context.Context, _ store.LocalTaskAssignment, _ string, policy string) error {
	if policy == "hangup" {
		f.events = append(f.events, "hangup")
	}
	return f.err
}

func TestTaskControlPauseClosesAdmissionBeforeStoppingConsumer(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer server.Close()
	defer st.Close()
	reader := &fakeTaskControlStatusReader{status: localControlStatus("paused", 3)}
	queues := &fakeTaskQueueController{}
	processor := newLocalTaskControlProcessor(d, reader, queues)
	if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "pause")); err != nil {
		t.Fatal(err)
	}
	assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
	if err != nil || len(assignments) != 1 || assignments[0].AdmissionState != "paused" {
		t.Fatalf("assignments=%+v err=%v", assignments, err)
	}
	if len(queues.events) != 2 || queues.events[0] != "stop" || queues.events[1] != "hangup" || reader.calls != 1 {
		t.Fatalf("queue events=%v status calls=%d", queues.events, reader.calls)
	}
	assertTaskControlReceipt(t, st, "pause", "applied", "applied", "paused")
}

func TestTaskControlStopDrainsOnlyAfterAuthoritativeStop(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer server.Close()
	defer st.Close()
	reader := &fakeTaskControlStatusReader{status: localControlStatus("stopped", 3)}
	queues := &fakeTaskQueueController{store: st}
	processor := newLocalTaskControlProcessor(d, reader, queues)
	if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "stop")); err != nil {
		t.Fatal(err)
	}
	if len(queues.events) != 3 || queues.events[0] != "stop" || queues.events[1] != "hangup" || queues.events[2] != "drain" {
		t.Fatalf("stop/hangup/drain ordering = %v", queues.events)
	}
	assertTaskControlReceipt(t, st, "stop", "applied", "applied", "stopped")
}

func TestTaskControlStopPersistsReceiptOnlyAfterSuccessfulDrain(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer server.Close()
	defer st.Close()
	reader := &fakeTaskControlStatusReader{status: localControlStatus("stopped", 3)}
	queues := &fakeTaskQueueController{store: st, drainErr: errors.New("injected drain failure")}
	processor := newLocalTaskControlProcessor(d, reader, queues)
	body := localTaskControlBody(t, now, "stop")
	if err := processor.Handle(context.Background(), localControlRoutingKey(), body); !errors.Is(err, queues.drainErr) {
		t.Fatalf("first stop error=%v, want drain failure", err)
	}
	assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID)
	if err != nil || assignment.Status != "stopped" || assignment.AdmissionState != "stopped" {
		t.Fatalf("authoritative stop was not retained for recovery: %+v err=%v", assignment, err)
	}
	var outboxCount int
	if err := st.DB().QueryRow(`SELECT COUNT(*) FROM outbox`).Scan(&outboxCount); err != nil || outboxCount != 0 {
		t.Fatalf("receipt exists before successful drain: count=%d err=%v", outboxCount, err)
	}

	queues.drainErr = nil
	if err := processor.Handle(context.Background(), localControlRoutingKey(), body); err != nil {
		t.Fatalf("retry stop after drain recovery: %v", err)
	}
	assertTaskControlReceipt(t, st, "stop", "applied", "applied", "stopped")
}

func TestTaskControlUnassignedTaskEmitsRejectedReceipt(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer server.Close()
	defer st.Close()
	body := localTaskControlBody(t, now, "stop")
	var command map[string]any
	if err := json.Unmarshal(body, &command); err != nil {
		t.Fatal(err)
	}
	command["payload"].(map[string]any)["task_id"] = "task-not-assigned"
	body, err := json.Marshal(command)
	if err != nil {
		t.Fatal(err)
	}
	processor := newLocalTaskControlProcessor(d, &fakeTaskControlStatusReader{}, &fakeTaskQueueController{})
	if err := processor.Handle(context.Background(), localControlRoutingKey(), body); err != nil {
		t.Fatal(err)
	}
	assertTaskControlReceipt(t, st, "stop", "rejected", "task_unavailable", "stopped")
}

func TestTaskControlResumeKeepsAdmissionPausedWhenQueueCannotStart(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer server.Close()
	defer st.Close()
	assignment, err := st.SetLocalTaskAdmissionBarrier(localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey, "paused")
	if err != nil {
		t.Fatal(err)
	}
	reader := &fakeTaskControlStatusReader{status: localControlStatus("running", assignment.TaskRevision+1)}
	queues := &fakeTaskQueueController{err: errors.New("queue unavailable")}
	processor := newLocalTaskControlProcessor(d, reader, queues)
	if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "resume")); err != nil {
		t.Fatal(err)
	}
	current, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID)
	if err != nil || current.AdmissionState != "paused" {
		t.Fatalf("failed resume opened admission: %+v err=%v", current, err)
	}
	assertTaskControlReceipt(t, st, "resume", "rejected", "queue_unavailable", "paused")
}

func TestTaskControlPauseMismatchKeepsAdmissionClosed(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer server.Close()
	defer st.Close()
	reader := &fakeTaskControlStatusReader{status: localControlStatus("running", 2)}
	queues := &fakeTaskQueueController{}
	processor := newLocalTaskControlProcessor(d, reader, queues)
	if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "pause")); err != nil {
		t.Fatal(err)
	}
	assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
	if err != nil || assignments[0].AdmissionState != "paused" {
		t.Fatalf("assignments=%+v err=%v", assignments, err)
	}
	assertTaskControlReceipt(t, st, "pause", "rejected", "state_mismatch", "paused")
}

func TestTaskControlResumeReopensPausedTaskAfterFreshRunningStatus(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer server.Close()
	defer st.Close()
	assignment, err := st.SetLocalTaskAdmissionBarrier(localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey, "paused")
	if err != nil {
		t.Fatal(err)
	}
	reader := &fakeTaskControlStatusReader{status: localControlStatus("running", assignment.TaskRevision+1)}
	queues := &fakeTaskQueueController{}
	processor := newLocalTaskControlProcessor(d, reader, queues)
	if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "resume")); err != nil {
		t.Fatal(err)
	}
	current, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID)
	if err != nil || current.Status != "running" || current.AdmissionState != "running" || current.TaskRevision != assignment.TaskRevision+1 {
		t.Fatalf("resumed assignment=%+v err=%v", current, err)
	}
	if len(queues.events) != 1 || queues.events[0] != "start" {
		t.Fatalf("resume queue events=%v", queues.events)
	}
	assertTaskControlReceipt(t, st, "resume", "applied", "applied", "running")
}

func TestTaskControlResumeCannotReopenStoppedTask(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer server.Close()
	defer st.Close()
	if _, err := st.SetLocalTaskAdmissionBarrier(localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey, "stopped"); err != nil {
		t.Fatal(err)
	}
	reader := &fakeTaskControlStatusReader{status: localControlStatus("running", 2)}
	queues := &fakeTaskQueueController{}
	processor := newLocalTaskControlProcessor(d, reader, queues)
	if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "resume")); err != nil {
		t.Fatal(err)
	}
	assignments, err := st.LocalTaskAssignments(localTestDispatcherID)
	if err != nil || assignments[0].AdmissionState != "stopped" {
		t.Fatalf("assignments=%+v err=%v", assignments, err)
	}
	if len(queues.events) != 0 {
		t.Fatalf("stopped task started queues: %v", queues.events)
	}
	assertTaskControlReceipt(t, st, "resume", "rejected", "task_stopped", "stopped")
}

var errUnexpectedTaskState = taskStateError("task queue drain attempted before stopped barrier")

type taskStateError string

func (e taskStateError) Error() string { return string(e) }

func localControlStatus(status string, revision int64) configread.TaskStatus {
	return configread.TaskStatus{
		DispatcherID: localTestDispatcherID,
		TaskID:       localTestTaskID,
		TenantID:     localTestTenantID,
		TenantKey:    localTestTenantKey,
		Status:       status,
		TaskRevision: revision,
	}
}

func localControlRoutingKey() string {
	return "d." + localTestDispatcherID + ".control.in"
}

func TestTaskControlUnknownTaskBeforeCompleteSnapshotRequeues(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	st, err := store.Open(filepath.Join(t.TempDir(), "dispatcher.db"))
	if err != nil {
		t.Fatal(err)
	}
	defer st.Close()
	d, err := NewV3(localTestDispatcherID, st, nil, func() time.Time { return now })
	if err != nil {
		t.Fatal(err)
	}
	processor := newLocalTaskControlProcessor(d, &fakeTaskControlStatusReader{status: localControlStatus("paused", 3)}, &fakeTaskQueueController{})
	err = processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "pause"))
	if err == nil || mq.IsPermanent(err) {
		t.Fatalf("unresolved ownership was ACKed/rejected instead of requeued: %v", err)
	}
	var outboxCount int
	if err := st.DB().QueryRow(`SELECT COUNT(*) FROM outbox`).Scan(&outboxCount); err != nil || outboxCount != 0 {
		t.Fatalf("unknown task produced receipt before snapshot: count=%d err=%v", outboxCount, err)
	}
}

func TestOldQueuedStopStillHangsUpAndDrains(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer st.Close()
	defer server.Close()
	queues := &fakeTaskQueueController{store: st}
	processor := newLocalTaskControlProcessor(d, &fakeTaskControlStatusReader{status: localControlStatus("stopped", 3)}, queues)
	if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now.Add(-30*24*time.Hour), "stop")); err != nil {
		t.Fatal(err)
	}
	if len(queues.events) != 3 || queues.events[0] != "stop" || queues.events[1] != "hangup" || queues.events[2] != "drain" {
		t.Fatalf("old stop did not hang up and drain: %v", queues.events)
	}
	assertTaskControlReceipt(t, st, "stop", "applied", "applied", "stopped")
}

func TestOldQueuedPauseResumeStopPreservesOrderAndEveryReceipt(t *testing.T) {
	now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC)
	d, st, server := newLocalV01TestDispatcher(t, now)
	defer st.Close()
	defer server.Close()
	reader := &fakeTaskControlStatusReader{}
	queues := &fakeTaskQueueController{}
	processor := newLocalTaskControlProcessor(d, reader, queues)
	for index, step := range []struct {
		action string
		status string
	}{{"pause", "paused"}, {"resume", "running"}, {"stop", "stopped"}} {
		reader.status = localControlStatus(step.status, int64(index+3))
		if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now.Add(-30*24*time.Hour), step.action)); err != nil {
			t.Fatalf("old %s: %v", step.action, err)
		}
	}
	wantEvents := []string{"stop", "hangup", "start", "stop", "hangup", "drain"}
	if !reflect.DeepEqual(queues.events, wantEvents) || reader.calls != 3 {
		t.Fatalf("old control effects skipped/reordered: events=%v reads=%d", queues.events, reader.calls)
	}
	rows, err := st.DB().Query(`SELECT body FROM outbox ORDER BY rowid`)
	if err != nil {
		t.Fatal(err)
	}
	defer rows.Close()
	var actions []string
	for rows.Next() {
		var body []byte
		if err := rows.Scan(&body); err != nil {
			t.Fatal(err)
		}
		var result struct {
			Payload struct {
				Action string `json:"action"`
				Status string `json:"status"`
			} `json:"payload"`
		}
		if err := json.Unmarshal(body, &result); err != nil {
			t.Fatal(err)
		}
		if result.Payload.Status != "applied" {
			t.Fatalf("control receipt not applied: %+v", result.Payload)
		}
		actions = append(actions, result.Payload.Action)
	}
	if err := rows.Err(); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actions, []string{"pause", "resume", "stop"}) {
		t.Fatalf("lost or reordered old control receipts: %v", actions)
	}
}

func localTaskControlBody(t *testing.T, now time.Time, action string) []byte {
	t.Helper()
	payload := map[string]any{
		"task_id": localTestTaskID,
		"action":  action,
		"reason":  "local test control",
	}
	body, err := json.Marshal(map[string]any{
		"schema_version": "task-control.v0.4-proposal",
		"dispatcher_id":  localTestDispatcherID,
		"tenant_id":      localTestTenantID,
		"tenant_key":     localTestTenantKey,
		"trace_id":       "11111111-1111-4111-8111-111111111111",
		"issued_at":      now.Add(-time.Minute).Format(time.RFC3339Nano),
		"command_type":   "task.control",
		"payload":        payload,
	})
	if err != nil {
		t.Fatal(err)
	}
	return body
}

func assertTaskControlReceipt(t *testing.T, st *store.Store, action, status, reason, taskState string) {
	t.Helper()
	records, err := st.ClaimOutbox(8)
	if err != nil {
		t.Fatal(err)
	}
	if len(records) != 1 {
		t.Fatalf("outbox records = %d, want one", len(records))
	}
	var event struct {
		EventType string `json:"event_type"`
		Payload   struct {
			Action     string `json:"action"`
			Status     string `json:"status"`
			ReasonCode string `json:"reason_code"`
			TaskState  string `json:"task_state"`
		} `json:"payload"`
	}
	if err := json.Unmarshal(records[0].Body, &event); err != nil {
		t.Fatal(err)
	}
	if event.EventType != "command.result" || event.Payload.Action != action || event.Payload.Status != status ||
		event.Payload.ReasonCode != reason || event.Payload.TaskState != taskState {
		t.Fatalf("control receipt = %+v", event)
	}
}