300 lines
11 KiB
Go
300 lines
11 KiB
Go
package store
|
|
|
|
import (
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
type LocalTaskControlResultRecord struct {
|
|
DispatcherID string
|
|
TaskID string
|
|
TenantID string
|
|
TenantKey string
|
|
Action string
|
|
ResultStatus string
|
|
ReasonCode string
|
|
ObservedStatus string
|
|
TaskRevision int64
|
|
AdmissionState string
|
|
Event LocalEventRecord
|
|
}
|
|
|
|
// SetLocalTaskAdmissionBarrier durably closes admission before the broker
|
|
// consumer is cancelled. A stopped/finished/removed task can never be reopened.
|
|
func (s *Store) SetLocalTaskAdmissionBarrier(dispatcherID, taskID, tenantID, tenantKey, target string) (LocalTaskAssignment, error) {
|
|
if target != "paused" && target != "stopped" {
|
|
return LocalTaskAssignment{}, fmt.Errorf("unsupported local task barrier %q", target)
|
|
}
|
|
now := s.now().UTC().Format(time.RFC3339Nano)
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return LocalTaskAssignment{}, err
|
|
}
|
|
defer tx.Rollback()
|
|
assignment, err := scanLocalTaskAssignment(tx.QueryRow(`SELECT dispatcher_id,task_id,tenant_id,tenant_key,task_revision,saas_status,admission_state,removed,
|
|
queue_exchange,routing_key,binding_key,queue_name,updated_at FROM local_v01_task_assignments WHERE dispatcher_id=? AND task_id=?`, dispatcherID, taskID))
|
|
if err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return LocalTaskAssignment{}, ErrLocalTaskUnassigned
|
|
}
|
|
return LocalTaskAssignment{}, err
|
|
}
|
|
if assignment.TenantID != tenantID || assignment.TenantKey != tenantKey {
|
|
return LocalTaskAssignment{}, ErrTenantBindingConflict
|
|
}
|
|
state := assignment.AdmissionState
|
|
switch state {
|
|
case "removed", "finished":
|
|
case "stopped":
|
|
case "running", "paused":
|
|
state = target
|
|
default:
|
|
return LocalTaskAssignment{}, fmt.Errorf("unsupported local task admission state %q", state)
|
|
}
|
|
if state != assignment.AdmissionState {
|
|
if _, err := tx.Exec(`UPDATE local_v01_task_assignments SET admission_state=?,updated_at=? WHERE dispatcher_id=? AND task_id=?`,
|
|
state, now, dispatcherID, taskID); err != nil {
|
|
return LocalTaskAssignment{}, err
|
|
}
|
|
assignment.AdmissionState = state
|
|
assignment.UpdatedAt = now
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return LocalTaskAssignment{}, err
|
|
}
|
|
return assignment, nil
|
|
}
|
|
|
|
// ResumeLocalTaskAdmission reopens only a non-terminal task after the caller
|
|
// has freshly verified the SaaS task resource reports running.
|
|
func (s *Store) ResumeLocalTaskAdmission(dispatcherID, taskID, tenantID, tenantKey, observedStatus string, taskRevision int64) (LocalTaskAssignment, error) {
|
|
if observedStatus != "running" || taskRevision <= 0 {
|
|
return LocalTaskAssignment{}, errors.New("resume requires a positive-revision authoritative running status")
|
|
}
|
|
now := s.now().UTC().Format(time.RFC3339Nano)
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return LocalTaskAssignment{}, err
|
|
}
|
|
defer tx.Rollback()
|
|
assignment, err := scanLocalTaskAssignment(tx.QueryRow(`SELECT dispatcher_id,task_id,tenant_id,tenant_key,task_revision,saas_status,admission_state,removed,
|
|
queue_exchange,routing_key,binding_key,queue_name,updated_at FROM local_v01_task_assignments WHERE dispatcher_id=? AND task_id=?`, dispatcherID, taskID))
|
|
if err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return LocalTaskAssignment{}, ErrLocalTaskUnassigned
|
|
}
|
|
return LocalTaskAssignment{}, err
|
|
}
|
|
if assignment.TenantID != tenantID || assignment.TenantKey != tenantKey {
|
|
return LocalTaskAssignment{}, ErrTenantBindingConflict
|
|
}
|
|
if assignment.Removed || assignment.AdmissionState == "stopped" || assignment.AdmissionState == "finished" || assignment.AdmissionState == "removed" {
|
|
return assignment, ErrLocalTaskStopped
|
|
}
|
|
if assignment.AdmissionState != "paused" && assignment.AdmissionState != "running" {
|
|
return LocalTaskAssignment{}, fmt.Errorf("unsupported local task admission state %q", assignment.AdmissionState)
|
|
}
|
|
if taskRevision < assignment.TaskRevision {
|
|
return LocalTaskAssignment{}, ErrConfigRevisionRollback
|
|
}
|
|
if taskRevision == assignment.TaskRevision && assignment.Status != observedStatus {
|
|
return LocalTaskAssignment{}, ErrConfigRevisionConflict
|
|
}
|
|
assignment.Status = observedStatus
|
|
assignment.TaskRevision = taskRevision
|
|
assignment.AdmissionState = "running"
|
|
assignment.UpdatedAt = now
|
|
if _, err := tx.Exec(`UPDATE local_v01_task_assignments SET task_revision=?,saas_status=?,admission_state='running',updated_at=? WHERE dispatcher_id=? AND task_id=?`,
|
|
taskRevision, observedStatus, now, dispatcherID, taskID); err != nil {
|
|
return LocalTaskAssignment{}, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return LocalTaskAssignment{}, err
|
|
}
|
|
return assignment, nil
|
|
}
|
|
|
|
// RecordLocalTaskStatusObservation stores a fresh SaaS status before operations
|
|
// that require the authoritative state, without writing a control receipt.
|
|
func (s *Store) RecordLocalTaskStatusObservation(dispatcherID, taskID, tenantID, tenantKey, observedStatus string, taskRevision int64) error {
|
|
if taskRevision <= 0 {
|
|
return errors.New("observed task status requires a positive revision")
|
|
}
|
|
switch observedStatus {
|
|
case "running", "paused", "stopped", "finished":
|
|
default:
|
|
return fmt.Errorf("unsupported observed task status %q", observedStatus)
|
|
}
|
|
now := s.now().UTC().Format(time.RFC3339Nano)
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
assignment, err := scanLocalTaskAssignment(tx.QueryRow(`SELECT dispatcher_id,task_id,tenant_id,tenant_key,task_revision,saas_status,admission_state,removed,
|
|
queue_exchange,routing_key,binding_key,queue_name,updated_at FROM local_v01_task_assignments WHERE dispatcher_id=? AND task_id=?`, dispatcherID, taskID))
|
|
if err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return ErrLocalTaskUnassigned
|
|
}
|
|
return err
|
|
}
|
|
if assignment.TenantID != tenantID || assignment.TenantKey != tenantKey {
|
|
return ErrTenantBindingConflict
|
|
}
|
|
if taskRevision < assignment.TaskRevision {
|
|
return ErrConfigRevisionRollback
|
|
}
|
|
if taskRevision == assignment.TaskRevision && observedStatus != assignment.Status {
|
|
return ErrConfigRevisionConflict
|
|
}
|
|
if taskRevision > assignment.TaskRevision {
|
|
if _, err := tx.Exec(`UPDATE local_v01_task_assignments SET task_revision=?,saas_status=?,updated_at=? WHERE dispatcher_id=? AND task_id=?`,
|
|
taskRevision, observedStatus, now, dispatcherID, taskID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// PersistLocalTaskControlResult applies the final local state and writes its
|
|
// command.result outbox event in the same SQLite transaction.
|
|
func (s *Store) PersistLocalTaskControlResult(result LocalTaskControlResultRecord) (LocalTaskAssignment, bool, error) {
|
|
if err := validateLocalTaskControlResult(result); err != nil {
|
|
return LocalTaskAssignment{}, false, err
|
|
}
|
|
now := s.now().UTC().Format(time.RFC3339Nano)
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return LocalTaskAssignment{}, false, err
|
|
}
|
|
defer tx.Rollback()
|
|
assignment, err := scanLocalTaskAssignment(tx.QueryRow(`SELECT dispatcher_id,task_id,tenant_id,tenant_key,task_revision,saas_status,admission_state,removed,
|
|
queue_exchange,routing_key,binding_key,queue_name,updated_at FROM local_v01_task_assignments WHERE dispatcher_id=? AND task_id=?`,
|
|
result.DispatcherID, result.TaskID))
|
|
if err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return LocalTaskAssignment{}, false, ErrLocalTaskUnassigned
|
|
}
|
|
return LocalTaskAssignment{}, false, err
|
|
}
|
|
if assignment.TenantID != result.TenantID || assignment.TenantKey != result.TenantKey {
|
|
return LocalTaskAssignment{}, false, ErrTenantBindingConflict
|
|
}
|
|
if result.ObservedStatus != "" {
|
|
if result.TaskRevision < assignment.TaskRevision {
|
|
return LocalTaskAssignment{}, false, ErrConfigRevisionRollback
|
|
}
|
|
if result.TaskRevision == assignment.TaskRevision && result.ObservedStatus != assignment.Status {
|
|
return LocalTaskAssignment{}, false, ErrConfigRevisionConflict
|
|
}
|
|
assignment.Status = result.ObservedStatus
|
|
assignment.TaskRevision = result.TaskRevision
|
|
}
|
|
state, err := controlResultAdmissionState(assignment.AdmissionState, result)
|
|
if err != nil {
|
|
return LocalTaskAssignment{}, false, err
|
|
}
|
|
assignment.AdmissionState = state
|
|
assignment.UpdatedAt = now
|
|
if _, err := tx.Exec(`UPDATE local_v01_task_assignments SET task_revision=?,saas_status=?,admission_state=?,updated_at=? WHERE dispatcher_id=? AND task_id=?`,
|
|
assignment.TaskRevision, assignment.Status, assignment.AdmissionState, now, result.DispatcherID, result.TaskID); err != nil {
|
|
return LocalTaskAssignment{}, false, err
|
|
}
|
|
duplicate, err := enqueueLocalEventTx(tx, result.Event, now)
|
|
if err != nil {
|
|
return LocalTaskAssignment{}, false, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return LocalTaskAssignment{}, false, err
|
|
}
|
|
return assignment, duplicate, nil
|
|
}
|
|
|
|
func validateLocalTaskControlResult(result LocalTaskControlResultRecord) error {
|
|
if result.DispatcherID == "" || result.TaskID == "" || result.TenantID == "" || result.TenantKey == "" {
|
|
return errors.New("task-control result identity is required")
|
|
}
|
|
switch result.Action {
|
|
case "pause", "resume", "stop":
|
|
default:
|
|
return fmt.Errorf("unsupported task-control action %q", result.Action)
|
|
}
|
|
switch result.ResultStatus {
|
|
case "applied", "rejected":
|
|
default:
|
|
return fmt.Errorf("unsupported task-control result status %q", result.ResultStatus)
|
|
}
|
|
switch result.AdmissionState {
|
|
case "running", "paused", "stopped", "finished":
|
|
default:
|
|
return fmt.Errorf("unsupported task-control result state %q", result.AdmissionState)
|
|
}
|
|
if result.ObservedStatus != "" {
|
|
switch result.ObservedStatus {
|
|
case "running", "paused", "stopped", "finished":
|
|
default:
|
|
return fmt.Errorf("unsupported observed task status %q", result.ObservedStatus)
|
|
}
|
|
if result.TaskRevision <= 0 {
|
|
return errors.New("observed task status requires a positive revision")
|
|
}
|
|
} else if result.TaskRevision != 0 {
|
|
return errors.New("task revision requires an observed task status")
|
|
}
|
|
return validateLocalEventRecord(result.Event)
|
|
}
|
|
|
|
func controlResultAdmissionState(current string, result LocalTaskControlResultRecord) (string, error) {
|
|
terminal := current == "stopped" || current == "finished" || current == "removed"
|
|
if terminal {
|
|
if current == "removed" && result.AdmissionState == "stopped" && result.ResultStatus == "rejected" {
|
|
return current, nil
|
|
}
|
|
if result.AdmissionState != current {
|
|
return "", ErrLocalTaskStopped
|
|
}
|
|
}
|
|
if result.ResultStatus == "applied" {
|
|
switch result.Action {
|
|
case "pause":
|
|
if result.ObservedStatus != "paused" || result.AdmissionState != "paused" {
|
|
return "", errors.New("applied pause requires authoritative paused state")
|
|
}
|
|
case "resume":
|
|
if result.ObservedStatus != "running" || result.AdmissionState != "running" || terminal {
|
|
return "", ErrLocalTaskStopped
|
|
}
|
|
case "stop":
|
|
if result.ObservedStatus != "stopped" || result.AdmissionState != "stopped" {
|
|
return "", errors.New("applied stop requires authoritative stopped state")
|
|
}
|
|
}
|
|
return result.AdmissionState, nil
|
|
}
|
|
if result.AdmissionState == current {
|
|
return current, nil
|
|
}
|
|
if current == "running" && result.AdmissionState == "paused" {
|
|
return "paused", nil
|
|
}
|
|
return "", errors.New("rejected task-control result cannot reopen admission")
|
|
}
|
|
|
|
func validateLocalEventRecord(event LocalEventRecord) error {
|
|
if event.EventID == "" || event.TenantKey == "" || event.Exchange == "" || event.RoutingKey == "" || len(event.Body) == 0 {
|
|
return errors.New("local event identity, route, and body are required")
|
|
}
|
|
return nil
|
|
}
|