Advance durable Agent session generations on restart

internal/dispatcher/agent.go

@@ -11,6 +11,7 @@ import (
 	agentpb "git.ipao.vip/rogee/go-sip/gen/agent"
 	"git.ipao.vip/rogee/go-sip/internal/configread"
 	"git.ipao.vip/rogee/go-sip/internal/store"
+	"github.com/google/uuid"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -156,7 +157,11 @@ func (c *AgentCoordinator) Activate(ctx context.Context, agentID, cellID, bootID
 	if cellID == "" || bootID == "" || epoch == "" {
 		return AgentSession{}, errors.New("cell, boot and dispatcher epoch are required")
 	}
-	operationID := fmt.Sprintf("activate:%s:%s", agentID, bootID)
+	activationID, err := uuid.NewRandom()
+	if err != nil {
+		return AgentSession{}, fmt.Errorf("create Agent activation identity: %w", err)
+	}
+	operationID := "activate:" + activationID.String()
 	meta := &agentpb.RequestMeta{ProtocolVersion: "agent.v1", RequestId: operationID + ":request", TraceId: operationID, OperationId: operationID, DispatcherEpoch: epoch, AgentId: agentID, CellId: cellID, BootId: bootID}
 	response, err := client.ActivateAgent(ctx, &agentpb.ActivateAgentRequest{Meta: meta, Binding: &agentpb.AgentBinding{AgentId: agentID, CellId: cellID, ExpectedBootId: bootID, DispatcherEpoch: epoch, SessionGeneration: generation}, ActivationOperationId: operationID})
 	if err != nil {

internal/dispatcher/agent_test.go

@@ -55,8 +55,12 @@ func TestAgentCoordinatorProbesBeforeActivation(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if first.SessionGeneration != 1 || second.SessionGeneration != 2 {
-		t.Fatalf("unexpected generations: first=%d second=%d", first.SessionGeneration, second.SessionGeneration)
+	third, err := coordinator.Activate(context.Background(), "agent-1", "cell-1", status.BootId, "epoch-2", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first.SessionGeneration != 1 || second.SessionGeneration != 2 || third.SessionGeneration != 3 {
+		t.Fatalf("a new activation replayed an old session: first=%d second=%d third=%d", first.SessionGeneration, second.SessionGeneration, third.SessionGeneration)
 	}
 }
 

internal/rpc/server.go

@@ -294,6 +294,9 @@ func (r *SessionRegistry) Activate(binding *agentpb.AgentBinding, activationOper
 			return cloneSession(existing.session), true, nil
 		}
 		if binding.SessionGeneration == 0 {
+			if existing.binding.SessionGeneration == ^uint64(0) {
+				return nil, false, status.Error(codes.Aborted, "session generation is exhausted")
+			}
 			binding = proto.Clone(binding).(*agentpb.AgentBinding)
 			binding.SessionGeneration = existing.binding.SessionGeneration + 1
 		}
@@ -304,6 +307,12 @@ func (r *SessionRegistry) Activate(binding *agentpb.AgentBinding, activationOper
 	if binding.SessionGeneration == 0 {
 		binding = proto.Clone(binding).(*agentpb.AgentBinding)
 		binding.SessionGeneration = 1
+		if previous, ok := r.generations[binding.AgentId]; ok {
+			if previous == ^uint64(0) {
+				return nil, false, status.Error(codes.Aborted, "session generation is exhausted")
+			}
+			binding.SessionGeneration = previous + 1
+		}
 	}
 	if previous, ok := r.generations[binding.AgentId]; ok && binding.SessionGeneration <= previous {
 		return nil, false, status.Error(codes.Aborted, "persisted session generation is fenced")

internal/rpc/session_test.go

@@ -1,6 +1,7 @@
 package rpc
 
 import (
+	"strings"
 	"testing"
 	"time"
 
@@ -25,3 +26,45 @@ func TestSessionRegistryPersistsGenerationAcrossRestart(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestSessionRegistryZeroGenerationAdvancesDurableHighwaterAfterRestart(t *testing.T) {
+	path := t.TempDir() + "/rpc-session.json"
+	first := NewSessionRegistry(path)
+	now := time.Unix(100, 0)
+	if _, _, err := first.Activate(&agentpb.AgentBinding{AgentId: "agent-1", CellId: "cell-1", ExpectedBootId: "boot-1", DispatcherEpoch: "epoch-1", SessionGeneration: 7}, "activate-1", "digest-1", now); err != nil {
+		t.Fatal(err)
+	}
+	second := NewSessionRegistry(path)
+	session, replay, err := second.Activate(&agentpb.AgentBinding{AgentId: "agent-1", CellId: "cell-1", ExpectedBootId: "boot-2", DispatcherEpoch: "epoch-2"}, "activate-2", "digest-2", now)
+	if err != nil || replay || session.GetSessionGeneration() != 8 {
+		t.Fatalf("Agent could not advance persisted fencing after restart: session=%v replay=%t err=%v", session, replay, err)
+	}
+	third := NewSessionRegistry(path)
+	if _, _, err := third.Activate(&agentpb.AgentBinding{AgentId: "agent-1", CellId: "cell-1", ExpectedBootId: "boot-3", DispatcherEpoch: "epoch-3", SessionGeneration: 7}, "activate-3", "digest-3", now); status.Code(err) != codes.Aborted {
+		t.Fatalf("explicit stale generation bypassed the durable fence: %v", err)
+	}
+}
+
+func TestSessionRegistryZeroGenerationDoesNotWrapAtExhaustion(t *testing.T) {
+	path := t.TempDir() + "/rpc-session.json"
+	first := NewSessionRegistry(path)
+	now := time.Unix(100, 0)
+	if _, _, err := first.Activate(&agentpb.AgentBinding{AgentId: "agent-1", CellId: "cell-1", ExpectedBootId: "boot-1", DispatcherEpoch: "epoch-1", SessionGeneration: ^uint64(0)}, "activate-1", "digest-1", now); err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range []struct {
+		name     string
+		registry *SessionRegistry
+		boot     string
+	}{
+		{"active", first, "boot-1"},
+		{"after restart", NewSessionRegistry(path), "boot-2"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			_, _, err := tc.registry.Activate(&agentpb.AgentBinding{AgentId: "agent-1", CellId: "cell-1", ExpectedBootId: tc.boot, DispatcherEpoch: "epoch-2"}, "activate-2", "digest-2", now)
+			if status.Code(err) != codes.Aborted || !strings.Contains(err.Error(), "exhausted") {
+				t.Fatalf("exhausted generation wrapped or was hidden: %v", err)
+			}
+		})
+	}
+}