fix: restore web widget realtime replies (HH-556) (#136)

* fix: restore web widget realtime replies (HH-556)

* fix: make realtime delivery durable (HH-556)

* fix: make realtime message outbox atomic (HH-556)

---------

Co-authored-by: Rogee <rogee@ipao.vip>

backend/internal/handler/ws/ws_test.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -28,10 +30,38 @@ import (
 	"github.com/gochat/gochat/internal/model"
 	"github.com/gochat/gochat/internal/repository"
 	"github.com/gochat/gochat/internal/service"
+	"github.com/gochat/gochat/internal/worker"
 	wspkg "github.com/gochat/gochat/internal/ws"
 	"github.com/gochat/gochat/internal/wsevent"
 )
 
+type failTokenPublishOnceHook struct {
+	mu      sync.Mutex
+	channel string
+	failed  bool
+}
+
+func (h *failTokenPublishOnceHook) DialHook(next redis.DialHook) redis.DialHook { return next }
+
+func (h *failTokenPublishOnceHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
+	return func(ctx context.Context, cmd redis.Cmder) error {
+		if cmd.Name() == "publish" && len(cmd.Args()) >= 2 && fmt.Sprint(cmd.Args()[1]) == h.channel {
+			h.mu.Lock()
+			if !h.failed {
+				h.failed = true
+				h.mu.Unlock()
+				return errors.New("token room unavailable")
+			}
+			h.mu.Unlock()
+		}
+		return next(ctx, cmd)
+	}
+}
+
+func (h *failTokenPublishOnceHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
+	return next
+}
+
 // --- Protocol Tests ---
 
 func TestCommandType_Constants(t *testing.T) {
@@ -288,6 +318,79 @@ func TestHub_SendToAccount(t *testing.T) {
 	}
 }
 
+func TestDurableWidgetRetryDoesNotDuplicateRealHubSubscribers(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s-%d?mode=memory&cache=shared", t.Name(), time.Now().UnixNano())), &gorm.Config{})
+	require.NoError(t, err)
+	require.NoError(t, db.AutoMigrate(&model.BackgroundJob{}))
+	sqlDB, err := db.DB()
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
+	mini := miniredis.RunT(t)
+	rdb := redis.NewClient(&redis.Options{Addr: mini.Addr()})
+	rdb.AddHook(&failTokenPublishOnceHook{channel: wspkg.RedisPrefixRoom + "pubsub_token_visitor"})
+	t.Cleanup(func() { require.NoError(t, rdb.Close()) })
+
+	hub := NewHubSimple()
+	dashboard := NewClient(1, 1, nil, hub)
+	dashboard.Identifier = `{"channel":"AccountChannel","account_id":1}`
+	hub.Register(dashboard)
+	visitor := NewClient(0, 1, nil, hub)
+	visitor.IsContact = true
+	visitor.PubsubToken = "visitor"
+	visitor.Identifier = `{"channel":"RoomChannel","pubsub_token":"visitor"}`
+	hub.Register(visitor)
+
+	relay := wspkg.NewBroadcastRelay(rdb, hub)
+	ctx, cancel := context.WithCancel(context.Background())
+	require.NoError(t, relay.Start(ctx))
+	t.Cleanup(func() {
+		cancel()
+		require.NoError(t, relay.Stop())
+	})
+	pool := worker.NewWorkerPoolWithOptions(db, worker.WithBackoff(func(int) time.Duration { return 0 }))
+	publisher := wspkg.NewEventPublisher(hub, nil, relay)
+	publisher.SetWorkerPool(pool)
+	require.NoError(t, publisher.PublishWidgetEvent(1, "visitor", wspkg.EventMessageCreated, map[string]any{"id": 7}))
+
+	processed, err := pool.ProcessOne(context.Background())
+	require.True(t, processed)
+	require.NoError(t, err)
+	select {
+	case <-dashboard.Send:
+	case <-time.After(time.Second):
+		t.Fatal("account subscriber did not receive message.created")
+	}
+
+	processed, err = pool.ProcessOne(context.Background())
+	require.True(t, processed)
+	require.ErrorContains(t, err, "token room unavailable")
+	select {
+	case <-visitor.Send:
+		t.Fatal("token subscriber received failed delivery")
+	default:
+	}
+
+	processed, err = pool.ProcessOne(context.Background())
+	require.True(t, processed)
+	require.NoError(t, err)
+	select {
+	case <-visitor.Send:
+	case <-time.After(time.Second):
+		t.Fatal("token subscriber did not receive recovered message.created")
+	}
+	time.Sleep(20 * time.Millisecond)
+	select {
+	case <-dashboard.Send:
+		t.Fatal("account subscriber received duplicate message.created")
+	default:
+	}
+	select {
+	case <-visitor.Send:
+		t.Fatal("token subscriber received duplicate message.created")
+	default:
+	}
+}
+
 func TestHub_SendToAccountSanitizesVisitorIdentity(t *testing.T) {
 	hub := NewHubSimple()
 	agent := NewClient(1, 10, nil, hub)
@@ -721,6 +824,8 @@ func TestDashboardOutgoingReachesDashboardAndReconnectedWidget(t *testing.T) {
 	))
 	account := &model.Account{Name: "Realtime account", Active: true}
 	require.NoError(t, db.Create(account).Error)
+	user := &model.User{Base: model.Base{ID: 7}, Name: "Agent", Email: "agent@example.com", Provider: "local", Active: true}
+	require.NoError(t, db.Create(user).Error)
 	inbox := &model.Inbox{AccountID: account.ID, Name: "Website", ChannelType: "web_widget", Enabled: true}
 	require.NoError(t, db.Create(inbox).Error)
 	contact := &model.Contact{AccountID: account.ID, Name: "Visitor"}
@@ -758,7 +863,6 @@ func TestDashboardOutgoingReachesDashboardAndReconnectedWidget(t *testing.T) {
 	server := httptest.NewServer(router)
 	t.Cleanup(server.Close)
 
-	user := &model.User{Base: model.Base{ID: 7}, Provider: "local"}
 	tokenPair, err := jwtService.GenerateTokenPair(user, account.ID, "agent")
 	require.NoError(t, err)
 	dashboard := dialCable(t, server.URL, "?token="+tokenPair.AccessToken)