376 lines
13 KiB
Go
376 lines
13 KiB
Go
package api
|
|
|
|
// 平台侧网关出站通道：网关主动连入的 WS 会话、统一任务模型（持久化 + 下发 + 结果等待）
|
|
// 与事件流路由。方向反转后平台不再拨号连网关；设计见 docs/gateway-outbound-channel-plan.md。
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
|
hub "git.ipao.vip/rogee/creator-hub/internal/environment"
|
|
"github.com/fasthttp/websocket"
|
|
"github.com/gofiber/fiber/v3"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// gatewayChannelStore 是出站通道所需的存储能力；生产实现为 *hub.Store，测试用内存桩。
|
|
type gatewayChannelStore interface {
|
|
CreateGatewayTask(ctx context.Context, gatewayID int64, method, path string, payload []byte, timeout time.Duration) (hub.GatewayTask, error)
|
|
DispatchGatewayTask(ctx context.Context, taskID int64) (bool, error)
|
|
FinishGatewayTask(ctx context.Context, taskID int64, status string, responseStatus int, taskError string, responseBytes int64) (bool, error)
|
|
InterruptDispatchedTasks(ctx context.Context, gatewayID int64, reason string) error
|
|
NextPendingGatewayTasks(ctx context.Context, gatewayID int64, limit int) ([]hub.GatewayTask, error)
|
|
ResetGatewayConnections(ctx context.Context) error
|
|
TouchGatewaySession(ctx context.Context, gatewayID int64, version string) error
|
|
SetGatewayOnline(ctx context.Context, gatewayID int64, online bool) error
|
|
}
|
|
|
|
// gatewayChannelStoreHolder 由应用启动时注入生产 store；测试可直接替换。
|
|
var gatewayChannelStoreHolder gatewayChannelStore
|
|
|
|
// SetGatewayChannelStore 注入通道存储实现。
|
|
func SetGatewayChannelStore(store gatewayChannelStore) { gatewayChannelStoreHolder = store }
|
|
|
|
func channelStore() gatewayChannelStore {
|
|
if gatewayChannelStoreHolder == nil {
|
|
panic("gateway channel store is not configured")
|
|
}
|
|
return gatewayChannelStoreHolder
|
|
}
|
|
|
|
// TaskResult 是任务 result 帧的载荷。
|
|
type TaskResult struct {
|
|
Status int `json:"status"`
|
|
Body []byte `json:"body"`
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
// EventBatch 是网关事件投递帧的载荷；语义沿用原 /v1/channel deliveries。
|
|
type EventBatch struct {
|
|
Subscription string `json:"subscription"`
|
|
Alias string `json:"alias"`
|
|
Deliveries []creator.ListenerDelivery `json:"deliveries"`
|
|
}
|
|
|
|
// gatewaySession 是一个网关的活动出站连接。
|
|
type gatewaySession struct {
|
|
gateway hub.Gateway
|
|
conn *websocket.Conn
|
|
write sync.Mutex
|
|
done chan struct{}
|
|
finish sync.Once
|
|
|
|
mu sync.Mutex
|
|
waiter chan TaskResult // 当前在等结果的同步任务（单连接串行下发场景按 id 匹配）
|
|
events map[string]chan EventBatch // alias -> 事件队列
|
|
}
|
|
|
|
var gatewaySessions = struct {
|
|
sync.Mutex
|
|
items map[int64]*gatewaySession
|
|
}{items: map[int64]*gatewaySession{}}
|
|
|
|
func sessionByGateway(gatewayID int64) *gatewaySession {
|
|
gatewaySessions.Lock()
|
|
defer gatewaySessions.Unlock()
|
|
return gatewaySessions.items[gatewayID]
|
|
}
|
|
|
|
func (s *gatewaySession) send(value any) error {
|
|
s.write.Lock()
|
|
defer s.write.Unlock()
|
|
select {
|
|
case <-s.done:
|
|
return errors.New("gateway session disconnected")
|
|
default:
|
|
}
|
|
if err := s.conn.SetWriteDeadline(time.Now().Add(10 * time.Second)); err != nil {
|
|
return err
|
|
}
|
|
return s.conn.WriteJSON(value)
|
|
}
|
|
|
|
type taskFrame struct {
|
|
Type string `json:"type"`
|
|
ID int64 `json:"id"`
|
|
Method string `json:"method"`
|
|
Path string `json:"path"`
|
|
Payload json.RawMessage `json:"payload"`
|
|
}
|
|
|
|
type resultFrame struct {
|
|
Type string `json:"type"`
|
|
ID int64 `json:"id"`
|
|
TaskResult
|
|
}
|
|
|
|
// dispatchGatewayTask 创建持久化任务；网关在线时立即下发并（按需）等待结果。
|
|
// wait=false 仅入队/补发，由通道下发器异步推进。
|
|
func dispatchGatewayTask(ctx context.Context, gateway hub.Gateway, method, path string, body any, timeout time.Duration, wait bool) (int, []byte, error) {
|
|
payload, err := json.Marshal(body)
|
|
if err != nil {
|
|
return 0, nil, fmt.Errorf("encode gateway task payload: %w", err)
|
|
}
|
|
task, err := channelStore().CreateGatewayTask(ctx, gateway.ID, method, path, payload, timeout)
|
|
if err != nil {
|
|
return 0, nil, fmt.Errorf("persist gateway task: %w", err)
|
|
}
|
|
session := sessionByGateway(gateway.ID)
|
|
if session == nil {
|
|
// 同步等待的任务立即终态化；异步任务保持 pending 由下发器补发。
|
|
if wait {
|
|
if _, markErr := channelStore().FinishGatewayTask(ctx, task.ID, hub.TaskFailed, 0, "网关离线", 0); markErr != nil {
|
|
logrus.WithError(markErr).Error("persist gateway offline task failure failed")
|
|
}
|
|
}
|
|
return 0, nil, hub.ErrGatewayOffline
|
|
}
|
|
result, err := session.execute(ctx, task, wait)
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
return result.Status, result.Body, nil
|
|
}
|
|
|
|
// execute 下发单个任务并等待结果；下发失败立即终态化，等待按任务超时。
|
|
func (s *gatewaySession) execute(ctx context.Context, task hub.GatewayTask, wait bool) (*TaskResult, error) {
|
|
var waiter chan TaskResult
|
|
if _, err := channelStore().DispatchGatewayTask(ctx, task.ID); err != nil {
|
|
return nil, fmt.Errorf("mark gateway task dispatched: %w", err)
|
|
}
|
|
frame := taskFrame{Type: "task", ID: task.ID, Method: task.Method, Path: task.Path, Payload: task.Payload}
|
|
if wait {
|
|
// 先建 waiter 再下发：网关可能秒回 result，晚建会丢结果导致同步调用假超时。
|
|
s.mu.Lock()
|
|
s.waiter = make(chan TaskResult, 1)
|
|
waiter = s.waiter
|
|
s.mu.Unlock()
|
|
defer func() {
|
|
s.mu.Lock()
|
|
if s.waiter == waiter {
|
|
s.waiter = nil
|
|
}
|
|
s.mu.Unlock()
|
|
}()
|
|
}
|
|
if err := s.send(frame); err != nil {
|
|
_, _ = channelStore().FinishGatewayTask(ctx, task.ID, hub.TaskFailed, 0, "任务下发失败："+err.Error(), 0)
|
|
return nil, fmt.Errorf("send gateway task: %w", err)
|
|
}
|
|
if !wait {
|
|
return &TaskResult{}, nil
|
|
}
|
|
waitCtx, cancel := context.WithTimeout(ctx, time.Duration(task.TimeoutSeconds)*time.Second)
|
|
defer cancel()
|
|
for {
|
|
select {
|
|
case r := <-waiter:
|
|
if r.Error != "" {
|
|
_, _ = channelStore().FinishGatewayTask(ctx, task.ID, hub.TaskFailed, r.Status, r.Error, int64(len(r.Body)))
|
|
return nil, fmt.Errorf("gateway task %d %s %s failed: %s", task.ID, task.Method, task.Path, r.Error)
|
|
}
|
|
_, _ = channelStore().FinishGatewayTask(ctx, task.ID, hub.TaskSucceeded, r.Status, "", int64(len(r.Body)))
|
|
return &r, nil
|
|
case <-s.done:
|
|
_, _ = channelStore().FinishGatewayTask(ctx, task.ID, hub.TaskFailed, 0, "网关连接中断，结果未确认", 0)
|
|
return nil, errors.New("gateway session disconnected")
|
|
case <-waitCtx.Done():
|
|
_, _ = channelStore().FinishGatewayTask(ctx, task.ID, hub.TaskFailed, 0, "网关任务等待超时", 0)
|
|
return nil, fmt.Errorf("gateway task %d timed out after %ds", task.ID, task.TimeoutSeconds)
|
|
}
|
|
}
|
|
}
|
|
|
|
// receiveLoop 读取网关帧：result 唤醒等待者，event 路由到订阅队列，heartbeat 刷新健康。
|
|
func (s *gatewaySession) receiveLoop(store gatewaySessionStore) {
|
|
for {
|
|
_ = s.conn.SetReadDeadline(time.Now().Add(60 * time.Second))
|
|
var frame struct {
|
|
Type string `json:"type"`
|
|
ID int64 `json:"id"`
|
|
Version string `json:"version"`
|
|
*TaskResult
|
|
Event *EventBatch `json:"event"`
|
|
}
|
|
if err := s.conn.ReadJSON(&frame); err != nil {
|
|
s.fail(fmt.Sprintf("网关连接断开：%v", err), store)
|
|
return
|
|
}
|
|
switch frame.Type {
|
|
case "result":
|
|
s.mu.Lock()
|
|
waiter := s.waiter
|
|
s.mu.Unlock()
|
|
if waiter != nil && frame.TaskResult != nil {
|
|
select {
|
|
case waiter <- *frame.TaskResult:
|
|
default:
|
|
}
|
|
}
|
|
case "heartbeat":
|
|
if err := store.TouchGatewaySession(context.Background(), s.gateway.ID, frame.Version); err != nil {
|
|
logrus.WithError(err).Warn("gateway heartbeat persistence failed")
|
|
}
|
|
case "event":
|
|
if frame.Event != nil {
|
|
s.deliverEvent(*frame.Event)
|
|
}
|
|
default:
|
|
s.fail("网关通道消息类型无效:"+frame.Type, store)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *gatewaySession) deliverEvent(batch EventBatch) {
|
|
s.mu.Lock()
|
|
queue := s.events[batch.Alias]
|
|
s.mu.Unlock()
|
|
if queue == nil {
|
|
// 无订阅者的投递直接确认丢弃：worker 注册订阅后才允许网关启动监听，
|
|
// 该场景只出现在 worker 刚失败重启的间隙，批次仍留在网关待重推。
|
|
logrus.WithFields(logrus.Fields{"gateway": s.gateway.Name, "alias": batch.Alias}).Warn("gateway event batch has no subscriber; asking gateway to retain it")
|
|
_ = s.send(map[string]any{"type": "event_nack", "subscription": batch.Subscription, "alias": batch.Alias})
|
|
return
|
|
}
|
|
select {
|
|
case queue <- batch:
|
|
default:
|
|
// 单账号消费过载只隔离该订阅；批次未确认，网关保留并重推。
|
|
logrus.WithFields(logrus.Fields{"gateway": s.gateway.Name, "alias": batch.Alias}).Error("gateway event queue overflow; isolating subscription")
|
|
s.removeEventQueue(batch.Alias)
|
|
s.failSubscription(batch.Alias, queue)
|
|
}
|
|
}
|
|
|
|
func (s *gatewaySession) failSubscription(alias string, queue chan EventBatch) {
|
|
queue <- EventBatch{Subscription: "", Alias: alias}
|
|
}
|
|
|
|
func (s *gatewaySession) removeEventQueue(alias string) {
|
|
s.mu.Lock()
|
|
delete(s.events, alias)
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
func (s *gatewaySession) fail(reason string, store gatewaySessionStore) {
|
|
s.finish.Do(func() {
|
|
close(s.done)
|
|
_ = s.conn.Close()
|
|
gatewaySessions.Lock()
|
|
if gatewaySessions.items[s.gateway.ID] == s {
|
|
delete(gatewaySessions.items, s.gateway.ID)
|
|
}
|
|
gatewaySessions.Unlock()
|
|
logrus.WithFields(logrus.Fields{"gateway": s.gateway.Name, "reason": reason}).Error("gateway session closed")
|
|
if store != nil {
|
|
_ = store.InterruptDispatchedTasks(context.Background(), s.gateway.ID, reason)
|
|
_ = store.SetGatewayOnline(context.Background(), s.gateway.ID, false)
|
|
}
|
|
})
|
|
}
|
|
|
|
// subscribeEvents 注册某账号事件队列；订阅先于监听启动任务，保证网关首批投递不丢失。
|
|
func (s *gatewaySession) subscribeEvents(alias string) <-chan EventBatch {
|
|
queue := make(chan EventBatch, 64)
|
|
s.mu.Lock()
|
|
s.events[alias] = queue
|
|
s.mu.Unlock()
|
|
return queue
|
|
}
|
|
|
|
func (s *gatewaySession) unsubscribeEvents(alias string) {
|
|
s.mu.Lock()
|
|
queue := s.events[alias]
|
|
delete(s.events, alias)
|
|
s.mu.Unlock()
|
|
if queue != nil {
|
|
close(queue)
|
|
}
|
|
}
|
|
|
|
func (s *gatewaySession) acknowledgeEvents(alias string, deliveryIDs []string) error {
|
|
return s.send(map[string]any{"type": "event_ack", "alias": alias, "delivery_ids": deliveryIDs})
|
|
}
|
|
|
|
// HandleGatewayChannel 对外暴露网关出站通道 WS 路由注册。
|
|
func HandleGatewayChannel(app *fiber.App, store *hub.Store) {
|
|
handleGatewayChannel(app, store)
|
|
}
|
|
|
|
// gatewaySessionStore 是会话读循环所需的存储能力；生产实现为 *hub.Store。
|
|
type gatewaySessionStore interface {
|
|
TouchGatewaySession(ctx context.Context, gatewayID int64, version string) error
|
|
SetGatewayOnline(ctx context.Context, gatewayID int64, online bool) error
|
|
InterruptDispatchedTasks(ctx context.Context, gatewayID int64, reason string) error
|
|
GetGatewayByAccessKey(ctx context.Context, accessKey string) (hub.Gateway, error)
|
|
}
|
|
|
|
// serveGatewaySession 完成认证、会话注册与读循环；fiber 路由与测试共用。
|
|
func serveGatewaySession(conn *websocket.Conn, authorization string, store gatewaySessionStore) {
|
|
gateway, err := store.GetGatewayByAccessKey(context.Background(), authorization)
|
|
if err != nil {
|
|
logrus.Warn("gateway agent rejected: invalid access key")
|
|
_ = conn.Close()
|
|
return
|
|
}
|
|
session := &gatewaySession{gateway: gateway, conn: conn, done: make(chan struct{}), events: map[string]chan EventBatch{}}
|
|
gatewaySessions.Lock()
|
|
gatewaySessions.items[gateway.ID] = session
|
|
gatewaySessions.Unlock()
|
|
if err := store.SetGatewayOnline(context.Background(), gateway.ID, true); err != nil {
|
|
logrus.WithError(err).Warn("gateway online persistence failed")
|
|
}
|
|
logrus.WithFields(logrus.Fields{"gateway": gateway.Name, "gateway_id": gateway.ID}).Info("gateway agent connected")
|
|
session.receiveLoop(store)
|
|
}
|
|
|
|
// handleGatewayChannel 处理网关出站 WS 升级与会话生命周期。
|
|
func handleGatewayChannel(app *fiber.App, store *hub.Store) {
|
|
app.Get("/v1/agent", func(c fiber.Ctx) error {
|
|
upgrader := websocket.FastHTTPUpgrader{}
|
|
return upgrader.Upgrade(c.RequestCtx(), func(conn *websocket.Conn) {
|
|
authorization := string(c.RequestCtx().Request.Header.Peek("Authorization"))
|
|
serveGatewaySession(conn, authorization, store)
|
|
})
|
|
})
|
|
}
|
|
|
|
// RunGatewayTaskDispatcher 周期扫描 pending 任务，对在线网关按 id 序补发。
|
|
func RunGatewayTaskDispatcher(ctx context.Context, store *hub.Store) {
|
|
for ctx.Err() == nil {
|
|
gateways, err := store.ListGateways(ctx)
|
|
if err != nil {
|
|
logrus.WithError(err).Error("list gateways for task dispatch failed")
|
|
} else {
|
|
for _, gateway := range gateways {
|
|
session := sessionByGateway(gateway.ID)
|
|
if session == nil {
|
|
continue
|
|
}
|
|
tasks, err := store.NextPendingGatewayTasks(ctx, gateway.ID, 64)
|
|
if err != nil {
|
|
logrus.WithError(err).WithField("gateway", gateway.Name).Error("read pending gateway tasks failed")
|
|
continue
|
|
}
|
|
for _, task := range tasks {
|
|
if _, err := session.execute(ctx, task, false); err != nil {
|
|
logrus.WithError(err).WithField("gateway", gateway.Name).Error("dispatch pending gateway task failed")
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(2 * time.Second):
|
|
}
|
|
}
|
|
}
|