H-162: confirm Shangwutong outbound delivery from echoes (#36)

* H-162: confirm Shangwutong sends from echoes

* fix(shangwutong): match echoes during send

* fix(shangwutong): preserve accepted delivery metric

---------

Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
2026-08-16 12:35:40 +08:00
committed by GitHub
co-authored by rogee
parent f604466d4a
commit d1b11ce3ec
12 changed files with 185 additions and 34 deletions
@@ -10,6 +10,37 @@ import (
"time"
)
const awaitOutboundEcho = `-- name: AwaitOutboundEcho :exec
UPDATE outbound_messages SET
delivery_status = 'uncertain',
external_id = NULL,
external_error_code = NULL,
status_sync_status = 'not_required',
claimed_at = NULL,
last_error = NULL,
updated_at = CURRENT_TIMESTAMP
WHERE id = ? AND delivery_status = 'delivering'
`
func (q *Queries) AwaitOutboundEcho(ctx context.Context, id int64) error {
_, err := q.db.ExecContext(ctx, awaitOutboundEcho, id)
return err
}
const awaitOutboundPartEcho = `-- name: AwaitOutboundPartEcho :exec
UPDATE outbound_parts SET
delivery_status = 'uncertain',
external_id = NULL,
last_error = NULL,
updated_at = CURRENT_TIMESTAMP
WHERE id = ? AND delivery_status = 'pending'
`
func (q *Queries) AwaitOutboundPartEcho(ctx context.Context, id int64) error {
_, err := q.db.ExecContext(ctx, awaitOutboundPartEcho, id)
return err
}
const claimOutboundMessage = `-- name: ClaimOutboundMessage :one
UPDATE outbound_messages SET
delivery_status = 'delivering',
@@ -199,7 +230,7 @@ UPDATE outbound_messages SET
SELECT 1 FROM outbound_parts
WHERE outbound_message_id = outbound_messages.id
AND delivery_status <> 'delivered'
) THEN 'pending'
) THEN 'uncertain'
ELSE 'delivered'
END,
result_version = result_version + 1,
@@ -214,10 +245,11 @@ UPDATE outbound_messages SET
ELSE 'pending'
END,
status_sync_next_at = NULL,
claimed_at = NULL,
last_error = NULL,
updated_at = CURRENT_TIMESTAMP
WHERE outbound_messages.id = ?2
AND delivery_status IN ('delivered', 'uncertain', 'failed')
AND delivery_status IN ('delivering', 'delivered', 'uncertain', 'failed')
AND (delivery_status <> 'failed' OR external_error_code = 'uncertain_timeout')
`
@@ -242,7 +274,7 @@ UPDATE outbound_parts SET
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
AND external_id IS NULL
AND delivery_status IN ('delivered', 'uncertain')
AND delivery_status IN ('pending', 'delivered', 'uncertain')
`
type ConfirmOutboundPartEchoParams struct {
@@ -476,13 +508,13 @@ const listOutboundEchoCandidates = `-- name: ListOutboundEchoCandidates :many
SELECT id, account_id, swt_sid, event_id, occurred_at, gochat_message_id, retry_version, message_type, content, payload, delivery_status, result_version, external_id, external_error_code, claimed_at, attempts, next_attempt_at, status_sync_status, status_sync_attempts, status_sync_next_at, status_reported_at, last_error, created_at, updated_at FROM outbound_messages
WHERE account_id = ?
AND swt_sid = ?
AND delivery_status IN ('delivered', 'uncertain', 'failed')
AND delivery_status IN ('delivering', 'delivered', 'uncertain', 'failed')
AND (delivery_status <> 'failed' OR external_error_code = 'uncertain_timeout')
AND EXISTS (
SELECT 1 FROM outbound_parts
WHERE outbound_message_id = outbound_messages.id
AND external_id IS NULL
AND delivery_status IN ('delivered', 'uncertain')
AND delivery_status IN ('pending', 'delivered', 'uncertain')
)
ORDER BY id
LIMIT 20
+25 -5
View File
@@ -29,6 +29,14 @@ UPDATE outbound_parts SET
updated_at = CURRENT_TIMESTAMP
WHERE id = ? AND delivery_status = 'pending';
-- name: AwaitOutboundPartEcho :exec
UPDATE outbound_parts SET
delivery_status = 'uncertain',
external_id = NULL,
last_error = NULL,
updated_at = CURRENT_TIMESTAMP
WHERE id = ? AND delivery_status = 'pending';
-- name: MarkOutboundPartUncertain :exec
UPDATE outbound_parts SET
delivery_status = 'uncertain',
@@ -110,6 +118,17 @@ UPDATE outbound_messages SET
updated_at = CURRENT_TIMESTAMP
WHERE id = ? AND delivery_status = 'delivering';
-- name: AwaitOutboundEcho :exec
UPDATE outbound_messages SET
delivery_status = 'uncertain',
external_id = NULL,
external_error_code = NULL,
status_sync_status = 'not_required',
claimed_at = NULL,
last_error = NULL,
updated_at = CURRENT_TIMESTAMP
WHERE id = ? AND delivery_status = 'delivering';
-- name: MarkOutboundUncertain :exec
UPDATE outbound_messages SET
delivery_status = 'uncertain',
@@ -230,13 +249,13 @@ WHERE id = ? AND status_sync_status = 'syncing';
SELECT * FROM outbound_messages
WHERE account_id = ?
AND swt_sid = ?
AND delivery_status IN ('delivered', 'uncertain', 'failed')
AND delivery_status IN ('delivering', 'delivered', 'uncertain', 'failed')
AND (delivery_status <> 'failed' OR external_error_code = 'uncertain_timeout')
AND EXISTS (
SELECT 1 FROM outbound_parts
WHERE outbound_message_id = outbound_messages.id
AND external_id IS NULL
AND delivery_status IN ('delivered', 'uncertain')
AND delivery_status IN ('pending', 'delivered', 'uncertain')
)
ORDER BY id
LIMIT 20;
@@ -248,7 +267,7 @@ UPDATE outbound_messages SET
SELECT 1 FROM outbound_parts
WHERE outbound_message_id = outbound_messages.id
AND delivery_status <> 'delivered'
) THEN 'pending'
) THEN 'uncertain'
ELSE 'delivered'
END,
result_version = result_version + 1,
@@ -263,10 +282,11 @@ UPDATE outbound_messages SET
ELSE 'pending'
END,
status_sync_next_at = NULL,
claimed_at = NULL,
last_error = NULL,
updated_at = CURRENT_TIMESTAMP
WHERE outbound_messages.id = sqlc.arg(id)
AND delivery_status IN ('delivered', 'uncertain', 'failed')
AND delivery_status IN ('delivering', 'delivered', 'uncertain', 'failed')
AND (delivery_status <> 'failed' OR external_error_code = 'uncertain_timeout');
-- name: ConfirmOutboundPartEcho :execrows
@@ -277,7 +297,7 @@ UPDATE outbound_parts SET
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
AND external_id IS NULL
AND delivery_status IN ('delivered', 'uncertain');
AND delivery_status IN ('pending', 'delivered', 'uncertain');
-- name: FailExpiredUncertainMessages :execrows
UPDATE outbound_messages SET
@@ -362,7 +362,7 @@ func (i *Inbound) confirmOutboundEcho(ctx context.Context, account *dbgen.Accoun
}
func outboundPartMatchesEcho(part *dbgen.OutboundPart, mapped mappedEvent, eventTime time.Time) bool {
if part == nil || part.ExternalID != nil || (part.DeliveryStatus != "delivered" && part.DeliveryStatus != "uncertain") {
if part == nil || part.ExternalID != nil || (part.DeliveryStatus != "pending" && part.DeliveryStatus != "delivered" && part.DeliveryStatus != "uncertain") {
return false
}
if delta := eventTime.Sub(part.UpdatedAt); delta < -echoMatchWindow || delta > echoMatchWindow {
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"sync"
"testing"
@@ -280,6 +281,48 @@ func TestInboundKind3ConfirmsUniqueOutboundEcho(t *testing.T) {
}
}
func TestAcceptedOutboundWaitsForEastEightKind3EchoBeforeSent(t *testing.T) {
ctx := context.Background()
database, account := deliveryDatabase(t, ctx)
if _, _, err := database.EnqueueOutbound(ctx, store.OutboundInput{
AccountID: account.ID, SWTSessionID: "visitor", EventID: "message:77:created", OccurredAt: time.Now(), GoChatMessageID: 77,
MessageType: "text", Content: stringPointer("hello"), Payload: deliveryPayload(t, nil),
}, false); err != nil {
t.Fatal(err)
}
results := &resultRecorder{}
outbound, _ := NewOutbound(database, sessionStub{}, senderStub{}, results, nil, 1)
if worked, err := outbound.processOutbound(ctx); err != nil || !worked {
t.Fatalf("accepted delivery = %v, %v", worked, err)
}
queued, err := database.Reader().GetOutboundByGoChatMessageID(ctx, 77)
if err != nil || queued.DeliveryStatus != "uncertain" || queued.StatusSyncStatus != "not_required" {
t.Fatalf("accepted outbound = %#v, %v", queued, err)
}
if worked, err := outbound.processStatus(ctx); err != nil || worked || results.result.Status != "" {
t.Fatalf("accepted send must remain progress: worked=%v err=%v result=%#v", worked, err, results.result)
}
const dotNetUnixEpochTicks int64 = 621355968000000000
localNow := time.Now().UTC().Add(8 * time.Hour)
ticks := dotNetUnixEpochTicks + localNow.Unix()*10_000_000 + int64(localNow.Nanosecond()/100)
persistInboundEvent(t, database, account, swt.HeartbeatEvent{
SessionID: "visitor", Kind: 3, OpName: "agent", Text: "hello", SeqID: 51,
Timestamp: strconv.FormatInt(ticks, 10), RawLine: "visitor 3 agent|hello 51 ticks",
})
inbound, _ := NewInbound(database, &inboundRecorder{}, nil, 1)
if worked, err := inbound.processInbound(ctx); err != nil || !worked {
t.Fatalf("process echo = %v, %v", worked, err)
}
queued, err = database.Reader().GetOutboundByGoChatMessageID(ctx, 77)
if err != nil || queued.DeliveryStatus != "delivered" || value(queued.ExternalID) != "51" || queued.StatusSyncStatus != "pending" {
t.Fatalf("confirmed outbound = %#v, %v", queued, err)
}
if worked, err := outbound.processStatus(ctx); err != nil || !worked || results.result.Status != "sent" || value(results.result.ExternalID) != "51" {
t.Fatalf("confirmed status = worked:%v err:%v result:%#v", worked, err, results.result)
}
}
func TestInboundKind3ConfirmsMultipleOutboundPartsIndependently(t *testing.T) {
ctx := context.Background()
database, account := deliveryDatabase(t, ctx)
@@ -751,10 +751,13 @@ func parseSWTTime(value string, fallback time.Time) time.Time {
}
}
if numeric, err := strconv.ParseInt(value, 10, 64); err == nil {
const dotNetUnixEpochTicks int64 = 621355968000000000
const (
dotNetUnixEpochTicks int64 = 621355968000000000
swtUTCOffset = 8 * time.Hour
)
if numeric >= dotNetUnixEpochTicks {
delta := numeric - dotNetUnixEpochTicks
return time.Unix(delta/10_000_000, (delta%10_000_000)*100).UTC()
return time.Unix(delta/10_000_000, (delta%10_000_000)*100).Add(-swtUTCOffset).UTC()
}
if numeric > 1_000_000_000_000 {
return time.UnixMilli(numeric).UTC()
@@ -1,6 +1,7 @@
package delivery
import (
"strconv"
"strings"
"testing"
"time"
@@ -260,9 +261,13 @@ func TestShangwutongLocationAttributesDefaultToChina(t *testing.T) {
}
func TestParseSWTTimeSupportsDotNetTicks(t *testing.T) {
got := parseSWTTime("639183484614616556", time.Time{})
if got.Year() != 2026 || got.Month() != time.June || got.Location() != time.UTC {
t.Fatalf("parsed time = %s", got)
const dotNetUnixEpochTicks int64 = 621355968000000000
localWallTime := time.Date(2026, time.August, 16, 12, 30, 0, 0, time.UTC)
ticks := dotNetUnixEpochTicks + localWallTime.Unix()*10_000_000
got := parseSWTTime(strconv.FormatInt(ticks, 10), time.Time{})
want := time.Date(2026, time.August, 16, 4, 30, 0, 0, time.UTC)
if !got.Equal(want) {
t.Fatalf("parsed time = %s, want %s", got, want)
}
}
@@ -201,13 +201,13 @@ func (o *Outbound) processOutbound(ctx context.Context) (bool, error) {
if err := o.sendPart(ctx, message, part); err != nil {
return true, o.handlePartError(message, part, err)
}
if err := o.store.Writer().CompleteOutboundPart(ctx, dbgen.CompleteOutboundPartParams{ID: part.ID}); err != nil {
if err := o.store.Writer().AwaitOutboundPartEcho(ctx, part.ID); err != nil {
return true, err
}
}
err = o.store.Writer().CompleteOutboundMessage(ctx, dbgen.CompleteOutboundMessageParams{ExternalID: nil, ID: message.ID})
err = o.store.Writer().AwaitOutboundEcho(ctx, message.ID)
if err == nil {
o.metrics.Delivery("outbound", "delivered")
o.metrics.Delivery("outbound", "accepted")
}
return true, err
}
@@ -15,7 +15,7 @@ import (
"github.com/gochat/gochat/channels/shangwutong/internal/swt"
)
func TestOutboundSuccessIsPersistedBeforeStatusSync(t *testing.T) {
func TestAcceptedOutboundTimesOutThroughExistingFailurePath(t *testing.T) {
ctx := context.Background()
database, account := deliveryDatabase(t, ctx)
payload := deliveryPayload(t, nil)
@@ -35,16 +35,52 @@ func TestOutboundSuccessIsPersistedBeforeStatusSync(t *testing.T) {
t.Fatalf("delivery = %v, %v", worked, err)
}
queued, err := database.Writer().GetOutboundByGoChatMessageID(ctx, 77)
if err != nil || queued.DeliveryStatus != "delivered" || queued.StatusSyncStatus != "pending" {
if err != nil || queued.DeliveryStatus != "uncertain" || queued.StatusSyncStatus != "not_required" {
t.Fatalf("queued = %#v, %v", queued, err)
}
worked, err = worker.processStatus(ctx)
if err != nil || !worked || results.result.Status != "sent" || results.inboxID != account.GochatInboxID {
if err != nil || worked || results.result.Status != "" {
t.Fatalf("status sync = %v, %v, %#v", worked, err, results)
}
queued, _ = database.Writer().GetOutboundByGoChatMessageID(ctx, 77)
if queued.StatusSyncStatus != "synced" {
t.Fatalf("status sync state = %s", queued.StatusSyncStatus)
if messages, _, err := worker.expireUncertain(ctx, time.Now().Add(uncertainObservationWindow+time.Second)); err != nil || messages != 1 {
t.Fatalf("expiry = %d, %v", messages, err)
}
if worked, err = worker.processStatus(ctx); err != nil || !worked || results.result.Status != "failed" || results.inboxID != account.GochatInboxID {
t.Fatalf("failed status sync = %v, %v, %#v", worked, err, results)
}
}
func TestOutboundEchoBeforeSendReturnsConfirmsDelivery(t *testing.T) {
ctx := context.Background()
database, account := deliveryDatabase(t, ctx)
if _, _, err := database.EnqueueOutbound(ctx, store.OutboundInput{
AccountID: account.ID, SWTSessionID: "visitor", EventID: "message:77:created", OccurredAt: time.Now(), GoChatMessageID: 77,
MessageType: "text", Content: stringPointer("hello"), Payload: deliveryPayload(t, nil),
}, false); err != nil {
t.Fatal(err)
}
client := &inboundRecorder{}
inbound, _ := NewInbound(database, client, nil, 1)
sender := senderStub{beforeReturn: func() {
persistInboundEvent(t, database, account, swt.HeartbeatEvent{
SessionID: "visitor", Kind: 3, OpName: "agent", Text: "hello", SeqID: 51,
Timestamp: time.Now().Format(time.RFC3339Nano), RawLine: "visitor 3 agent|hello 51 timestamp",
})
if worked, err := inbound.processInbound(ctx); err != nil || !worked {
t.Fatalf("process echo during send = %v, %v", worked, err)
}
}}
results := &resultRecorder{}
outbound, _ := NewOutbound(database, sessionStub{}, sender, results, nil, 1)
if worked, err := outbound.processOutbound(ctx); err != nil || !worked {
t.Fatalf("delivery = %v, %v", worked, err)
}
queued, err := database.Reader().GetOutboundByGoChatMessageID(ctx, 77)
if err != nil || queued.DeliveryStatus != "delivered" || value(queued.ExternalID) != "51" || len(client.imports) != 0 {
t.Fatalf("queued=%#v imports=%#v err=%v", queued, client.imports, err)
}
if worked, err := outbound.processStatus(ctx); err != nil || !worked || results.result.Status != "sent" {
t.Fatalf("status = %v, %v, %#v", worked, err, results.result)
}
}
@@ -257,7 +293,7 @@ func TestAcceptedTransferUpdatesOwnershipAndKeepsOutboundSendable(t *testing.T)
}
}
func TestOutboundRetrySkipsAlreadyDeliveredParts(t *testing.T) {
func TestOutboundRetryResendsPartsWithoutEcho(t *testing.T) {
ctx := context.Background()
database, account := deliveryDatabase(t, ctx)
content, dataURL, fileName, fileSize := "hello", "https://gochat.test/image.png", "image.png", int64(9)
@@ -280,7 +316,7 @@ func TestOutboundRetrySkipsAlreadyDeliveredParts(t *testing.T) {
}
queued, _ := database.Reader().GetOutboundByGoChatMessageID(ctx, 77)
parts, _ := database.Reader().ListOutboundParts(ctx, queued.ID)
if queued.DeliveryStatus != "failed" || parts[0].DeliveryStatus != "delivered" || parts[1].DeliveryStatus != "failed" {
if queued.DeliveryStatus != "failed" || parts[0].DeliveryStatus != "uncertain" || parts[1].DeliveryStatus != "failed" {
t.Fatalf("queued=%#v parts=%#v", queued, parts)
}
@@ -294,7 +330,7 @@ func TestOutboundRetrySkipsAlreadyDeliveredParts(t *testing.T) {
}
queued, _ = database.Reader().GetOutboundByGoChatMessageID(ctx, 77)
parts, _ = database.Reader().ListOutboundParts(ctx, queued.ID)
if sender.textCalls != 1 || sender.imageCalls != 2 || queued.DeliveryStatus != "delivered" || parts[0].DeliveryStatus != "delivered" || parts[1].DeliveryStatus != "delivered" {
if sender.textCalls != 2 || sender.imageCalls != 2 || queued.DeliveryStatus != "uncertain" || parts[0].DeliveryStatus != "uncertain" || parts[1].DeliveryStatus != "uncertain" {
t.Fatalf("sender=%#v queued=%#v parts=%#v", sender, queued, parts)
}
}
@@ -398,9 +434,15 @@ func (s *sessionRecorder) InvalidateSession(context.Context, int64) error {
return nil
}
type senderStub struct{ err error }
type senderStub struct {
err error
beforeReturn func()
}
func (s senderStub) SendText(context.Context, swt.Session, string, string) (swt.SendResult, error) {
if s.beforeReturn != nil {
s.beforeReturn()
}
return swt.SendResult{Status: "ok"}, s.err
}
@@ -52,7 +52,7 @@ func (m *Metrics) Presence(result string) {
func (m *Metrics) Delivery(direction, result string) {
m.inc("swt_connector_delivery_total", labels(
"direction", bounded(direction, "inbound", "outbound"),
"result", bounded(result, "delivered", "retry", "uncertain", "failed", "ambiguous"),
"result", bounded(result, "accepted", "delivered", "retry", "uncertain", "failed", "ambiguous"),
))
}
@@ -13,6 +13,7 @@ func TestMetricsRenderProductionSeriesWithoutHighCardinalityLabels(t *testing.T)
metrics.Login("success")
metrics.Presence("retryable_error")
metrics.Delivery("inbound", "delivered")
metrics.Delivery("outbound", "accepted")
metrics.StatusSync("success", "sent")
metrics.Mapping(2, "native_message", "delivered")
metrics.Mapping(24, "contact_attributes", "delivered")
@@ -49,6 +50,9 @@ func TestMetricsRenderProductionSeriesWithoutHighCardinalityLabels(t *testing.T)
if !strings.Contains(payload, `swt_connector_event_mapping_total{kind="24",strategy="contact_attributes",result="delivered"} 1`) {
t.Fatalf("redacted kind=24 mapping diagnostic missing from:\n%s", payload)
}
if !strings.Contains(payload, `swt_connector_delivery_total{direction="outbound",result="accepted"} 1`) {
t.Fatalf("accepted outbound delivery metric missing from:\n%s", payload)
}
}
func TestMetricKindKeepsEveryDocumentedKindBounded(t *testing.T) {
+5 -3
View File
@@ -179,7 +179,7 @@ type SendResult struct {
}
func (c *Client) SendText(ctx context.Context, session Session, sid, text string) (SendResult, error) {
return c.sendHTML(ctx, session, sid, wrapHTML(text), "send_text")
return c.sendHTML(ctx, session, sid, textHTML(text), "send_text")
}
func (c *Client) sendHTML(ctx context.Context, session Session, sid, content, operation string) (SendResult, error) {
@@ -257,13 +257,15 @@ func sessionAuthForm(session Session) url.Values {
}
}
func wrapHTML(text string) string {
func textHTML(text string) string {
text = html.EscapeString(text)
text = strings.ReplaceAll(text, "\r\n", "\n")
text = strings.ReplaceAll(text, "\r", "\n")
return "<P>" + strings.ReplaceAll(text, "\n", "<BR>") + "</P>"
return strings.ReplaceAll(text, "\n", "<BR>")
}
func wrapHTML(text string) string { return "<P>" + textHTML(text) + "</P>" }
func (c *Client) postForm(ctx context.Context, baseURL, endpoint string, form url.Values) (string, *http.Response, error) {
body, response, _, err := c.postFormTracked(ctx, baseURL, endpoint, form)
return body, response, err
@@ -56,7 +56,7 @@ func TestClientSendTextEscapesHTML(t *testing.T) {
if err := request.ParseForm(); err != nil {
t.Fatal(err)
}
if got := request.Form.Get("html"); got != "<P>&lt;b&gt;x&lt;/b&gt;<BR>next</P>" {
if got := request.Form.Get("html"); got != "&lt;b&gt;x&lt;/b&gt;<BR>next" {
t.Fatalf("html = %q", got)
}
response.Header().Set("r", "ok")