fix: preserve opaque dialogue ownership and project typed diagnostics
This commit is contained in:
@@ -5,7 +5,9 @@ import (
|
||||
"io"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
@@ -117,22 +119,52 @@ func redactForm(body string) (string, bool) {
|
||||
var textFieldLine = regexp.MustCompile(`^\s*([A-Za-z_][A-Za-z0-9_.-]*)\s*[:=]`)
|
||||
var textFieldLabel = regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_.-]*)\s*[:=]`)
|
||||
|
||||
var diagnosticUUID = regexp.MustCompile(`^(?:call-)?[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||||
|
||||
// Free text has no reliable closing delimiter for content. Project only typed
|
||||
// diagnostic scalars from its tail; those never release content ownership.
|
||||
// Arbitrary field values and message schemas remain opaque, not guessed safe.
|
||||
func diagnosticTextLine(line string, contentIndent int) bool {
|
||||
field := textFieldLine.FindStringSubmatchIndex(line)
|
||||
if len(field) == 0 || len(line)-len(strings.TrimLeft(line, " \t")) > contentIndent {
|
||||
return false
|
||||
}
|
||||
key := normalizedField(line[field[2]:field[3]])
|
||||
value := strings.TrimSpace(line[field[1]:])
|
||||
switch key {
|
||||
case "taskid", "tenantid", "trunkid", "attempts", "remaining", "retryafter", "count", "elapsedms", "durationms":
|
||||
_, err := strconv.ParseUint(value, 10, 64)
|
||||
return err == nil
|
||||
case "httpstatus":
|
||||
n, err := strconv.ParseUint(value, 10, 16)
|
||||
return err == nil && n >= 100 && n <= 599
|
||||
case "eventid", "executionid":
|
||||
return diagnosticUUID.MatchString(value)
|
||||
case "nextat":
|
||||
_, err := time.Parse(time.RFC3339Nano, value)
|
||||
return err == nil
|
||||
case "status":
|
||||
switch value {
|
||||
case "ready", "retry_pending", "pending", "running", "paused", "stopped", "finished", "rejected", "ended", "failed", "unknown", "accepted", "completed", "succeeded", "waiting", "active", "inactive":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func redactConversationLines(body string) string {
|
||||
lines := strings.SplitAfter(body, "\n")
|
||||
hiding := false
|
||||
contentIndent := 0
|
||||
for i, line := range lines {
|
||||
// Whitespace separates paragraphs inside a conversation, not records.
|
||||
// Ownership ends only at the explicit non-conversation field below.
|
||||
field := textFieldLine.FindStringSubmatch(line)
|
||||
// Content owns its full tail, including unknown fields, speaker names,
|
||||
// nested message metadata and blank paragraphs. Typed diagnostics are a
|
||||
// projection of individual lines, never an inferred end of the dialogue.
|
||||
if hiding {
|
||||
// Continuation lines and conversation speaker labels are not business
|
||||
// fields. A distinct non-conversation key starts the next ordinary field.
|
||||
speaker := len(field) > 0 && (strings.EqualFold(field[1], "user") || strings.EqualFold(field[1], "assistant") || strings.EqualFold(field[1], "system") || strings.EqualFold(field[1], "agent"))
|
||||
if len(field) == 0 || speaker {
|
||||
if !diagnosticTextLine(line, contentIndent) {
|
||||
lines[i] = ""
|
||||
continue
|
||||
}
|
||||
hiding = false
|
||||
continue
|
||||
}
|
||||
// Labels establish ownership wherever they occur in an error chain, not
|
||||
// only at line start. Preserve the preceding actual failure description.
|
||||
@@ -147,6 +179,7 @@ func redactConversationLines(body string) string {
|
||||
}
|
||||
// Credentials own only this line; content owns the following paragraphs.
|
||||
hiding = conversationField(key)
|
||||
contentIndent = len(line) - len(strings.TrimLeft(line, " \t"))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,39 @@ func TestContentFieldClassificationAndEveryLoggingPathAgree(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownDialogueFieldsCannotEndContentOwnership(t *testing.T) {
|
||||
for _, dialogue := range []string{
|
||||
"role: user\nmessage: private-user-turn\nrole: assistant\nmessage: private-assistant-turn",
|
||||
"human: private-user-turn\nassistant: private-assistant-turn",
|
||||
"participant_42: private-user-turn\nparticipant_99: private-assistant-turn",
|
||||
"arbitrary_attribute: private-user-turn\n arbitrary_nested_attribute: private-assistant-turn",
|
||||
" status: private-user-turn\n message: private-assistant-turn",
|
||||
"role: user\nstatus=ready\nmessage: private-user-turn\nattempts=2\nhuman: private-assistant-turn",
|
||||
"status: private-user-turn\nmessage: private-assistant-turn",
|
||||
} {
|
||||
for _, label := range []string{"messages", "conversation", "history", "prompt"} {
|
||||
value := "upstream failed: " + label + ":\n" + dialogue + "\nstatus=retry_pending\ntask_id=4\nattempts=2"
|
||||
for _, debug := range []bool{false, true} {
|
||||
out := installTestLogger(t, debug)
|
||||
slog.Error("request failed", "error", errors.New(value), "detail", value)
|
||||
slog.Info(value, "stage", "failure_observed")
|
||||
TextBody(t.Context(), "HTTP text observed", []byte(value), "transport", "http")
|
||||
text := out.String()
|
||||
for _, secret := range []string{"private-user-turn", "private-assistant-turn"} {
|
||||
if strings.Contains(text, secret) {
|
||||
t.Fatalf("dialogue %q label %s debug %v leak: %s", dialogue, label, debug, text)
|
||||
}
|
||||
}
|
||||
for _, keep := range []string{"upstream failed", "status=retry_pending", "task_id=4", "attempts=2"} {
|
||||
if !strings.Contains(text, keep) {
|
||||
t.Fatal("business diagnostic lost", text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefixedContentLabelsKeepReasonAndOwnFollowingDialogue(t *testing.T) {
|
||||
for _, prefix := range []string{"upstream failed: ", "wrapped error: upstream failed: ", "upstream failed; ", "上游失败：upstream failed: "} {
|
||||
for _, field := range []string{"conversation", "history", "messages", "system_message", "prompt", "final_transcript"} {
|
||||
@@ -185,7 +218,7 @@ func TestReadableMarkupFormAndMultilineConversationAreSanitized(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestReadablePlainHTTPBodiesAreObservedSafelyWithoutChangingBytes(t *testing.T) {
|
||||
body := "status=ready\ntask_id=4\nremaining=27\nmessages:\n\nuser: private-http-user-turn\n\nassistant: private-http-assistant-turn\n\nstatus=ready\nAuthorization: Basic dXNlcjpwYXNzd29yZA==\napi_key=private-key\n-----BEGIN PRIVATE KEY-----\nprivate-pem-material\n-----END PRIVATE KEY-----\nretry_after=60\n" + strings.Repeat("business-data ", 12000)
|
||||
body := strings.Repeat("business-data ", 12000) + "\nstatus=ready\ntask_id=4\nremaining=27\nmessages:\n\nuser: private-http-user-turn\n\nassistant: private-http-assistant-turn\n\nstatus=ready\nAuthorization: Basic dXNlcjpwYXNzd29yZA==\napi_key=private-key\n-----BEGIN PRIVATE KEY-----\nprivate-pem-material\n-----END PRIVATE KEY-----\nretry_after=60\n"
|
||||
for _, debug := range []bool{false, true} {
|
||||
out := installTestLogger(t, debug)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
Reference in New Issue
Block a user