263 lines
8.3 KiB
Go
263 lines
8.3 KiB
Go
package logging
|
||
|
||
import (
|
||
"bytes"
|
||
"encoding/json"
|
||
"io"
|
||
"net/url"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"golang.org/x/net/html"
|
||
)
|
||
|
||
// The HTML tokenizer also accepts XML-shaped response fragments. Raw token
|
||
// bytes are retained; only explicitly sensitive tags/attributes are replaced.
|
||
func redactMarkup(body string) string {
|
||
if !strings.Contains(body, "<") {
|
||
return body
|
||
}
|
||
z := html.NewTokenizer(strings.NewReader(body))
|
||
var out strings.Builder
|
||
hiddenTag := ""
|
||
depth := 0
|
||
for {
|
||
kind := z.Next()
|
||
raw := bytes.Clone(z.Raw())
|
||
if kind == html.ErrorToken {
|
||
if z.Err() == io.EOF {
|
||
out.Write(raw)
|
||
}
|
||
break
|
||
}
|
||
token := z.Token()
|
||
if hiddenTag != "" {
|
||
if kind == html.StartTagToken && token.Data == hiddenTag {
|
||
depth++
|
||
}
|
||
if kind == html.EndTagToken && token.Data == hiddenTag {
|
||
depth--
|
||
if depth == 0 {
|
||
hiddenTag = ""
|
||
out.Write(raw)
|
||
}
|
||
}
|
||
continue
|
||
}
|
||
if kind == html.StartTagToken || kind == html.SelfClosingTagToken {
|
||
tag := string(raw)
|
||
hideValue := false
|
||
for _, a := range token.Attr {
|
||
if (a.Key == "name" && sensitive(a.Val)) || (a.Key == "type" && strings.EqualFold(a.Val, "password")) {
|
||
hideValue = true
|
||
}
|
||
}
|
||
for _, a := range token.Attr {
|
||
safe := redactInlineBinary(a.Val)
|
||
if sensitive(a.Key) || (hideValue && a.Key == "value") {
|
||
safe = redacted
|
||
} else if safe == a.Val && a.Val != "" && (sensitive(token.Data) || hideValue) && !diagnosticTextLine(a.Key+"="+a.Val, 0) && !isInlineBinarySummary(a.Val) {
|
||
// Private elements own their attribute payloads too, not only inner
|
||
// text. Preserve only typed diagnostic scalars or binary metadata.
|
||
safe = redacted
|
||
}
|
||
if safe != a.Val {
|
||
pattern := regexp.MustCompile(`(?i)(\b` + regexp.QuoteMeta(a.Key) + `\s*=\s*)(?:"[^"]*"|'[^']*'|[^\s>]+)`)
|
||
tag = pattern.ReplaceAllStringFunc(tag, func(match string) string { return pattern.FindStringSubmatch(match)[1] + strconv.Quote(safe) })
|
||
}
|
||
}
|
||
out.WriteString(tag)
|
||
// A named sensitive field owns both its value attribute and enclosed
|
||
// content (textarea/select/custom controls). Void inputs own only value;
|
||
// they must never swallow following non-sensitive business elements.
|
||
if kind == html.StartTagToken && (sensitive(token.Data) || (hideValue && !voidMarkupElement(token.Data))) {
|
||
out.WriteString(redacted)
|
||
hiddenTag = token.Data
|
||
depth = 1
|
||
}
|
||
} else {
|
||
out.Write(raw)
|
||
}
|
||
}
|
||
return out.String()
|
||
}
|
||
|
||
func voidMarkupElement(name string) bool {
|
||
switch name {
|
||
case "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// A form can be identified without guessing semantic values. Untouched pairs
|
||
// preserve their original encoding and order, including duplicate field names.
|
||
func redactForm(body string) (string, bool) {
|
||
if strings.ContainsAny(body, "\r\n<>") || !strings.Contains(body, "=") {
|
||
return body, false
|
||
}
|
||
parts := strings.Split(body, "&")
|
||
for i, part := range parts {
|
||
key, value, hasValue := strings.Cut(part, "=")
|
||
decodedKey, err := url.QueryUnescape(key)
|
||
if err != nil {
|
||
// A failed field decode must not restore earlier sensitive pairs or
|
||
// expose the unclassifiable field. Keep the failure explicit in the copy.
|
||
parts[i] = url.QueryEscape("[REDACTED_INVALID_FORM_KEY]")
|
||
continue
|
||
}
|
||
if sensitive(decodedKey) {
|
||
parts[i] = key + "=" + url.QueryEscape(redacted)
|
||
continue
|
||
}
|
||
// Bare flags are valid empty-value form fields, not a parse failure.
|
||
if !hasValue {
|
||
continue
|
||
}
|
||
decoded, err := url.QueryUnescape(value)
|
||
if err != nil {
|
||
parts[i] = key + "=" + url.QueryEscape("[REDACTED_INVALID_FORM_VALUE]")
|
||
continue
|
||
}
|
||
safe := cleanString(redactConversationLines(decoded))
|
||
if safe != decoded {
|
||
parts[i] = key + "=" + url.QueryEscape(safe)
|
||
}
|
||
}
|
||
return strings.Join(parts, "&"), true
|
||
}
|
||
|
||
// One field lexer for quoted/unquoted labels and diagnostic projections.
|
||
// JSON names are decoded, so escapes cannot bypass the content classifier.
|
||
const textFieldName = `((?:\\?"(?:\\.|[^"\\\r\n])*\\?")|(?:'(?:\\.|[^'\\\r\n])*')|` + "(?:`[^`\\r\\n]*`)" + `|(?:“[^”\r\n]*”)|(?:‘[^’\r\n]*’)|(?:[\p{L}_][\p{L}\p{N}_.-]*))`
|
||
|
||
var textFieldLine = regexp.MustCompile(`^\s*` + textFieldName + `\s*[:=]`)
|
||
var textFieldLabel = regexp.MustCompile(textFieldName + `\s*[:=]`)
|
||
|
||
func textFieldKey(token string) (string, bool) {
|
||
if strings.HasPrefix(token, `\"`) && strings.HasSuffix(token, `\"`) {
|
||
token = `"` + token[2:len(token)-2] + `"`
|
||
}
|
||
if strings.HasPrefix(token, `"`) {
|
||
var key string
|
||
if err := json.Unmarshal([]byte(token), &key); err != nil {
|
||
return "", false
|
||
}
|
||
return normalizedField(key), true
|
||
}
|
||
return normalizedField(token), true
|
||
}
|
||
|
||
var diagnosticUUID = regexp.MustCompile(`^(?:call-)?[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||
|
||
// Free text has no reliable closing delimiter for content. Project only typed
|
||
// diagnostic scalars from its tail; those never release content ownership.
|
||
// Arbitrary field values and message schemas remain opaque, not guessed safe.
|
||
func diagnosticTextLine(line string, contentIndent int) bool {
|
||
field := textFieldLine.FindStringSubmatchIndex(line)
|
||
if len(field) == 0 || len(line)-len(strings.TrimLeft(line, " \t")) > contentIndent {
|
||
return false
|
||
}
|
||
key, valid := textFieldKey(line[field[2]:field[3]])
|
||
if !valid {
|
||
return false
|
||
}
|
||
value := strings.TrimSpace(line[field[1]:])
|
||
switch key {
|
||
case "taskid", "tenantid", "trunkid", "attempts", "remaining", "retryafter", "count", "elapsedms", "durationms":
|
||
_, err := strconv.ParseUint(value, 10, 64)
|
||
return err == nil
|
||
case "httpstatus":
|
||
n, err := strconv.ParseUint(value, 10, 16)
|
||
return err == nil && n >= 100 && n <= 599
|
||
case "eventid", "executionid":
|
||
return diagnosticUUID.MatchString(value)
|
||
case "nextat":
|
||
_, err := time.Parse(time.RFC3339Nano, value)
|
||
return err == nil
|
||
case "status":
|
||
switch value {
|
||
case "ready", "retry_pending", "pending", "running", "paused", "stopped", "finished", "rejected", "ended", "failed", "unknown", "accepted", "completed", "succeeded", "waiting", "active", "inactive":
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// Complete embedded JSON belongs to structural redaction, not the free-text
|
||
// lexer. Keep its boundaries so quoted keys cannot discard adjacent diagnostics.
|
||
func structuredTextSpans(body string) [][2]int {
|
||
var spans [][2]int
|
||
for offset := 0; offset < len(body); {
|
||
next := strings.IndexAny(body[offset:], "{[")
|
||
if next < 0 {
|
||
break
|
||
}
|
||
start := offset + next
|
||
dec := json.NewDecoder(strings.NewReader(body[start:]))
|
||
var raw json.RawMessage
|
||
if err := dec.Decode(&raw); err != nil {
|
||
offset = start + 1
|
||
continue
|
||
}
|
||
end := start + int(dec.InputOffset())
|
||
spans = append(spans, [2]int{start, end})
|
||
offset = end
|
||
}
|
||
return spans
|
||
}
|
||
|
||
func redactConversationLines(body string) string {
|
||
spans := structuredTextSpans(body)
|
||
offset := 0
|
||
lines := strings.SplitAfter(body, "\n")
|
||
hiding := false
|
||
contentIndent := 0
|
||
for i, line := range lines {
|
||
lineOffset := offset
|
||
offset += len(line)
|
||
// Content owns its full tail, including unknown fields, speaker names,
|
||
// nested message metadata and blank paragraphs. Typed diagnostics are a
|
||
// projection of individual lines, never an inferred end of the dialogue.
|
||
if hiding {
|
||
if !diagnosticTextLine(line, contentIndent) {
|
||
lines[i] = ""
|
||
}
|
||
continue
|
||
}
|
||
// Labels establish ownership wherever they occur in an error chain, not
|
||
// only at line start. Preserve the preceding actual failure description.
|
||
for _, span := range textFieldLabel.FindAllStringSubmatchIndex(line, -1) {
|
||
position := lineOffset + span[0]
|
||
structured := false
|
||
for _, region := range spans {
|
||
if position >= region[0] && position < region[1] {
|
||
structured = true
|
||
break
|
||
}
|
||
}
|
||
if structured {
|
||
continue
|
||
}
|
||
key, valid := textFieldKey(line[span[2]:span[3]])
|
||
if valid && !sensitive(key) {
|
||
continue
|
||
}
|
||
lines[i] = line[:span[0]] + "[REDACTED_FIELD]"
|
||
if strings.HasSuffix(line, "\n") {
|
||
lines[i] += "\n"
|
||
}
|
||
// Credentials own only this line; content owns the following paragraphs.
|
||
// Malformed quoted labels have no safely classifiable tail. Keep it
|
||
// opaque too, rather than silently exposing a failed name decode.
|
||
hiding = !valid || conversationField(key)
|
||
contentIndent = len(line) - len(strings.TrimLeft(line, " \t"))
|
||
break
|
||
}
|
||
}
|
||
return strings.Join(lines, "")
|
||
}
|