package logging import ( "bytes" "encoding/json" "errors" "fmt" "io" "net/url" "regexp" "sort" "strings" "unicode" "unicode/utf8" ) const redacted = "[REDACTED]" func normalizedField(key string) string { return strings.Map(func(r rune) rune { if unicode.IsLetter(r) || unicode.IsDigit(r) { return unicode.ToLower(r) } return -1 }, key) } // Content ownership is defined once for JSON, markup, scalar logs and text // continuation. It always implies sensitivity; credentials own only values. func conversationField(key string) bool { normalized := normalizedField(key) switch normalized { case "prompt", "systemprompt", "systemmessage", "instructions", "messages", "conversation", "history", "chathistory", "conversationhistory", "dialogue", "dialog", "transcript", "transcription", "finaltranscript", "messagecontent", "text", "content", "utterance", "sentence", "sentences", "words", "openingremark", "openingremarks", "greeting", "closingremark", "audio", "audiodata", "recordingdata", "raw": return true } return strings.HasSuffix(normalized, "text") } func sensitive(key string) bool { if conversationField(key) { return true } normalized := normalizedField(key) switch normalized { case "privatekey", "privatekeypem", "clientkey", "clientkeypem", "serverkey", "serverkeypem", "headers", "sessionnonce", "credential", "credentials", "key", "authorization", "proxyauthorization", "apikey", "accesskeyid", "accesskeysecret", "appkey", "appsecret", "secret", "secretkey", "password", "passwd", "token", "accesstoken", "refreshtoken", "uploadtoken", "providertoken", "sessioncredential", "clientnonce", "servernonce", "cookie", "setcookie", "signature", "targeturl": return true } return strings.HasSuffix(normalized, "password") || strings.HasSuffix(normalized, "secret") || strings.HasSuffix(normalized, "apikey") || strings.HasSuffix(normalized, "token") } type replacement struct { start, end int value string } // RedactJSON preserves the original lexical representation, including duplicate // keys, numeric spelling and whitespace, except explicitly sensitive values. // A JSON re-marshal would hide precisely the malformed/duplicate input under investigation. func RedactJSON(body []byte) (string, error) { if len(body) == 0 { return "", nil } if !json.Valid(body) { return "", errors.New("body is not valid JSON; original omitted") } trimmed := bytes.TrimSpace(body) if trimmed[0] != '{' && trimmed[0] != '[' { return "", errors.New("unstructured JSON body omitted") } dec := json.NewDecoder(strings.NewReader(string(body))) dec.UseNumber() var replacements []replacement var value func(bool) error startOffset := func() int { n := int(dec.InputOffset()) for n < len(body) && (body[n] == ' ' || body[n] == '\n' || body[n] == '\r' || body[n] == '\t' || body[n] == ':' || body[n] == ',') { n++ } return n } value = func(hide bool) error { start := startOffset() token, err := dec.Token() if err != nil { return err } switch token { case json.Delim('{'): for dec.More() { key, err := dec.Token() if err != nil { return err } if err := value(!hide && sensitive(key.(string))); err != nil { return err } } if _, err := dec.Token(); err != nil { return err } case json.Delim('['): for dec.More() { if err := value(false); err != nil { return err } } if _, err := dec.Token(); err != nil { return err } default: if s, ok := token.(string); ok && !hide { clean := s nested := strings.TrimSpace(s) if json.Valid([]byte(nested)) && (strings.HasPrefix(nested, "{") || strings.HasPrefix(nested, "[")) { var err error clean, err = RedactJSON([]byte(s)) if err != nil { return err } } else { clean = cleanString(s) } if clean != s { encoded, _ := json.Marshal(clean) replacements = append(replacements, replacement{start, int(dec.InputOffset()), string(encoded)}) } } } if hide { end := int(dec.InputOffset()) // Hiding an entire object/array also hides every replacement inside it. kept := replacements[:0] for _, r := range replacements { if r.start < start || r.end > end { kept = append(kept, r) } } replacements = append(kept, replacement{start, end, `"[REDACTED]"`}) } return nil } if err := value(false); err != nil { return "", fmt.Errorf("JSON redaction failed: %w", err) } if _, err := dec.Token(); err != io.EOF { return "", errors.New("unexpected trailing JSON data") } sort.Slice(replacements, func(i, j int) bool { return replacements[i].start < replacements[j].start }) var out strings.Builder cursor := 0 for _, r := range replacements { out.Write(body[cursor:r.start]) out.WriteString(r.value) cursor = r.end } out.Write(body[cursor:]) return out.String(), nil } var ( urlInText = regexp.MustCompile(`(?i)(?:https?|wss?)://[^\s<>"']+`) authHeaderInText = regexp.MustCompile(`(?im)\b(?:proxy-authorization|authorization|set-cookie|cookie)["']?\s*[=:]\s*[^\r\n]+`) privatePEMInText = regexp.MustCompile(`(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?(?:-----END [A-Z0-9 ]*PRIVATE KEY-----|$)`) sensitiveTextLine = regexp.MustCompile(`(?im)\b(?:prompt|system[_-]?prompt|messages|transcript|text|content|audio|closing[_-]?remark)\s*[=:]\s*[^\r\n]+`) bearerInText = regexp.MustCompile(`(?i)\bBearer\s+[^\s,;"']+`) secretInText = regexp.MustCompile(`(?i)\b(api[_-]?key|private[_-]?key(?:[_-]?pem)?|client[_-]?key(?:[_-]?pem)?|access[_-]?key[_-]?(secret|id)|password|token|secret|signature|prompt|system[_-]?prompt|content|transcript|text)["']?\s*[=:]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\s,;"']+)`) ) // Error messages can contain a provider's JSON response after a text prefix. // Use the same structural redaction, not regex-only masking of array/object data. func redactErrorJSON(s string) string { var out strings.Builder for offset := 0; offset < len(s); { next := strings.IndexAny(s[offset:], "{[") if next < 0 { out.WriteString(s[offset:]) break } start := offset + next out.WriteString(s[offset:start]) dec := json.NewDecoder(strings.NewReader(s[start:])) var raw json.RawMessage if err := dec.Decode(&raw); err != nil { suffix := strings.TrimSpace(s[start+1:]) if strings.HasPrefix(suffix, "\"") || strings.HasPrefix(suffix, "{") { out.WriteString("[OMITTED: invalid embedded error JSON]") break } out.WriteByte(s[start]) offset = start + 1 continue } safe, err := RedactJSON(raw) if err != nil { out.WriteString("[OMITTED: unsafe embedded error JSON]") } else { out.WriteString(safe) } offset = start + int(dec.InputOffset()) } return out.String() } func cleanString(s string) string { s = privatePEMInText.ReplaceAllString(s, redacted) s = redactMarkup(s) s = redactInlineBinary(s) if form, ok := redactForm(s); ok { s = form } // Scalar messages/errors and HTTP text share the same labelled-content // protection, rather than protecting conversational blocks only in bodies. s = redactConversationLines(s) s = redactErrorJSON(s) s = authHeaderInText.ReplaceAllStringFunc(s, func(header string) string { index := strings.IndexAny(header, ":=") if index >= 0 { value := strings.TrimSpace(header[index+1:]) if strings.HasPrefix(value, `"[REDACTED]"`) || strings.HasPrefix(value, redacted) { return header } } return "[REDACTED_AUTH_HEADER]" }) s = urlInText.ReplaceAllStringFunc(s, func(raw string) string { u, err := url.Parse(raw) if err != nil { return "[REDACTED_URL]" } if u.RawQuery != "" || u.User != nil || u.Fragment != "" { u.User = nil u.RawQuery = "" u.Fragment = "" return u.String() + "[REDACTED_URL_PARTS]" } return raw }) s = bearerInText.ReplaceAllString(s, "Bearer "+redacted) s = secretInText.ReplaceAllString(s, "$1="+redacted) return s } // RedactText is used only for readable textual HTTP content. JSON objects // inside free text use structural redaction; labelled conversational lines are // excluded while unrelated business lines and original bytes stay untouched. func RedactText(body []byte) (string, error) { if !utf8.Valid(body) { return "", errors.New("HTTP text is not valid UTF-8") } for _, r := range string(body) { if unicode.IsControl(r) && r != '\n' && r != '\r' && r != '\t' { return "", errors.New("HTTP content contains binary control data") } } safe := redactMarkup(string(body)) if form, ok := redactForm(safe); ok { return form, nil } safe = cleanString(redactConversationLines(safe)) safe = sensitiveTextLine.ReplaceAllString(safe, "[REDACTED_CONVERSATION]") return safe, nil } func safeValue(key string, v any) any { if sensitive(key) { return redacted } switch x := v.(type) { case error: return cleanString(x.Error()) case string: if json.Valid([]byte(x)) && (strings.HasPrefix(strings.TrimSpace(x), "{") || strings.HasPrefix(strings.TrimSpace(x), "[")) { safe, err := RedactJSON([]byte(x)) if err != nil { return "[OMITTED: JSON redaction failed]" } return safe } return cleanString(x) case []byte: return fmt.Sprintf("[BINARY: %d bytes]", len(x)) default: // Existing structured logs only need scalar identity/count/status fields. // Sanitize containers instead of allowing Logrus to serialize unchecked structs. body, err := json.Marshal(v) if err != nil { return "[OMITTED: unsupported log value]" } if len(body) > 0 && (body[0] == '{' || body[0] == '[') { safe, err := RedactJSON(body) if err != nil { return "[OMITTED: unsafe log value]" } return json.RawMessage(safe) } return v } }