236 lines
7.0 KiB
Go
236 lines
7.0 KiB
Go
package logging
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
const redacted = "[REDACTED]"
|
|
|
|
func sensitive(key string) bool {
|
|
normalized := strings.Map(func(r rune) rune {
|
|
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
|
return unicode.ToLower(r)
|
|
}
|
|
return -1
|
|
}, key)
|
|
switch normalized {
|
|
case "content", "headers", "history", "sentence", "sentences", "words", "openingremark", "openingremarks", "greeting", "sessionnonce", "credential", "credentials", "key", "authorization", "proxyauthorization", "apikey", "accesskeyid", "accesskeysecret", "appkey", "appsecret", "secret", "secretkey", "password", "passwd", "token", "accesstoken", "refreshtoken", "uploadtoken", "providertoken", "sessioncredential", "clientnonce", "servernonce", "cookie", "setcookie", "prompt", "systemprompt", "instructions", "messages", "messagecontent", "transcript", "transcription", "finaltranscript", "text", "utterance", "audio", "audiodata", "recordingdata", "raw", "closingremark", "signature", "targeturl":
|
|
return true
|
|
}
|
|
return strings.HasSuffix(normalized, "password") || strings.HasSuffix(normalized, "secret") || strings.HasSuffix(normalized, "apikey") || strings.HasSuffix(normalized, "token") || strings.HasSuffix(normalized, "text")
|
|
}
|
|
|
|
type replacement struct {
|
|
start, end int
|
|
value string
|
|
}
|
|
|
|
// RedactJSON preserves the original lexical representation, including duplicate
|
|
// keys, numeric spelling and whitespace, except explicitly sensitive values.
|
|
// A JSON re-marshal would hide precisely the malformed/duplicate input under investigation.
|
|
func RedactJSON(body []byte) (string, error) {
|
|
if len(body) == 0 {
|
|
return "", nil
|
|
}
|
|
if !json.Valid(body) {
|
|
return "", errors.New("body is not valid JSON; original omitted")
|
|
}
|
|
trimmed := bytes.TrimSpace(body)
|
|
if trimmed[0] != '{' && trimmed[0] != '[' {
|
|
return "", errors.New("unstructured JSON body omitted")
|
|
}
|
|
dec := json.NewDecoder(strings.NewReader(string(body)))
|
|
dec.UseNumber()
|
|
var replacements []replacement
|
|
var value func(bool) error
|
|
startOffset := func() int {
|
|
n := int(dec.InputOffset())
|
|
for n < len(body) && (body[n] == ' ' || body[n] == '\n' || body[n] == '\r' || body[n] == '\t' || body[n] == ':' || body[n] == ',') {
|
|
n++
|
|
}
|
|
return n
|
|
}
|
|
value = func(hide bool) error {
|
|
start := startOffset()
|
|
token, err := dec.Token()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch token {
|
|
case json.Delim('{'):
|
|
for dec.More() {
|
|
key, err := dec.Token()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := value(!hide && sensitive(key.(string))); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := dec.Token(); err != nil {
|
|
return err
|
|
}
|
|
case json.Delim('['):
|
|
for dec.More() {
|
|
if err := value(false); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := dec.Token(); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
if s, ok := token.(string); ok && !hide {
|
|
clean := s
|
|
nested := strings.TrimSpace(s)
|
|
if json.Valid([]byte(nested)) && (strings.HasPrefix(nested, "{") || strings.HasPrefix(nested, "[")) {
|
|
var err error
|
|
clean, err = RedactJSON([]byte(s))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
clean = cleanString(s)
|
|
}
|
|
if clean != s {
|
|
encoded, _ := json.Marshal(clean)
|
|
replacements = append(replacements, replacement{start, int(dec.InputOffset()), string(encoded)})
|
|
}
|
|
}
|
|
}
|
|
if hide {
|
|
end := int(dec.InputOffset())
|
|
// Hiding an entire object/array also hides every replacement inside it.
|
|
kept := replacements[:0]
|
|
for _, r := range replacements {
|
|
if r.start < start || r.end > end {
|
|
kept = append(kept, r)
|
|
}
|
|
}
|
|
replacements = append(kept, replacement{start, end, `"[REDACTED]"`})
|
|
}
|
|
return nil
|
|
}
|
|
if err := value(false); err != nil {
|
|
return "", fmt.Errorf("JSON redaction failed: %w", err)
|
|
}
|
|
if _, err := dec.Token(); err != io.EOF {
|
|
return "", errors.New("unexpected trailing JSON data")
|
|
}
|
|
sort.Slice(replacements, func(i, j int) bool { return replacements[i].start < replacements[j].start })
|
|
var out strings.Builder
|
|
cursor := 0
|
|
for _, r := range replacements {
|
|
out.Write(body[cursor:r.start])
|
|
out.WriteString(r.value)
|
|
cursor = r.end
|
|
}
|
|
out.Write(body[cursor:])
|
|
return out.String(), nil
|
|
}
|
|
|
|
var (
|
|
urlInText = regexp.MustCompile(`https?://[^\s<>"']+`)
|
|
bearerInText = regexp.MustCompile(`(?i)\bBearer\s+[^\s,;"']+`)
|
|
secretInText = regexp.MustCompile(`(?i)\b(api[_-]?key|access[_-]?key[_-]?(secret|id)|password|token|secret|signature|prompt|system[_-]?prompt|content|transcript|text)["']?\s*[=:]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\s,;"']+)`)
|
|
)
|
|
|
|
// Error messages can contain a provider's JSON response after a text prefix.
|
|
// Use the same structural redaction, not regex-only masking of array/object data.
|
|
func redactErrorJSON(s string) string {
|
|
var out strings.Builder
|
|
for offset := 0; offset < len(s); {
|
|
next := strings.IndexAny(s[offset:], "{[")
|
|
if next < 0 {
|
|
out.WriteString(s[offset:])
|
|
break
|
|
}
|
|
start := offset + next
|
|
out.WriteString(s[offset:start])
|
|
dec := json.NewDecoder(strings.NewReader(s[start:]))
|
|
var raw json.RawMessage
|
|
if err := dec.Decode(&raw); err != nil {
|
|
suffix := strings.TrimSpace(s[start+1:])
|
|
if strings.HasPrefix(suffix, "\"") || strings.HasPrefix(suffix, "{") {
|
|
out.WriteString("[OMITTED: invalid embedded error JSON]")
|
|
break
|
|
}
|
|
out.WriteByte(s[start])
|
|
offset = start + 1
|
|
continue
|
|
}
|
|
safe, err := RedactJSON(raw)
|
|
if err != nil {
|
|
out.WriteString("[OMITTED: unsafe embedded error JSON]")
|
|
} else {
|
|
out.WriteString(safe)
|
|
}
|
|
offset = start + int(dec.InputOffset())
|
|
}
|
|
return out.String()
|
|
}
|
|
|
|
func cleanString(s string) string {
|
|
s = redactErrorJSON(s)
|
|
s = urlInText.ReplaceAllStringFunc(s, func(raw string) string {
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
return "[REDACTED_URL]"
|
|
}
|
|
if u.RawQuery != "" || u.User != nil || u.Fragment != "" {
|
|
u.User = nil
|
|
u.RawQuery = ""
|
|
u.Fragment = ""
|
|
return u.String() + "[REDACTED_URL_PARTS]"
|
|
}
|
|
return raw
|
|
})
|
|
s = bearerInText.ReplaceAllString(s, "Bearer "+redacted)
|
|
return secretInText.ReplaceAllString(s, "$1="+redacted)
|
|
}
|
|
|
|
func safeValue(key string, v any) any {
|
|
if sensitive(key) {
|
|
return redacted
|
|
}
|
|
switch x := v.(type) {
|
|
case error:
|
|
return cleanString(x.Error())
|
|
case string:
|
|
if json.Valid([]byte(x)) && (strings.HasPrefix(strings.TrimSpace(x), "{") || strings.HasPrefix(strings.TrimSpace(x), "[")) {
|
|
safe, err := RedactJSON([]byte(x))
|
|
if err != nil {
|
|
return "[OMITTED: JSON redaction failed]"
|
|
}
|
|
return safe
|
|
}
|
|
return cleanString(x)
|
|
case []byte:
|
|
return fmt.Sprintf("[BINARY: %d bytes]", len(x))
|
|
default:
|
|
// Existing structured logs only need scalar identity/count/status fields.
|
|
// Sanitize containers instead of allowing Logrus to serialize unchecked structs.
|
|
body, err := json.Marshal(v)
|
|
if err != nil {
|
|
return "[OMITTED: unsupported log value]"
|
|
}
|
|
if len(body) > 0 && (body[0] == '{' || body[0] == '[') {
|
|
safe, err := RedactJSON(body)
|
|
if err != nil {
|
|
return "[OMITTED: unsafe log value]"
|
|
}
|
|
return json.RawMessage(safe)
|
|
}
|
|
return v
|
|
}
|
|
}
|