150 lines
5.1 KiB
Python
150 lines
5.1 KiB
Python
"""Daily UTF-8 business logs; bounded UI history, no raw requests or message bodies."""
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
from collections import deque
|
|
from contextlib import suppress
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
UI_LIMIT = 2000
|
|
|
|
|
|
CREDENTIAL = re.compile(
|
|
r"(?i)(cookie|authorization|sessionid|mstoken|a_bogus|password|token|secret|verifyfp|signature|x-bogus|x-tt-params)"
|
|
)
|
|
|
|
|
|
def clean(value):
|
|
text = str(value).replace("\r", r"\r").replace("\n", r"\n")
|
|
text = re.sub(r"[\x00-\x1f\x7f\u2028\u2029\u202a-\u202e]", " ", text)
|
|
text = re.sub(
|
|
r"(?i)([?&](?:sessionid|mstoken|a_bogus|token|verifyfp|signature|x-bogus|x-tt-params)=)[^&\s]+",
|
|
r"\1[凭据已隐藏]",
|
|
text,
|
|
)
|
|
text = re.sub(
|
|
r"(?i)\b(cookie|authorization)[\"']?\s*[:=].*",
|
|
r"\1=[凭据已隐藏]",
|
|
text,
|
|
)
|
|
return text
|
|
|
|
|
|
def visible(value, key=""):
|
|
if CREDENTIAL.search(str(key)):
|
|
return "[凭据已隐藏]"
|
|
if isinstance(value, dict):
|
|
return {str(k): visible(v, k) for k, v in value.items()}
|
|
if isinstance(value, (list, tuple)):
|
|
return [visible(item) for item in value]
|
|
return value
|
|
|
|
|
|
def recent_lines(directory, limit=UI_LIMIT):
|
|
"""Read backwards, not the entire accumulated log history."""
|
|
rows = []
|
|
for path in sorted(directory.glob("????-??-??.log"), reverse=True):
|
|
with path.open("rb") as stream:
|
|
stream.seek(0, 2)
|
|
position, carry = stream.tell(), b""
|
|
while position and len(rows) < limit:
|
|
size = min(position, 65536)
|
|
position -= size
|
|
stream.seek(position)
|
|
parts = (stream.read(size) + carry).split(b"\n")
|
|
carry = parts.pop(0)
|
|
for part in reversed(parts):
|
|
if part and len(rows) < limit:
|
|
rows.append(clean(part.decode("utf-8", errors="replace")))
|
|
if not position and carry and len(rows) < limit:
|
|
rows.append(clean(carry.decode("utf-8", errors="replace")))
|
|
if len(rows) >= limit:
|
|
break
|
|
return list(reversed(rows))
|
|
|
|
|
|
class DailyLog(logging.Handler):
|
|
def __init__(self, root):
|
|
super().__init__()
|
|
self.directory = Path(root) / "logs"
|
|
self.entries = deque(maxlen=UI_LIMIT)
|
|
self.sequence = 0
|
|
self.stream = None
|
|
self.day = None
|
|
self.file_error = ""
|
|
self.setFormatter(
|
|
logging.Formatter(
|
|
"%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s",
|
|
"%Y-%m-%d %H:%M:%S",
|
|
)
|
|
)
|
|
self.logger = logging.Logger("douyin.activity", logging.INFO)
|
|
self.logger.addHandler(self)
|
|
try:
|
|
for line in recent_lines(self.directory):
|
|
self.remember(line)
|
|
except OSError as exc:
|
|
self.file_error = "历史日志读取失败：" + type(exc).__name__
|
|
self.remember("[ERROR] " + self.file_error)
|
|
|
|
def remember(self, line):
|
|
self.sequence += 1
|
|
self.entries.append((self.sequence, line))
|
|
|
|
def since(self, cursor):
|
|
return [line for seq, line in self.entries if seq > cursor]
|
|
|
|
def record(self, stage, message, level=logging.INFO, **fields):
|
|
values = []
|
|
for key, value in fields.items():
|
|
if value is None:
|
|
continue
|
|
value = visible(value, key)
|
|
if not isinstance(value, (str, int, float, bool)):
|
|
value = json.dumps(value, ensure_ascii=False, default=str)
|
|
values.append(f"{clean(key)}={clean(value)}")
|
|
self.logger.log(
|
|
level, "[%s] %s %s", clean(stage), clean(message), " ".join(values)
|
|
)
|
|
|
|
def emit(self, record):
|
|
line = clean(self.format(record))
|
|
self.remember(line)
|
|
day = datetime.fromtimestamp(record.created).strftime("%Y-%m-%d")
|
|
try:
|
|
if self.day != day or self.stream is None:
|
|
if self.stream:
|
|
self.stream.close()
|
|
self.stream = None
|
|
self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
self.stream = (self.directory / (day + ".log")).open(
|
|
"a", encoding="utf-8", newline="\n"
|
|
)
|
|
self.day = day
|
|
self.stream.write(line + "\n")
|
|
self.stream.flush()
|
|
self.file_error = ""
|
|
except OSError as exc:
|
|
message = (
|
|
"日志文件写入失败（"
|
|
+ type(exc).__name__
|
|
+ "），当前仅保留内存最近 2000 条，请检查磁盘与权限"
|
|
)
|
|
if message != self.file_error:
|
|
self.remember("[ERROR] " + message)
|
|
self.file_error = message
|
|
if self.stream:
|
|
with suppress(OSError):
|
|
self.stream.close()
|
|
self.stream = None
|
|
|
|
def close(self):
|
|
if self.stream:
|
|
with suppress(OSError):
|
|
self.stream.close()
|
|
self.stream = None
|
|
self.logger.removeHandler(self)
|
|
super().close()
|