H-385: add Clark CDP network spike
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
multica-agent
@@ -5,3 +5,5 @@ Lume Ctrl is a CDP-only network observability and debugging control plane for cl
|
||||
It observes browser-internal HTTP(S), failure, cache, and WebSocket events without changing the TLS path. The MVP focuses on safe collection, correlation, redaction, bounded body handling, session-isolated query, and JSONL export.
|
||||
|
||||
VNC/noVNC, GUI control, MITM, proxy certificates, CA injection, packet capture, proxy pools, fingerprinting, account operations, DOM scraping, and media decoding are out of scope.
|
||||
|
||||
The reproducible Clark Chromium network-event proof is documented in [docs/m0-spike.md](docs/m0-spike.md).
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request","request_id":"2352456.3","at":1787234901.120407,"request":{"method":"GET","url":"http://127.0.0.1:40745/ok","headers":{"accept-language":"en-US,en;q=0.9","authorization":"[REDACTED]","cookie":"[REDACTED]","referer":"[REDACTED]","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request_headers","request_id":"2352456.3","headers":{"accept":"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","authorization":"[REDACTED]","connection":"keep-alive","cookie":"[REDACTED]","host":"127.0.0.1:40745","sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"same-origin","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response","request_id":"2352456.2","response":{"url":"http://127.0.0.1:40745/favicon.ico","status":200,"protocol":"http/1.1","mime_type":"text/html","headers":{"connection":"close","content-length":"38","content-type":"text/html","date":"Thu, 20 Aug 2026 14:08:21 GMT"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_finished","request_id":"2352456.2","encoded_bytes":158}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response_headers","request_id":"2352456.3","status":200,"headers":{"access-control-allow-headers":"authorization","access-control-allow-origin":"*","connection":"close","content-length":"11","content-type":"application/json","date":"Thu, 20 Aug 2026 14:08:21 GMT","set-cookie":"[REDACTED]"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response","request_id":"2352456.3","response":{"url":"http://127.0.0.1:40745/ok","status":200,"protocol":"http/1.1","mime_type":"application/json","headers":{"access-control-allow-headers":"authorization","access-control-allow-origin":"*","connection":"close","content-length":"11","content-type":"application/json","date":"Thu, 20 Aug 2026 14:08:21 GMT"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_finished","request_id":"2352456.3","encoded_bytes":291}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request","request_id":"2352456.4","at":1787234901.124618,"request":{"method":"GET","url":"http://127.0.0.1:40745/large","headers":{"accept-language":"en-US,en;q=0.9","authorization":"[REDACTED]","cookie":"[REDACTED]","referer":"[REDACTED]","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request_headers","request_id":"2352456.4","headers":{"accept":"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","authorization":"[REDACTED]","connection":"keep-alive","cookie":"[REDACTED]","host":"127.0.0.1:40745","sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"same-origin","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response_headers","request_id":"2352456.4","status":200,"headers":{"access-control-allow-headers":"authorization","access-control-allow-origin":"*","connection":"close","content-length":"2048","content-type":"text/plain","date":"Thu, 20 Aug 2026 14:08:21 GMT"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response","request_id":"2352456.4","response":{"url":"http://127.0.0.1:40745/large","status":200,"protocol":"http/1.1","mime_type":"text/plain","headers":{"access-control-allow-headers":"authorization","access-control-allow-origin":"*","connection":"close","content-length":"2048","content-type":"text/plain","date":"Thu, 20 Aug 2026 14:08:21 GMT"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_finished","request_id":"2352456.4","encoded_bytes":2248}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request","request_id":"2352456.5","at":1787234901.128406,"request":{"method":"GET","url":"https://127.0.0.1:38819/secure","headers":{"accept-language":"en-US,en;q=0.9","authorization":"[REDACTED]","cookie":"[REDACTED]","referer":"[REDACTED]","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request","request_id":"A1C10D76513CFFAB16DEFD007045EF58","at":1787234901.129414,"request":{"method":"OPTIONS","url":"https://127.0.0.1:38819/secure","headers":{"accept":"*/*","access-control-request-headers":"authorization","access-control-request-method":"GET","origin":"http://127.0.0.1:40745","sec-fetch-mode":"cors","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request_headers","request_id":"A1C10D76513CFFAB16DEFD007045EF58","headers":{"accept":"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","access-control-request-headers":"authorization","access-control-request-method":"GET","connection":"keep-alive","host":"127.0.0.1:38819","origin":"http://127.0.0.1:40745","sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"cross-site","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response_headers","request_id":"A1C10D76513CFFAB16DEFD007045EF58","status":200,"headers":{"access-control-allow-headers":"authorization","access-control-allow-origin":"*","connection":"close","content-length":"15","content-type":"application/json","date":"Thu, 20 Aug 2026 14:08:21 GMT"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response","request_id":"A1C10D76513CFFAB16DEFD007045EF58","response":{"url":"https://127.0.0.1:38819/secure","status":200,"protocol":"http/1.1","mime_type":"application/json","headers":{"access-control-allow-headers":"authorization","access-control-allow-origin":"*","connection":"close","content-length":"15","content-type":"application/json","date":"Thu, 20 Aug 2026 14:08:21 GMT"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_finished","request_id":"A1C10D76513CFFAB16DEFD007045EF58","encoded_bytes":0}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request_headers","request_id":"2352456.5","headers":{"accept":"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","authorization":"[REDACTED]","connection":"keep-alive","cookie":"[REDACTED]","host":"127.0.0.1:38819","origin":"http://127.0.0.1:40745","sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"cross-site","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response_headers","request_id":"2352456.5","status":200,"headers":{"access-control-allow-headers":"authorization","access-control-allow-origin":"*","connection":"close","content-length":"15","content-type":"application/json","date":"Thu, 20 Aug 2026 14:08:21 GMT"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_response","request_id":"2352456.5","response":{"url":"https://127.0.0.1:38819/secure","status":200,"protocol":"http/1.1","mime_type":"application/json","headers":{"access-control-allow-headers":"authorization","access-control-allow-origin":"*","connection":"close","content-length":"15","content-type":"application/json","date":"Thu, 20 Aug 2026 14:08:21 GMT"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_finished","request_id":"2352456.5","encoded_bytes":219}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"websocket_created","connection_id":"2352456.6","url":"ws://127.0.0.1:40745/ws"}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"websocket_handshake_request","connection_id":"2352456.6","headers":{"accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","cache-control":"no-cache","connection":"Upgrade","cookie":"[REDACTED]","host":"127.0.0.1:40745","origin":"http://127.0.0.1:40745","pragma":"no-cache","sec-websocket-extensions":"permessage-deflate; client_max_window_bits","sec-websocket-key":"[REDACTED]","sec-websocket-version":"13","upgrade":"websocket","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"websocket_handshake_response","connection_id":"2352456.6","status":101,"headers":{"connection":"Upgrade","sec-websocket-accept":"[REDACTED]","upgrade":"websocket"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"websocket_frame","connection_id":"2352456.6","direction":"sent","opcode":1,"payload_bytes":14,"payload":{"state":"omitted","reason":"frame_policy"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"websocket_frame","connection_id":"2352456.6","direction":"received","opcode":1,"payload_bytes":14,"payload":{"state":"omitted","reason":"frame_policy"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"websocket_closed","connection_id":"2352456.6"}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request","request_id":"2352456.7","at":1787234901.156918,"request":{"method":"GET","url":"http://127.0.0.1:34677/missing","headers":{"accept-language":"en-US,en;q=0.9","authorization":"[REDACTED]","cookie":"[REDACTED]","referer":"[REDACTED]","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_request","request_id":"A8170B1659C31EA09293E5B60A446454","at":1787234901.15755,"request":{"method":"OPTIONS","url":"http://127.0.0.1:34677/missing","headers":{"accept":"*/*","access-control-request-headers":"authorization","access-control-request-method":"GET","origin":"http://127.0.0.1:40745","sec-fetch-mode":"cors","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/148.0.0.0 Safari/537.36"}}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_failed","request_id":"A8170B1659C31EA09293E5B60A446454","error":{"text":"net::ERR_CONNECTION_REFUSED","canceled":false}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_failed","request_id":"2352456.7","error":{"text":"net::ERR_CONNECTION_REFUSED","canceled":false}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_body","request_id":"2352456.3","body":{"state":"stored","bytes":11,"encoding":"utf8","text":"{\"ok\":true}"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_body","request_id":"2352456.4","body":{"state":"omitted","reason":"size_limit","bytes":2048}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_body","request_id":"2352456.5","body":{"state":"stored","bytes":15,"encoding":"utf8","text":"{\"secure\":true}"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"http_body","request_id":"2352456.7","body":{"state":"unavailable","reason":"cdp_error","error":"-32000: No data found for resource with given identifier"}}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"target_closed","reason":"Target.closeTarget"}
|
||||
{"schema_version":"1.0","session_id":"m0-2a6397b2-f63b-4f1a-9a75-4bad92a3dc94","target_id":"24707FCF8DFB0EA57823EF42C34ADD3B","kind":"cdp_disconnected","reason":"client_closed"}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"result": "pass",
|
||||
"clark": {
|
||||
"wrapper_version": "0.2.1",
|
||||
"chromium_version": "Chrome/148.0.7778.96",
|
||||
"protocol_version": "1.3",
|
||||
"cdp_bind": "127.0.0.1"
|
||||
},
|
||||
"coverage": {
|
||||
"http": "pass",
|
||||
"failed_request": "pass",
|
||||
"https_without_mitm": "pass",
|
||||
"websocket": "pass",
|
||||
"response_body_success": "pass",
|
||||
"response_body_size_limit": "pass",
|
||||
"response_body_unavailable": "pass",
|
||||
"target_close_survives": "pass",
|
||||
"cdp_disconnect_survives": "pass",
|
||||
"sensitive_header_redaction": "pass"
|
||||
},
|
||||
"resources": {
|
||||
"process_count": 9,
|
||||
"rss_kib": 792008,
|
||||
"cpu_percent_snapshot": 12.2
|
||||
},
|
||||
"event_count": 38,
|
||||
"max_body_bytes": 1024,
|
||||
"attempt_failures": []
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
## M0 CDP network spike
|
||||
|
||||
Run against the Clark binary declared by `clark-browser` 0.2.1:
|
||||
|
||||
```sh
|
||||
cd ../clark-browser
|
||||
python3 -m clarkbrowser fetch
|
||||
cd ../lume-ctrl
|
||||
M0_CDP_PORT=40746 M0_HTTP_PORT=40745 \
|
||||
CLARK_BINARY_PATH="$HOME/.clarkbrowser/chromium-148.0.7778.96/chrome" npm run spike
|
||||
npm test
|
||||
```
|
||||
|
||||
The two explicit loopback ports make the recorded run directly comparable; omit them to select free ports automatically, or change them if already occupied.
|
||||
|
||||
The spike starts only loopback listeners and uses generated, one-day self-signed TLS material in a temporary directory. Chromium accepts that local certificate with `--ignore-certificate-errors`; no proxy, CA installation, traffic interception, VNC, or external fixture is involved. `artifacts/m0-events.sample.jsonl` is the sanitized event output and `artifacts/m0-report.json` is the run result/resource snapshot.
|
||||
|
||||
### Connection and schema
|
||||
|
||||
Clark is launched directly with `--remote-debugging-address=127.0.0.1`. The runner waits on `/json/version`, discovers the ready startup page through `/json/list`, connects to that target's `webSocketDebuggerUrl`, and navigates to a controlled loopback fixture. The fixture runs in a separate process and waits for a CDP trigger before generating test traffic. This avoids `clarkserve`'s current `0.0.0.0` default; a protected gateway and short-lived credentials remain required outside a local Spike.
|
||||
|
||||
Every JSONL line has `schema_version`, `session_id`, `target_id`, and `kind`. HTTP records use `request_id`; WebSocket records use the CDP `requestId` as `connection_id`. Kind-specific fields are nested under `request`, `response`, `headers`, `body`, or `error`. Query strings are removed, sensitive headers are replaced by `[REDACTED]`, WebSocket payloads are never persisted, and bodies over the configured limit are represented by `{state:"omitted",reason:"size_limit",bytes:N}`.
|
||||
|
||||
### Coverage and limits
|
||||
|
||||
| Case | Clark 0.2.1 / Chromium 148.0.7778.96 | Evidence or limit |
|
||||
| --- | --- | --- |
|
||||
| HTTP request → response → finished | Pass | Same `request_id` asserted for `/ok`, `/large`, and `/secure`. |
|
||||
| Failed HTTP request | Pass | Unused loopback port produces `loadingFailed`; error text retained. |
|
||||
| HTTPS without MITM | Pass | Direct browser connection to local TLS fixture; no network intermediary. |
|
||||
| WebSocket lifecycle and frames | Pass | Created, handshake, sent, received, and closed share one `connection_id`; payload omitted. |
|
||||
| `getResponseBody` success | Pass | Called after `loadingFinished` for controlled responses. |
|
||||
| Body size limit | Pass | 2 KiB fixture is not persisted with the 1 KiB default. |
|
||||
| Body unavailable | Pass | Calling on the failed request returns the original CDP error as structured `cdp_error`. |
|
||||
| Target close | Pass | Target connection closes; browser process remains alive. |
|
||||
| Client disconnect | Pass | A second CDP client disconnects; browser process remains alive. |
|
||||
| Cookie / Authorization / Set-Cookie | Pass | `Network.*ExtraInfo` values are redacted before JSONL serialization. |
|
||||
| Cache-specific body availability | Not covered | Cache behavior varies by response/cache mode; retain optional-body semantics in M1. |
|
||||
| Sustained high-rate WebSocket | Not covered | M0 stores metadata only; add frame/event caps before M1 load testing. |
|
||||
| Multi-target/session isolation | Not covered | Explicitly M1 scope. |
|
||||
| Top-level loopback navigation | Intermittent | This build can report `Inspector.detached: Render process gone` before the request reaches the fixture; rerun preserves the same local reproduction condition. |
|
||||
|
||||
The resource numbers are a one-time `ps` snapshot of the browser process tree, not a benchmark. Repeat under the intended container limits before setting capacity targets.
|
||||
|
||||
The runner restarts Clark at most three times only for the observed `Inspector.detached` startup failure. Every failed attempt is retained in `attempt_failures`; other errors fail immediately.
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --test",
|
||||
"spike": "node scripts/m0-spike.mjs"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,659 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const REDACTED = "[REDACTED]";
|
||||
const DEBUG = process.env.M0_DEBUG === "1";
|
||||
const SENSITIVE_HEADERS = new Set([
|
||||
"authorization",
|
||||
"cookie",
|
||||
"location",
|
||||
"proxy-authorization",
|
||||
"referer",
|
||||
"sec-websocket-accept",
|
||||
"sec-websocket-key",
|
||||
"set-cookie",
|
||||
"x-api-key",
|
||||
]);
|
||||
|
||||
export function redactHeaders(headers = {}) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([name, value]) => [
|
||||
name.toLowerCase(),
|
||||
SENSITIVE_HEADERS.has(name.toLowerCase()) ? REDACTED : String(value),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
export function safeUrl(raw) {
|
||||
try {
|
||||
const value = new URL(raw);
|
||||
return `${value.origin}${value.pathname}${value.search ? "?<redacted>" : ""}`;
|
||||
} catch {
|
||||
return "<invalid-url>";
|
||||
}
|
||||
}
|
||||
|
||||
export function boundedBody(body, base64Encoded, maxBytes) {
|
||||
const bytes = Buffer.from(body, base64Encoded ? "base64" : "utf8");
|
||||
if (bytes.length > maxBytes) {
|
||||
return { state: "omitted", reason: "size_limit", bytes: bytes.length };
|
||||
}
|
||||
return {
|
||||
state: "stored",
|
||||
bytes: bytes.length,
|
||||
encoding: "utf8",
|
||||
text: bytes.toString("utf8"),
|
||||
};
|
||||
}
|
||||
|
||||
class CDP {
|
||||
constructor(url) {
|
||||
this.url = url;
|
||||
this.nextId = 1;
|
||||
this.pending = new Map();
|
||||
this.listeners = [];
|
||||
this.closed = new Promise((resolve) => (this.resolveClosed = resolve));
|
||||
}
|
||||
|
||||
async connect() {
|
||||
this.ws = new WebSocket(this.url);
|
||||
await new Promise((resolve, reject) => {
|
||||
this.ws.onopen = resolve;
|
||||
this.ws.onerror = () => reject(new Error("CDP WebSocket connection failed"));
|
||||
});
|
||||
this.ws.onmessage = ({ data }) => this.onMessage(JSON.parse(data));
|
||||
this.ws.onclose = () => {
|
||||
for (const { reject } of this.pending.values()) reject(new Error("CDP WebSocket closed"));
|
||||
this.pending.clear();
|
||||
this.resolveClosed();
|
||||
};
|
||||
return this;
|
||||
}
|
||||
|
||||
onEvent(listener) {
|
||||
this.listeners.push(listener);
|
||||
}
|
||||
|
||||
onMessage(message) {
|
||||
if (DEBUG) console.error(
|
||||
"CDP",
|
||||
message.id ?? message.method,
|
||||
message.method === "Inspector.detached" ? message.params :
|
||||
message.method === "Network.requestWillBeSent" ? safeUrl(message.params.request.url) : "",
|
||||
);
|
||||
if (message.id) {
|
||||
const pending = this.pending.get(message.id);
|
||||
if (!pending) return;
|
||||
this.pending.delete(message.id);
|
||||
if (message.error) pending.reject(new Error(`${message.error.code}: ${message.error.message}`));
|
||||
else pending.resolve(message.result);
|
||||
return;
|
||||
}
|
||||
if (message.method === "Inspector.detached") {
|
||||
const error = new Error(`Inspector.detached: ${message.params.reason}`);
|
||||
for (const { reject } of this.pending.values()) reject(error);
|
||||
this.pending.clear();
|
||||
}
|
||||
for (const listener of this.listeners) listener(message.method, message.params ?? {});
|
||||
}
|
||||
|
||||
call(method, params = {}) {
|
||||
const id = this.nextId++;
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(id);
|
||||
reject(new Error(`CDP command timed out: ${method}`));
|
||||
}, 20_000);
|
||||
this.pending.set(id, {
|
||||
resolve: (value) => { clearTimeout(timer); resolve(value); },
|
||||
reject: (error) => { clearTimeout(timer); reject(error); },
|
||||
});
|
||||
this.ws.send(JSON.stringify({ id, method, params }));
|
||||
});
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.ws.readyState < WebSocket.CLOSING) this.ws.close();
|
||||
await this.closed;
|
||||
}
|
||||
}
|
||||
|
||||
function listen(server, port = 0) {
|
||||
return new Promise((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(port, "127.0.0.1", () => resolve(server.address().port));
|
||||
});
|
||||
}
|
||||
|
||||
async function unusedPort() {
|
||||
const server = net.createServer();
|
||||
const port = await listen(server);
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
return port;
|
||||
}
|
||||
|
||||
function websocketFrame(opcode, payload = "") {
|
||||
const body = Buffer.from(payload);
|
||||
assert(body.length < 126, "fixture frame must stay small");
|
||||
return Buffer.concat([Buffer.from([0x80 | opcode, body.length]), body]);
|
||||
}
|
||||
|
||||
function acceptWebsocket(request, socket) {
|
||||
const accept = createHash("sha1")
|
||||
.update(`${request.headers["sec-websocket-key"]}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`)
|
||||
.digest("base64");
|
||||
socket.write(
|
||||
"HTTP/1.1 101 Switching Protocols\r\n" +
|
||||
"Upgrade: websocket\r\n" +
|
||||
"Connection: Upgrade\r\n" +
|
||||
`Sec-WebSocket-Accept: ${accept}\r\n\r\n`,
|
||||
);
|
||||
|
||||
let buffer = Buffer.alloc(0);
|
||||
socket.on("data", (chunk) => {
|
||||
buffer = Buffer.concat([buffer, chunk]);
|
||||
while (buffer.length >= 2) {
|
||||
const masked = Boolean(buffer[1] & 0x80);
|
||||
let length = buffer[1] & 0x7f;
|
||||
let offset = 2;
|
||||
if (length === 126) {
|
||||
if (buffer.length < 4) return;
|
||||
length = buffer.readUInt16BE(2);
|
||||
offset = 4;
|
||||
}
|
||||
const maskBytes = masked ? 4 : 0;
|
||||
if (buffer.length < offset + maskBytes + length) return;
|
||||
const opcode = buffer[0] & 0x0f;
|
||||
const mask = buffer.subarray(offset, offset + maskBytes);
|
||||
const payload = Buffer.from(buffer.subarray(offset + maskBytes, offset + maskBytes + length));
|
||||
if (masked) for (let i = 0; i < payload.length; i++) payload[i] ^= mask[i % 4];
|
||||
buffer = buffer.subarray(offset + maskBytes + length);
|
||||
if (opcode === 1) socket.write(websocketFrame(1, "server-message"));
|
||||
if (opcode === 8) {
|
||||
socket.end(websocketFrame(8));
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function startFixtures(tempDir) {
|
||||
const secrets = {
|
||||
authorization: randomUUID(),
|
||||
cookie: randomUUID(),
|
||||
setCookie: randomUUID(),
|
||||
};
|
||||
const send = (response, headers, body) => {
|
||||
response.writeHead(200, {
|
||||
...headers,
|
||||
connection: "close",
|
||||
"content-length": Buffer.byteLength(body),
|
||||
});
|
||||
response.end(body);
|
||||
};
|
||||
const key = path.join(tempDir, "fixture-key.pem");
|
||||
const cert = path.join(tempDir, "fixture-cert.pem");
|
||||
const openssl = spawnSync("openssl", [
|
||||
"req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1",
|
||||
"-subj", "/CN=127.0.0.1", "-addext", "subjectAltName=IP:127.0.0.1",
|
||||
"-keyout", key, "-out", cert,
|
||||
], { stdio: "ignore" });
|
||||
assert.equal(openssl.status, 0, "openssl must generate the local HTTPS fixture certificate");
|
||||
|
||||
const httpsServer = https.createServer(
|
||||
{ key: await readFile(key), cert: await readFile(cert) },
|
||||
(_request, response) => {
|
||||
send(response, {
|
||||
"access-control-allow-origin": "*",
|
||||
"access-control-allow-headers": "authorization",
|
||||
"content-type": "application/json",
|
||||
}, '{"secure":true}');
|
||||
},
|
||||
);
|
||||
const httpsPort = await listen(httpsServer);
|
||||
const failurePort = await unusedPort();
|
||||
|
||||
const pageHtml = "<!doctype html><title>M0_READY</title>";
|
||||
const httpServer = http.createServer((request, response) => {
|
||||
if (DEBUG) console.error("HTTP", request.url);
|
||||
const cors = {
|
||||
"access-control-allow-origin": "*",
|
||||
"access-control-allow-headers": "authorization",
|
||||
};
|
||||
if (request.method === "OPTIONS") {
|
||||
send(response, cors, "");
|
||||
return;
|
||||
}
|
||||
if (request.url === "/ok") {
|
||||
send(response, {
|
||||
...cors,
|
||||
"content-type": "application/json",
|
||||
"set-cookie": `m0_response=${secrets.setCookie}; SameSite=Lax`,
|
||||
}, '{"ok":true}');
|
||||
return;
|
||||
}
|
||||
if (request.url === "/large") {
|
||||
send(response, { ...cors, "content-type": "text/plain" }, "x".repeat(2_048));
|
||||
return;
|
||||
}
|
||||
send(response, { "content-type": "text/html" }, pageHtml);
|
||||
});
|
||||
httpServer.on("upgrade", (request, socket) => {
|
||||
if (DEBUG) console.error("HTTP upgrade", request.url);
|
||||
if (request.url === "/ws") acceptWebsocket(request, socket);
|
||||
else socket.destroy();
|
||||
});
|
||||
const httpPort = await listen(httpServer, Number(process.env.M0_HTTP_PORT ?? 0));
|
||||
const pageScript = `
|
||||
(async () => {
|
||||
document.title = "M0_RUNNING";
|
||||
try {
|
||||
await (await fetch("http://127.0.0.1:${httpPort}/ok")).text();
|
||||
await (await fetch("http://127.0.0.1:${httpPort}/large")).text();
|
||||
await (await fetch("https://127.0.0.1:${httpsPort}/secure")).text();
|
||||
await new Promise((resolve, reject) => {
|
||||
const ws = new WebSocket("ws://127.0.0.1:${httpPort}/ws");
|
||||
ws.onopen = () => ws.send("client-message");
|
||||
ws.onmessage = () => ws.close();
|
||||
ws.onclose = resolve;
|
||||
ws.onerror = reject;
|
||||
});
|
||||
try { await fetch("http://127.0.0.1:${failurePort}/missing"); } catch {}
|
||||
document.title = "M0_DONE";
|
||||
} catch (error) {
|
||||
document.title = "M0_ERROR:" + error.name;
|
||||
}
|
||||
})()`;
|
||||
|
||||
return {
|
||||
pageHtml,
|
||||
pageScript,
|
||||
secrets,
|
||||
url: `http://127.0.0.1:${httpPort}/`,
|
||||
close: async () => Promise.all([
|
||||
new Promise((resolve) => httpServer.close(resolve)),
|
||||
new Promise((resolve) => httpsServer.close(resolve)),
|
||||
]),
|
||||
};
|
||||
}
|
||||
|
||||
async function fixtureChild() {
|
||||
const fixtures = await startFixtures(process.env.M0_FIXTURE_DIR);
|
||||
const { close, ...config } = fixtures;
|
||||
process.stdout.write(`${JSON.stringify(config)}\n`);
|
||||
const stop = async () => {
|
||||
await close();
|
||||
process.exit(0);
|
||||
};
|
||||
process.on("SIGINT", stop);
|
||||
process.on("SIGTERM", stop);
|
||||
}
|
||||
|
||||
async function startFixtureProcess(tempDir) {
|
||||
const child = spawn(process.execPath, [fileURLToPath(import.meta.url), "--fixture-child"], {
|
||||
env: { ...process.env, M0_FIXTURE_DIR: tempDir },
|
||||
stdio: ["ignore", "pipe", DEBUG ? "inherit" : "ignore"],
|
||||
});
|
||||
const config = await new Promise((resolve, reject) => {
|
||||
let output = "";
|
||||
const onData = (chunk) => {
|
||||
output += chunk;
|
||||
const newline = output.indexOf("\n");
|
||||
if (newline < 0) return;
|
||||
child.stdout.off("data", onData);
|
||||
resolve(JSON.parse(output.slice(0, newline)));
|
||||
};
|
||||
child.stdout.on("data", onData);
|
||||
child.once("exit", (code) => reject(new Error(`fixture process exited before readiness: ${code}`)));
|
||||
});
|
||||
return {
|
||||
...config,
|
||||
close: async () => {
|
||||
if (child.exitCode !== null) return;
|
||||
child.kill("SIGTERM");
|
||||
await new Promise((resolve) => child.once("exit", resolve));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getJson(url, options = {}) {
|
||||
const response = await fetch(url, options);
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status} from CDP discovery endpoint`);
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function waitFor(check, label, timeoutMs = 10_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const value = await check();
|
||||
if (value) return value;
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${label}`);
|
||||
}
|
||||
|
||||
function processSnapshot(rootPid) {
|
||||
const result = spawnSync("ps", ["-eo", "pid=,ppid=,rss=,pcpu=,comm="], { encoding: "utf8" });
|
||||
assert.equal(result.status, 0, "ps must report the browser process tree");
|
||||
const rows = result.stdout.trim().split("\n").map((line) => {
|
||||
const [pid, ppid, rss, cpu, ...command] = line.trim().split(/\s+/);
|
||||
return { pid: Number(pid), ppid: Number(ppid), rss: Number(rss), cpu: Number(cpu), command: command.join(" ") };
|
||||
});
|
||||
const pids = new Set([rootPid]);
|
||||
for (let changed = true; changed;) {
|
||||
changed = false;
|
||||
for (const row of rows) if (pids.has(row.ppid) && !pids.has(row.pid)) {
|
||||
pids.add(row.pid);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
const tree = rows.filter((row) => pids.has(row.pid));
|
||||
return {
|
||||
process_count: tree.length,
|
||||
rss_kib: tree.reduce((total, row) => total + row.rss, 0),
|
||||
cpu_percent_snapshot: Number(tree.reduce((total, row) => total + row.cpu, 0).toFixed(1)),
|
||||
};
|
||||
}
|
||||
|
||||
function browserPidForPort(port) {
|
||||
const result = spawnSync("ps", ["-eo", "pid=,args="], { encoding: "utf8" });
|
||||
assert.equal(result.status, 0, "ps must locate the browser process");
|
||||
const marker = `--remote-debugging-port=${port}`;
|
||||
const line = result.stdout.split("\n").find((row) => row.includes(marker) && !row.includes("scripts/m0-spike.mjs"));
|
||||
assert(line, `browser process with ${marker} must exist`);
|
||||
return Number(line.trim().split(/\s+/, 1)[0]);
|
||||
}
|
||||
|
||||
function isRunning(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function eventCollector(sessionId, targetId) {
|
||||
const records = [];
|
||||
const requestState = new Map();
|
||||
const push = (kind, fields = {}) => records.push({
|
||||
schema_version: "1.0",
|
||||
session_id: sessionId,
|
||||
target_id: targetId,
|
||||
kind,
|
||||
...fields,
|
||||
});
|
||||
|
||||
const listener = (method, params) => {
|
||||
const requestId = params.requestId;
|
||||
if (requestId && !requestState.has(requestId)) requestState.set(requestId, new Set());
|
||||
const mark = (value) => requestState.get(requestId)?.add(value);
|
||||
switch (method) {
|
||||
case "Network.requestWillBeSent":
|
||||
mark("request");
|
||||
push("http_request", {
|
||||
request_id: requestId,
|
||||
at: params.wallTime,
|
||||
request: {
|
||||
method: params.request.method,
|
||||
url: safeUrl(params.request.url),
|
||||
headers: redactHeaders(params.request.headers),
|
||||
},
|
||||
});
|
||||
break;
|
||||
case "Network.requestWillBeSentExtraInfo":
|
||||
push("http_request_headers", { request_id: requestId, headers: redactHeaders(params.headers) });
|
||||
break;
|
||||
case "Network.responseReceived":
|
||||
mark("response");
|
||||
push("http_response", {
|
||||
request_id: requestId,
|
||||
response: {
|
||||
url: safeUrl(params.response.url),
|
||||
status: params.response.status,
|
||||
protocol: params.response.protocol,
|
||||
mime_type: params.response.mimeType,
|
||||
headers: redactHeaders(params.response.headers),
|
||||
},
|
||||
});
|
||||
break;
|
||||
case "Network.responseReceivedExtraInfo":
|
||||
push("http_response_headers", { request_id: requestId, status: params.statusCode, headers: redactHeaders(params.headers) });
|
||||
break;
|
||||
case "Network.loadingFinished":
|
||||
mark("finished");
|
||||
push("http_finished", { request_id: requestId, encoded_bytes: params.encodedDataLength });
|
||||
break;
|
||||
case "Network.loadingFailed":
|
||||
mark("failed");
|
||||
push("http_failed", {
|
||||
request_id: requestId,
|
||||
error: { text: params.errorText, canceled: params.canceled, blocked_reason: params.blockedReason },
|
||||
});
|
||||
break;
|
||||
case "Network.webSocketCreated":
|
||||
mark("ws_created");
|
||||
push("websocket_created", { connection_id: requestId, url: safeUrl(params.url) });
|
||||
break;
|
||||
case "Network.webSocketWillSendHandshakeRequest":
|
||||
push("websocket_handshake_request", { connection_id: requestId, headers: redactHeaders(params.request.headers) });
|
||||
break;
|
||||
case "Network.webSocketHandshakeResponseReceived":
|
||||
push("websocket_handshake_response", { connection_id: requestId, status: params.response.status, headers: redactHeaders(params.response.headers) });
|
||||
break;
|
||||
case "Network.webSocketFrameSent":
|
||||
case "Network.webSocketFrameReceived": {
|
||||
const direction = method.endsWith("Sent") ? "sent" : "received";
|
||||
mark(`ws_${direction}`);
|
||||
push("websocket_frame", {
|
||||
connection_id: requestId,
|
||||
direction,
|
||||
opcode: params.response.opcode,
|
||||
payload_bytes: Buffer.byteLength(params.response.payloadData),
|
||||
payload: { state: "omitted", reason: "frame_policy" },
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "Network.webSocketClosed":
|
||||
mark("ws_closed");
|
||||
push("websocket_closed", { connection_id: requestId });
|
||||
break;
|
||||
}
|
||||
};
|
||||
return { records, requestState, push, listener };
|
||||
}
|
||||
|
||||
function findRequest(records, pathname) {
|
||||
return records.find((record) => record.kind === "http_request" && new URL(record.request.url.replace("?<redacted>", "")).pathname === pathname)?.request_id;
|
||||
}
|
||||
|
||||
async function collectBody(cdp, collector, requestId, maxBodyBytes) {
|
||||
try {
|
||||
const result = await cdp.call("Network.getResponseBody", { requestId });
|
||||
collector.push("http_body", { request_id: requestId, body: boundedBody(result.body, result.base64Encoded, maxBodyBytes) });
|
||||
} catch (error) {
|
||||
collector.push("http_body", {
|
||||
request_id: requestId,
|
||||
body: { state: "unavailable", reason: "cdp_error", error: error.message },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function assertCoverage(collector, secrets) {
|
||||
const { records, requestState } = collector;
|
||||
for (const pathname of ["/ok", "/large", "/secure"]) {
|
||||
const id = findRequest(records, pathname);
|
||||
assert(id, `request observed for ${pathname}`);
|
||||
assert.deepEqual([...requestState.get(id)].filter((value) => ["request", "response", "finished"].includes(value)).sort(), ["finished", "request", "response"]);
|
||||
}
|
||||
const failed = [...requestState.entries()].find(([, state]) => state.has("failed"));
|
||||
assert(failed?.[1].has("request"), "failed request correlates to its requestId");
|
||||
const websocket = [...requestState.entries()].find(([, state]) => state.has("ws_created"));
|
||||
assert(websocket, "WebSocket connection observed");
|
||||
for (const event of ["ws_sent", "ws_received", "ws_closed"]) assert(websocket[1].has(event), `${event} correlates to the WebSocket connection`);
|
||||
|
||||
const sample = records.map((record) => JSON.stringify(record)).join("\n");
|
||||
for (const secret of Object.values(secrets)) assert(!sample.includes(secret), "sample does not leak a fixture credential");
|
||||
assert(records.some((record) => record.kind === "http_request_headers" && record.headers.cookie === REDACTED), "Cookie is redacted");
|
||||
assert(records.some((record) => record.kind === "http_request_headers" && record.headers.authorization === REDACTED), "Authorization is redacted");
|
||||
assert(records.some((record) => record.kind === "http_response_headers" && record.headers["set-cookie"] === REDACTED), "Set-Cookie is redacted");
|
||||
assert(records.some((record) => record.kind === "http_body" && record.body.reason === "size_limit"), "oversize body is omitted");
|
||||
assert(records.some((record) => record.kind === "http_body" && record.body.state === "unavailable"), "unavailable body is structured");
|
||||
}
|
||||
|
||||
async function main(attemptFailures = []) {
|
||||
const binary = process.env.CLARK_BINARY_PATH;
|
||||
assert(binary, "set CLARK_BINARY_PATH to the clark-browser Chromium binary");
|
||||
const outputDir = path.resolve(process.env.M0_OUTPUT_DIR ?? "artifacts");
|
||||
const maxBodyBytes = Number(process.env.M0_MAX_BODY_BYTES ?? 1_024);
|
||||
assert(Number.isSafeInteger(maxBodyBytes) && maxBodyBytes > 0, "M0_MAX_BODY_BYTES must be a positive integer");
|
||||
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "lume-ctrl-m0-"));
|
||||
const profile = path.join(tempDir, "profile");
|
||||
const cdpPort = Number(process.env.M0_CDP_PORT ?? 0) || await unusedPort();
|
||||
const fixtures = await startFixtureProcess(tempDir);
|
||||
assert.equal((await fetch(fixtures.url)).status, 200, "fixture process must be reachable before Clark starts");
|
||||
spawn("setsid", ["--fork", binary,
|
||||
"--headless=new",
|
||||
"--no-sandbox",
|
||||
"--use-mock-keychain",
|
||||
`--remote-debugging-port=${cdpPort}`,
|
||||
"--remote-debugging-address=127.0.0.1",
|
||||
"--remote-allow-origins=*",
|
||||
`--user-data-dir=${profile}`,
|
||||
"--ignore-certificate-errors",
|
||||
"--disable-gpu",
|
||||
"--disable-features=WebGPU",
|
||||
"about:blank",
|
||||
], { stdio: ["ignore", "ignore", DEBUG ? "inherit" : "ignore"] });
|
||||
|
||||
const sessionId = `m0-${randomUUID()}`;
|
||||
let browserPid;
|
||||
let page;
|
||||
let browserClient;
|
||||
try {
|
||||
const version = await waitFor(
|
||||
() => getJson(`http://127.0.0.1:${cdpPort}/json/version`).catch(() => null),
|
||||
"clark CDP endpoint",
|
||||
20_000,
|
||||
);
|
||||
browserPid = browserPidForPort(cdpPort);
|
||||
const targets = await getJson(`http://127.0.0.1:${cdpPort}/json/list`);
|
||||
page = targets.find((target) => target.type === "page" && target.url === "about:blank");
|
||||
assert(page, "clark must expose its startup page through /json/list");
|
||||
const cdp = await new CDP(page.webSocketDebuggerUrl).connect();
|
||||
const collector = eventCollector(sessionId, page.id);
|
||||
await cdp.call("Page.enable");
|
||||
await cdp.call("Runtime.enable");
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
await cdp.call("Page.navigate", { url: fixtures.url });
|
||||
await waitFor(async () => {
|
||||
const { result } = await cdp.call("Runtime.evaluate", { expression: "document.title", returnByValue: true });
|
||||
return result.value === "M0_READY";
|
||||
}, "fixture page readiness");
|
||||
cdp.onEvent(collector.listener);
|
||||
await cdp.call("Network.enable", { maxTotalBufferSize: 10_000_000, maxResourceBufferSize: 2_000_000 });
|
||||
await cdp.call("Network.setExtraHTTPHeaders", { headers: {
|
||||
Authorization: `Bearer ${fixtures.secrets.authorization}`,
|
||||
Cookie: `m0_cookie=${fixtures.secrets.cookie}`,
|
||||
} });
|
||||
await cdp.call("Runtime.evaluate", { expression: fixtures.pageScript });
|
||||
await waitFor(async () => {
|
||||
const { result } = await cdp.call("Runtime.evaluate", { expression: "document.title", returnByValue: true });
|
||||
if (String(result.value).startsWith("M0_ERROR")) throw new Error(result.value);
|
||||
return result.value === "M0_DONE";
|
||||
}, "fixture completion");
|
||||
await waitFor(() => [...collector.requestState.values()].some((state) => state.has("ws_closed")), "WebSocket closure");
|
||||
|
||||
for (const pathname of ["/ok", "/large", "/secure"]) {
|
||||
await collectBody(cdp, collector, findRequest(collector.records, pathname), maxBodyBytes);
|
||||
}
|
||||
const failedId = [...collector.requestState.entries()].find(([, state]) => state.has("failed"))?.[0];
|
||||
await collectBody(cdp, collector, failedId, maxBodyBytes);
|
||||
|
||||
const resources = processSnapshot(browserPid);
|
||||
browserClient = await new CDP(version.webSocketDebuggerUrl).connect();
|
||||
await browserClient.call("Target.closeTarget", { targetId: page.id });
|
||||
await cdp.closed;
|
||||
collector.push("target_closed", { reason: "Target.closeTarget" });
|
||||
assert(isRunning(browserPid), "closing the page does not exit clark-browser");
|
||||
|
||||
const disconnectTarget = await getJson(`http://127.0.0.1:${cdpPort}/json/new?about:blank`, { method: "PUT" });
|
||||
const disconnectClient = await new CDP(disconnectTarget.webSocketDebuggerUrl).connect();
|
||||
await disconnectClient.close();
|
||||
collector.push("cdp_disconnected", { reason: "client_closed" });
|
||||
assert(isRunning(browserPid), "disconnecting CDP does not exit clark-browser");
|
||||
|
||||
assertCoverage(collector, fixtures.secrets);
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
await writeFile(path.join(outputDir, "m0-events.sample.jsonl"), `${collector.records.map((record) => JSON.stringify(record)).join("\n")}\n`);
|
||||
await writeFile(path.join(outputDir, "m0-report.json"), `${JSON.stringify({
|
||||
result: "pass",
|
||||
clark: {
|
||||
wrapper_version: "0.2.1",
|
||||
chromium_version: version.Browser,
|
||||
protocol_version: version["Protocol-Version"],
|
||||
cdp_bind: "127.0.0.1",
|
||||
},
|
||||
coverage: {
|
||||
http: "pass",
|
||||
failed_request: "pass",
|
||||
https_without_mitm: "pass",
|
||||
websocket: "pass",
|
||||
response_body_success: "pass",
|
||||
response_body_size_limit: "pass",
|
||||
response_body_unavailable: "pass",
|
||||
target_close_survives: "pass",
|
||||
cdp_disconnect_survives: "pass",
|
||||
sensitive_header_redaction: "pass",
|
||||
},
|
||||
resources,
|
||||
event_count: collector.records.length,
|
||||
max_body_bytes: maxBodyBytes,
|
||||
attempt_failures: attemptFailures,
|
||||
}, null, 2)}\n`);
|
||||
console.log(`M0 PASS: ${collector.records.length} sanitized events; RSS snapshot ${resources.rss_kib} KiB`);
|
||||
} finally {
|
||||
if (browserClient) await browserClient.close().catch(() => {});
|
||||
if (browserPid && isRunning(browserPid)) process.kill(browserPid, "SIGTERM");
|
||||
if (browserPid) await waitFor(() => !isRunning(browserPid), "browser shutdown").catch(() => {});
|
||||
await fixtures.close();
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function runSpike() {
|
||||
const failures = [];
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
try {
|
||||
await main(failures);
|
||||
return;
|
||||
} catch (error) {
|
||||
failures.push(error.message);
|
||||
const retryable = error.message.startsWith("Inspector.detached:") ||
|
||||
error.message === "CDP command timed out: Page.navigate";
|
||||
if (!retryable || attempt === 3) throw error;
|
||||
console.error(`M0 retry ${attempt}/3: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const run = process.argv[2] === "--fixture-child" ? fixtureChild : runSpike;
|
||||
run().catch((error) => {
|
||||
console.error(`M0 FAIL: ${error.message}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import test from "node:test";
|
||||
|
||||
import { boundedBody, redactHeaders, safeUrl } from "../scripts/m0-spike.mjs";
|
||||
|
||||
test("export policy redacts credentials and bounds bodies", () => {
|
||||
const sensitive = randomUUID();
|
||||
assert.deepEqual(redactHeaders({ Cookie: sensitive, Authorization: sensitive, Referer: "https://example.test/private", "Set-Cookie": sensitive, Accept: "*/*" }), {
|
||||
accept: "*/*",
|
||||
authorization: "[REDACTED]",
|
||||
cookie: "[REDACTED]",
|
||||
referer: "[REDACTED]",
|
||||
"set-cookie": "[REDACTED]",
|
||||
});
|
||||
assert.equal(safeUrl("https://example.test/path?token=secret"), "https://example.test/path?<redacted>");
|
||||
assert.deepEqual(boundedBody("hello", false, 5), { state: "stored", bytes: 5, encoding: "utf8", text: "hello" });
|
||||
assert.deepEqual(boundedBody("hello!", false, 5), { state: "omitted", reason: "size_limit", bytes: 6 });
|
||||
});
|
||||
Reference in New Issue
Block a user