H-391: bound network collection follow-ups (#3)

This commit was merged in pull request #3.
This commit is contained in:
2026-08-21 13:44:16 +08:00
parent b4fb8ddbde
commit d4b9c66629
5 changed files with 471 additions and 103 deletions
+273 -43
View File
@@ -45,26 +45,45 @@ export function safeUrl(raw) {
}
}
export function boundedBody(body, base64Encoded, maxBytes) {
export function boundedBody(body, base64Encoded, maxBytes, source = "network") {
const bytes = Buffer.from(body, base64Encoded ? "base64" : "utf8");
if (bytes.length > maxBytes) {
return { state: "size_limit", storage: "omitted", bytes: bytes.length };
return { state: "size_limit", source, encoding: base64Encoded ? "base64" : "utf8", storage: "omitted", reason: "size_limit", bytes: bytes.length };
}
return {
state: "fetched",
source,
encoding: base64Encoded ? "base64" : "utf8",
storage: "omitted",
reason: "default_body_policy",
bytes: bytes.length,
};
}
export function queryRecords(records, sessionId, { cursor = 0, limit = 100 } = {}) {
assert(sessionId, "sessionId is required");
assert(Number.isSafeInteger(cursor) && cursor >= 0, "cursor must be a non-negative integer");
assert(Number.isSafeInteger(limit) && limit > 0, "limit must be a positive integer");
// ponytail: linear scan fits the single-instance Spike; index by session if retained volume grows.
const matching = records.filter((record) => record.session_id === sessionId);
const items = matching.slice(cursor, cursor + limit);
return { items, next_cursor: cursor + items.length < matching.length ? cursor + items.length : null };
}
export function exportSession(records, sessionId) {
return queryRecords(records, sessionId, { limit: Number.MAX_SAFE_INTEGER }).items
.map((record) => JSON.stringify(record))
.join("\n");
}
export function shouldRetryStartup(detachedReason) {
return detachedReason === "Render process gone.";
}
class StartupNavigateFailure extends Error {
constructor(cause) {
constructor(cause, detachedReason) {
super(cause.message, { cause });
this.detachedReason = detachedReason;
}
}
@@ -79,11 +98,15 @@ class CDP {
this.detached = new Promise((resolve) => (this.resolveDetached = resolve));
}
async connect() {
async connect(timeoutMs = 20_000) {
this.ws = new WebSocket(this.url);
await new Promise((resolve, reject) => {
this.ws.onopen = resolve;
this.ws.onerror = () => reject(new Error("CDP WebSocket connection failed"));
const timer = setTimeout(() => {
this.ws.close();
reject(new Error("CDP WebSocket connection timed out"));
}, timeoutMs);
this.ws.onopen = () => { clearTimeout(timer); resolve(); };
this.ws.onerror = () => { clearTimeout(timer); reject(new Error("CDP WebSocket connection failed")); };
});
this.ws.onmessage = ({ data }) => this.onMessage(JSON.parse(data));
this.ws.onclose = ({ code, reason, wasClean }) => {
@@ -173,8 +196,11 @@ async function unusedPort() {
function websocketFrame(opcode, payload = "") {
const body = Buffer.from(payload);
assert(body.length < 126, "fixture frame must stay small");
return Buffer.concat([Buffer.from([0x80 | opcode, body.length]), body]);
assert(body.length <= 65_535, "fixture frame must fit a 16-bit length");
const header = body.length < 126
? Buffer.from([0x80 | opcode, body.length])
: Buffer.from([0x80 | opcode, 126, body.length >> 8, body.length & 0xff]);
return Buffer.concat([header, body]);
}
function acceptWebsocket(request, socket) {
@@ -276,6 +302,14 @@ async function startFixtures(tempDir) {
send(response, { ...cors, "content-type": "text/plain" }, "x".repeat(2_048));
return;
}
if (request.url === "/binary") {
send(response, { ...cors, "content-type": "application/octet-stream" }, Buffer.from([0, 255, 1, 254]));
return;
}
if (request.url === "/cache") {
send(response, { ...cors, "cache-control": "public, max-age=3600", "content-type": "application/json" }, '{"cached":true}');
return;
}
if (request.url === "/secret") {
send(response, { ...cors, "content-type": "application/json" }, JSON.stringify({ secret: secrets.body }));
return;
@@ -295,11 +329,18 @@ async function startFixtures(tempDir) {
await (await fetch("http://127.0.0.1:${httpPort}/ok")).text();
await (await fetch("http://127.0.0.1:${httpPort}/secret")).text();
await (await fetch("http://127.0.0.1:${httpPort}/large")).text();
await (await fetch("http://127.0.0.1:${httpPort}/binary")).arrayBuffer();
await (await fetch("http://127.0.0.1:${httpPort}/cache", { cache: "force-cache" })).text();
await (await fetch("http://127.0.0.1:${httpPort}/cache", { cache: "force-cache" })).text();
await (await fetch("https://127.0.0.1:${httpsPort}/secure")).text();
await new Promise((resolve, reject) => {
const ws = new WebSocket("ws://127.0.0.1:${httpPort}/ws");
ws.onopen = () => ws.send("client-message");
ws.onmessage = () => ws.close();
let received = 0;
ws.onopen = () => {
ws.send("x".repeat(2048));
for (let index = 0; index < 20; index++) ws.send("client-message");
};
ws.onmessage = () => { if (++received === 21) ws.close(); };
ws.onclose = resolve;
ws.onerror = reject;
});
@@ -400,6 +441,35 @@ function processSnapshot(rootPid) {
};
}
export function summarizeProcessSamples(samples, intervalMs) {
assert(samples.length > 1, "resource baseline requires multiple samples");
const summary = (name) => {
const values = samples.map((sample) => sample[name]).sort((a, b) => a - b);
return {
min: values[0],
max: values.at(-1),
mean: Number((values.reduce((total, value) => total + value, 0) / values.length).toFixed(1)),
p95: values[Math.ceil(values.length * 0.95) - 1],
};
};
return {
sample_count: samples.length,
interval_ms: intervalMs,
process_count: { min: Math.min(...samples.map((sample) => sample.process_count)), max: Math.max(...samples.map((sample) => sample.process_count)) },
rss_kib: summary("rss_kib"),
cpu_percent: summary("cpu_percent_snapshot"),
};
}
async function processBaseline(rootPid, sampleCount, intervalMs) {
const samples = [];
for (let index = 0; index < sampleCount; index++) {
if (index) await new Promise((resolve) => setTimeout(resolve, intervalMs));
samples.push(processSnapshot(rootPid));
}
return summarizeProcessSamples(samples, intervalMs);
}
function isRunning(pid) {
try {
process.kill(pid, 0);
@@ -409,16 +479,54 @@ function isRunning(pid) {
}
}
function eventCollector(sessionId, targetId) {
export function eventCollector(sessionId, targetId, {
maxWebSocketFrameBytes = 1_024,
maxWebSocketEventsPerSecond = 16,
maxWebSocketEventsPerConnection = 16,
} = {}) {
for (const [name, value] of Object.entries({ maxWebSocketFrameBytes, maxWebSocketEventsPerSecond, maxWebSocketEventsPerConnection })) {
assert(Number.isSafeInteger(value) && value > 0, `${name} must be a positive integer`);
}
const records = [];
const requestState = new Map();
const push = (kind, fields = {}) => records.push({
schema_version: "1.0",
session_id: sessionId,
target_id: targetId,
kind,
...fields,
});
const bodySources = new Map();
const webSocketLimits = new Map();
const push = (kind, fields = {}) => {
const record = {
schema_version: "1.0",
session_id: sessionId,
target_id: targetId,
kind,
...fields,
};
records.push(record);
return record;
};
const dropWebSocketFrame = (connectionId, reason, payloadBytes) => {
let state = webSocketLimits.get(connectionId);
if (!state) {
state = { window: null, windowEvents: 0, emitted: 0 };
webSocketLimits.set(connectionId, state);
}
if (!state.dropRecord) state.dropRecord = push("websocket_frames_dropped", {
connection_id: connectionId,
dropped: {
count: 0,
reasons: {},
max_payload_bytes: 0,
limits: {
frame_bytes: maxWebSocketFrameBytes,
events_per_second: maxWebSocketEventsPerSecond,
events_per_connection: maxWebSocketEventsPerConnection,
},
},
});
const dropped = state.dropRecord.dropped;
dropped.count++;
dropped.reasons[reason] = (dropped.reasons[reason] ?? 0) + 1;
dropped.max_payload_bytes = Math.max(dropped.max_payload_bytes, payloadBytes);
};
const listener = (method, params) => {
const requestId = params.requestId;
@@ -427,6 +535,7 @@ function eventCollector(sessionId, targetId) {
switch (method) {
case "Network.requestWillBeSent":
mark("request");
bodySources.set(requestId, "network");
push("http_request", {
request_id: requestId,
at: params.wallTime,
@@ -437,11 +546,18 @@ function eventCollector(sessionId, targetId) {
},
});
break;
case "Network.requestServedFromCache":
bodySources.set(requestId, "memory_cache");
push("http_cache_hit", { request_id: requestId, source: "memory_cache" });
break;
case "Network.requestWillBeSentExtraInfo":
push("http_request_headers", { request_id: requestId, headers: redactHeaders(params.headers) });
break;
case "Network.responseReceived":
mark("response");
if (params.response.fromServiceWorker) bodySources.set(requestId, "service_worker");
else if (params.response.fromDiskCache) bodySources.set(requestId, "disk_cache");
else if (params.response.fromPrefetchCache) bodySources.set(requestId, "prefetch_cache");
push("http_response", {
request_id: requestId,
response: {
@@ -450,6 +566,10 @@ function eventCollector(sessionId, targetId) {
protocol: params.response.protocol,
mime_type: params.response.mimeType,
headers: redactHeaders(params.response.headers),
cache: bodySources.has(requestId) ? {
state: bodySources.get(requestId) === "network" ? "miss" : "hit",
source: bodySources.get(requestId),
} : { state: "unknown" },
},
});
break;
@@ -481,11 +601,34 @@ function eventCollector(sessionId, targetId) {
case "Network.webSocketFrameReceived": {
const direction = method.endsWith("Sent") ? "sent" : "received";
mark(`ws_${direction}`);
const payloadBytes = params.response.opcode === 2
? Buffer.from(params.response.payloadData, "base64").length
: Buffer.byteLength(params.response.payloadData);
let state = webSocketLimits.get(requestId);
if (!state) {
state = { window: null, windowEvents: 0, emitted: 0 };
webSocketLimits.set(requestId, state);
}
const window = Math.floor(params.timestamp ?? 0);
if (state.window !== window) {
state.window = window;
state.windowEvents = 0;
}
state.windowEvents++;
let dropReason;
if (payloadBytes > maxWebSocketFrameBytes) dropReason = "frame_size_limit";
else if (state.windowEvents > maxWebSocketEventsPerSecond) dropReason = "rate_limit";
else if (state.emitted >= maxWebSocketEventsPerConnection) dropReason = "event_limit";
if (dropReason) {
dropWebSocketFrame(requestId, dropReason, payloadBytes);
break;
}
state.emitted++;
push("websocket_frame", {
connection_id: requestId,
direction,
opcode: params.response.opcode,
payload_bytes: Buffer.byteLength(params.response.payloadData),
payload_bytes: payloadBytes,
payload: { state: "omitted", reason: "frame_policy" },
});
break;
@@ -496,28 +639,33 @@ function eventCollector(sessionId, targetId) {
break;
}
};
return { records, requestState, push, listener };
return { records, requestState, bodySources, push, listener };
}
function findRequest(records, pathname) {
return records.find((record) => record.kind === "http_request" && new URL(record.request.url.replace("?<redacted>", "")).pathname === pathname)?.request_id;
}
function findRequests(records, pathname) {
return records.filter((record) => record.kind === "http_request" && new URL(record.request.url.replace("?<redacted>", "")).pathname === pathname).map((record) => record.request_id);
}
async function collectBody(cdp, collector, requestId, maxBodyBytes) {
const source = collector.bodySources.get(requestId) ?? "network";
try {
const result = await cdp.call("Network.getResponseBody", { requestId });
collector.push("http_body", { request_id: requestId, body: boundedBody(result.body, result.base64Encoded, maxBodyBytes) });
collector.push("http_body", { request_id: requestId, body: boundedBody(result.body, result.base64Encoded, maxBodyBytes, source) });
} catch (error) {
collector.push("http_body", {
request_id: requestId,
body: { state: "unavailable", storage: "omitted", reason: "cdp_error", error: error.message },
body: { state: "unavailable", source, encoding: "unknown", storage: "omitted", reason: "cdp_error", error: error.message },
});
}
}
function assertCoverage(collector, secrets) {
const { records, requestState } = collector;
for (const pathname of ["/ok", "/secret", "/large", "/secure"]) {
for (const pathname of ["/ok", "/secret", "/large", "/binary", "/cache", "/secure"]) {
const id = findRequest(records, pathname);
assert(id, `request observed for ${pathname}`);
assert.deepEqual([...requestState.get(id)].filter((value) => ["request", "response", "finished"].includes(value)).sort(), ["finished", "request", "response"]);
@@ -534,19 +682,47 @@ function assertCoverage(collector, secrets) {
assert(records.some((record) => record.kind === "http_request_headers" && record.headers.authorization === REDACTED), "Authorization is redacted");
assert(records.some((record) => record.kind === "http_response_headers" && record.headers["set-cookie"] === REDACTED), "Set-Cookie is redacted");
const secretBody = records.find((record) => record.kind === "http_body" && record.request_id === findRequest(records, "/secret"));
assert.deepEqual(secretBody?.body, { state: "fetched", storage: "omitted", reason: "default_body_policy", bytes: Buffer.byteLength(JSON.stringify({ secret: secrets.body })) });
assert.deepEqual(secretBody?.body, { state: "fetched", source: "network", encoding: "utf8", storage: "omitted", reason: "default_body_policy", bytes: Buffer.byteLength(JSON.stringify({ secret: secrets.body })) });
assert(records.some((record) => record.kind === "http_body" && record.body.state === "size_limit" && record.body.storage === "omitted"), "oversize body is omitted");
assert(records.some((record) => record.kind === "http_body" && record.body.state === "unavailable"), "unavailable body is structured");
assert(records.some((record) => record.kind === "http_body" && record.body.source.endsWith("_cache")), "cached body source is explicit");
assert(records.some((record) => record.kind === "http_body" && record.body.encoding === "base64"), "binary body encoding is explicit");
assert(records.some((record) => record.kind === "websocket_frames_dropped" && record.dropped.reasons.frame_size_limit), "oversize WebSocket frame drop is observable");
assert(records.some((record) => record.kind === "websocket_frames_dropped" && record.dropped.count > 0), "high-rate WebSocket drops are aggregated");
assert(records.some((record) => record.kind === "cdp_disconnected" && record.source === "socket"), "real CDP socket close is recorded");
assert(records.some((record) => record.kind === "cdp_reconnected" && record.verified), "CDP reconnect is verified");
}
async function main(attemptFailures = []) {
function assertIsolation(records, sessions) {
for (const { sessionId, targetId, otherSessionId } of sessions) {
const queried = queryRecords(records, sessionId, { limit: 1_000 }).items;
assert(queried.length > 0, `records exist for ${sessionId}`);
assert(queried.every((record) => record.session_id === sessionId && record.target_id === targetId), `query stays inside ${sessionId}`);
const exported = exportSession(records, sessionId);
assert(!exported.includes(otherSessionId), `export does not leak ${otherSessionId}`);
assert(queried.some((record) => record.kind === "cdp_disconnected"), `disconnect stays observable for ${sessionId}`);
}
}
async function main(attemptFailures = [], startup = { requested_samples: 1, successful_samples: 1, max_renderer_failure_rate: 1 }) {
const binary = process.env.CLARK_BINARY_PATH;
assert(binary, "set CLARK_BINARY_PATH to the clark-browser Chromium binary");
const outputDir = path.resolve(process.env.M0_OUTPUT_DIR ?? "artifacts");
const maxBodyBytes = Number(process.env.M0_MAX_BODY_BYTES ?? 1_024);
assert(Number.isSafeInteger(maxBodyBytes) && maxBodyBytes > 0, "M0_MAX_BODY_BYTES must be a positive integer");
const limits = {
maxWebSocketFrameBytes: Number(process.env.M0_MAX_WS_FRAME_BYTES ?? 1_024),
maxWebSocketEventsPerSecond: Number(process.env.M0_MAX_WS_EVENTS_PER_SECOND ?? 16),
maxWebSocketEventsPerConnection: Number(process.env.M0_MAX_WS_EVENTS_PER_CONNECTION ?? 16),
};
const resourceSampleCount = Number(process.env.M0_RESOURCE_SAMPLES ?? 5);
const resourceSampleIntervalMs = Number(process.env.M0_RESOURCE_SAMPLE_INTERVAL_MS ?? 200);
const maxRssKib = Number(process.env.M0_MAX_RSS_KIB ?? 1_500_000);
const maxProcessCount = Number(process.env.M0_MAX_PROCESS_COUNT ?? 20);
assert(Number.isSafeInteger(resourceSampleCount) && resourceSampleCount > 1, "M0_RESOURCE_SAMPLES must be an integer greater than one");
assert(Number.isSafeInteger(resourceSampleIntervalMs) && resourceSampleIntervalMs > 0, "M0_RESOURCE_SAMPLE_INTERVAL_MS must be a positive integer");
assert(Number.isSafeInteger(maxRssKib) && maxRssKib > 0, "M0_MAX_RSS_KIB must be a positive integer");
assert(Number.isSafeInteger(maxProcessCount) && maxProcessCount > 0, "M0_MAX_PROCESS_COUNT must be a positive integer");
const tempDir = await mkdtemp(path.join(os.tmpdir(), "lume-ctrl-m0-"));
const profile = path.join(tempDir, "profile");
@@ -583,7 +759,7 @@ async function main(attemptFailures = []) {
page = targets.find((target) => target.type === "page" && target.url === "about:blank");
assert(page, "clark must expose its startup page through /json/list");
const cdp = await new CDP(page.webSocketDebuggerUrl).connect();
const collector = eventCollector(sessionId, page.id);
const collector = eventCollector(sessionId, page.id, limits);
cdp.onClose((details) => collector.push("cdp_disconnected", { source: "socket", ...details }));
await cdp.call("Page.enable");
await cdp.call("Runtime.enable");
@@ -596,7 +772,7 @@ async function main(attemptFailures = []) {
} catch (error) {
const detachedReason = cdp.detachedReason ?? await cdp.waitForDetached(1_000);
if (DEBUG) console.error("startup failure", { detached_reason: detachedReason, error: error.message });
if (shouldRetryStartup(detachedReason)) throw new StartupNavigateFailure(error);
if (shouldRetryStartup(detachedReason)) throw new StartupNavigateFailure(error, detachedReason);
throw error;
}
cdp.onEvent(collector.listener);
@@ -613,13 +789,21 @@ async function main(attemptFailures = []) {
}, "fixture completion");
await waitFor(() => [...collector.requestState.values()].some((state) => state.has("ws_closed")), "WebSocket closure");
for (const pathname of ["/ok", "/secret", "/large", "/secure"]) {
for (const pathname of ["/ok", "/secret", "/large", "/binary", "/secure"]) {
await collectBody(cdp, collector, findRequest(collector.records, pathname), maxBodyBytes);
}
const cachedId = findRequests(collector.records, "/cache").find((requestId) => {
const source = collector.bodySources.get(requestId);
return source && source !== "network";
});
assert(cachedId, "a repeated force-cache request must be served from browser cache");
await collectBody(cdp, collector, cachedId, maxBodyBytes);
const failedId = [...collector.requestState.entries()].find(([, state]) => state.has("failed"))?.[0];
await collectBody(cdp, collector, failedId, maxBodyBytes);
const resources = processSnapshot(browserPid);
const resources = await processBaseline(browserPid, resourceSampleCount, resourceSampleIntervalMs);
assert(resources.rss_kib.max <= maxRssKib, `browser RSS ${resources.rss_kib.max} KiB exceeds ${maxRssKib} KiB`);
assert(resources.process_count.max <= maxProcessCount, `browser process count ${resources.process_count.max} exceeds ${maxProcessCount}`);
browserClient = await new CDP(version.webSocketDebuggerUrl).connect();
await browserClient.call("Target.closeTarget", { targetId: page.id });
await cdp.closed;
@@ -629,14 +813,33 @@ async function main(attemptFailures = []) {
assert(collector.records.some((record) => record.kind === "cdp_disconnected" && record.source === "socket"), "target closure emits a socket disconnect event");
const reconnectTarget = await getJson(`http://127.0.0.1:${cdpPort}/json/new?about:blank`, { method: "PUT" });
reconnectClient = await new CDP(reconnectTarget.webSocketDebuggerUrl).connect();
const reconnectSessionId = `m0-${randomUUID()}`;
const reconnectCollector = eventCollector(reconnectSessionId, reconnectTarget.id, limits);
reconnectClient.onClose((details) => reconnectCollector.push("cdp_disconnected", { source: "socket", ...details }));
const { result: reconnectResult } = await reconnectClient.call("Runtime.evaluate", { expression: "1 + 1", returnByValue: true });
assert.equal(reconnectResult.value, 2, "CDP reconnect can execute a command");
collector.push("cdp_reconnected", { target_id: reconnectTarget.id, verified: true });
reconnectCollector.push("cdp_reconnected", { verified: true });
assert(isRunning(browserPid), "disconnecting and reconnecting CDP does not exit clark-browser");
await browserClient.call("Target.closeTarget", { targetId: reconnectTarget.id });
await reconnectClient.closed;
reconnectCollector.push("target_closed", { reason: "Target.closeTarget" });
assert(isRunning(browserPid), "closing a second session target does not exit clark-browser");
assertCoverage(collector, fixtures.secrets);
const records = [...collector.records, ...reconnectCollector.records];
assertCoverage({ ...collector, records }, fixtures.secrets);
assertIsolation(records, [
{ sessionId, targetId: page.id, otherSessionId: reconnectSessionId },
{ sessionId: reconnectSessionId, targetId: reconnectTarget.id, otherSessionId: sessionId },
]);
const rendererFailures = attemptFailures.filter((failure) => failure.detached_reason === "Render process gone.").length;
const startupAttempts = startup.successful_samples + attemptFailures.length;
const rendererFailureRate = Number((rendererFailures / startupAttempts).toFixed(3));
const rendererConclusion = rendererFailureRate <= startup.max_renderer_failure_rate ? "within_threshold" : "exceeds_threshold";
if (startup.successful_samples === startup.requested_samples) {
assert.equal(rendererConclusion, "within_threshold", `renderer failure rate ${rendererFailureRate} exceeds ${startup.max_renderer_failure_rate}`);
}
await mkdir(outputDir, { recursive: true });
await writeFile(path.join(outputDir, "m0-events.sample.jsonl"), `${collector.records.map((record) => JSON.stringify(record)).join("\n")}\n`);
await writeFile(path.join(outputDir, "m0-events.sample.jsonl"), `${records.map((record) => JSON.stringify(record)).join("\n")}\n`);
await writeFile(path.join(outputDir, "m0-report.json"), `${JSON.stringify({
result: "pass",
clark: {
@@ -653,17 +856,34 @@ async function main(attemptFailures = []) {
response_body_success: "pass",
response_body_size_limit: "pass",
response_body_unavailable: "pass",
cached_body_semantics: "pass",
binary_body_semantics: "pass",
websocket_limits: "pass",
multi_session_isolation: "pass",
target_close_survives: "pass",
cdp_disconnect_reconnect: "pass",
sensitive_header_redaction: "pass",
sensitive_body_redaction: "pass",
},
resources,
event_count: collector.records.length,
resource_thresholds: { max_rss_kib: maxRssKib, max_process_count: maxProcessCount, conclusion: "within_threshold" },
startup: {
...startup,
attempts: startupAttempts,
renderer_failures: rendererFailures,
renderer_failure_rate: rendererFailureRate,
conclusion: rendererConclusion,
},
event_count: records.length,
max_body_bytes: maxBodyBytes,
websocket_limits: {
frame_bytes: limits.maxWebSocketFrameBytes,
events_per_second: limits.maxWebSocketEventsPerSecond,
events_per_connection: limits.maxWebSocketEventsPerConnection,
},
attempt_failures: attemptFailures,
}, null, 2)}\n`);
console.log(`M0 PASS: ${collector.records.length} sanitized events; RSS snapshot ${resources.rss_kib} KiB`);
console.log(`M0 PASS: ${records.length} sanitized events; RSS p95 ${resources.rss_kib.p95} KiB`);
} finally {
if (reconnectClient) await reconnectClient.close().catch(() => {});
if (browserClient) await browserClient.close().catch(() => {});
@@ -676,15 +896,25 @@ async function main(attemptFailures = []) {
async function runSpike() {
const failures = [];
for (let attempt = 1; attempt <= 3; attempt++) {
try {
await main(failures);
return;
} catch (error) {
const original = error.cause ?? error;
failures.push(original.message);
if (!(error instanceof StartupNavigateFailure) || attempt === 3) throw original;
console.error(`M0 retry ${attempt}/3: ${original.message}`);
const startup = {
requested_samples: Number(process.env.M0_STARTUP_SAMPLES ?? 3),
successful_samples: 0,
max_renderer_failure_rate: Number(process.env.M0_MAX_RENDERER_FAILURE_RATE ?? 0.34),
};
assert(Number.isSafeInteger(startup.requested_samples) && startup.requested_samples > 1, "M0_STARTUP_SAMPLES must be an integer greater than one");
assert(Number.isFinite(startup.max_renderer_failure_rate) && startup.max_renderer_failure_rate >= 0 && startup.max_renderer_failure_rate <= 1, "M0_MAX_RENDERER_FAILURE_RATE must be between zero and one");
for (let sample = 1; sample <= startup.requested_samples; sample++) {
for (let attempt = 1; attempt <= 3; attempt++) {
startup.successful_samples = sample;
try {
await main(failures, startup);
break;
} catch (error) {
const original = error.cause ?? error;
failures.push({ sample, attempt, error: original.message, detached_reason: error.detachedReason ?? null });
if (!(error instanceof StartupNavigateFailure) || attempt === 3) throw original;
console.error(`M0 startup sample ${sample}/${startup.requested_samples} retry ${attempt}/3: ${original.message}`);
}
}
}
}