feat: harden control-plane deployment

scripts/control-plane-data.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  echo "usage: $0 backup <data-file> <backup-dir> | restore <backup-file> <data-file>" >&2
+  exit 2
+}
+
+[ "$#" -eq 3 ] || usage
+command=$1
+source=$2
+target=$3
+
+case "$command" in
+  backup)
+    [ -f "$source" ] || { echo "data file does not exist: $source" >&2; exit 1; }
+    mkdir -p "$target"
+    name="$(basename "$source").$(date -u +%Y%m%dT%H%M%SZ).manual.json"
+    temporary="$target/.wxagent-backup-$$.tmp"
+    trap 'rm -f "$temporary"' EXIT
+    install -m 600 "$source" "$temporary"
+    mv -f "$temporary" "$target/$name"
+    sha256sum "$target/$name"
+    ;;
+  restore)
+    [ -f "$source" ] || { echo "backup file does not exist: $source" >&2; exit 1; }
+    directory=$(dirname "$target")
+    mkdir -p "$directory"
+    lock="$target.lock"
+    exec 9>"$lock"
+    flock -n 9 || { echo "data file is in use; stop the active control plane first" >&2; exit 1; }
+    temporary="$directory/.wxagent-restore-$$.tmp"
+    trap 'rm -f "$temporary"' EXIT
+    install -m 600 "$source" "$temporary"
+    python3 -m json.tool "$temporary" >/dev/null
+    mv -f "$temporary" "$target"
+    sha256sum "$target"
+    ;;
+  *)
+    usage
+    ;;
+esac

scripts/control-plane-scale-smoke.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Bounded control-plane readiness/read-task concurrency smoke test."""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import http.client
+import json
+import os
+import ssl
+import statistics
+import sys
+import time
+import uuid
+from urllib.parse import urlsplit
+
+
+def request(base_url: str, path: str, method: str, body: object | None, headers: dict[str, str], timeout: float):
+    url = base_url.rstrip("/") + path
+    parts = urlsplit(url)
+    if parts.scheme.lower() not in {"http", "https"} or not parts.hostname or parts.username or parts.password:
+        raise ValueError("base URL must be an http/https URL without user information")
+    target = parts.path or "/"
+    if parts.query:
+        target += "?" + parts.query
+    data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
+    connection: http.client.HTTPConnection
+    if parts.scheme.lower() == "https":
+        connection = http.client.HTTPSConnection(parts.hostname, parts.port, timeout=timeout, context=ssl.create_default_context())
+    else:
+        connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=timeout)
+    started = time.perf_counter()
+    try:
+        connection.request(method, target, body=data, headers={"Accept": "application/json", **headers})
+        response = connection.getresponse()
+        return response.status, time.perf_counter() - started, response.read()
+    except OSError as error:
+        return 599, time.perf_counter() - started, str(error).encode()
+    finally:
+        connection.close()
+
+
+def percentile(values: list[float], p: float) -> float:
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    index = min(len(ordered) - 1, round((len(ordered) - 1) * p))
+    return ordered[index]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base-url", default=os.getenv("WXAGENT_CONTROL_PLANE_URL", "http://127.0.0.1:8090"))
+    parser.add_argument("--mode", choices=("ready", "reads"), default="reads")
+    parser.add_argument("--web-user", default=os.getenv("WXAGENT_SCALE_WEB_USER", "admin"))
+    parser.add_argument("--node-id", default=os.getenv("WXAGENT_SCALE_NODE_ID", ""))
+    parser.add_argument("--account-id", default=os.getenv("WXAGENT_SCALE_ACCOUNT_ID", ""))
+    parser.add_argument("--requests", type=int, default=100)
+    parser.add_argument("--workers", type=int, default=10)
+    parser.add_argument("--timeout", type=float, default=15.0)
+    args = parser.parse_args()
+    if args.requests < 1 or args.requests > 5000 or args.workers < 1 or args.workers > 100:
+        parser.error("requests must be 1..5000 and workers must be 1..100")
+
+    headers: dict[str, str] = {}
+    if args.mode == "reads":
+        password = os.getenv("WXAGENT_SCALE_WEB_PASSWORD")
+        if not password or not args.node_id or not args.account_id:
+            parser.error("reads mode requires WXAGENT_SCALE_WEB_PASSWORD, --node-id and --account-id")
+        status, _, payload = request(
+            args.base_url,
+            "/v1/auth/login",
+            "POST",
+            {"username": args.web_user, "password": password},
+            {"Content-Type": "application/json"},
+            args.timeout,
+        )
+        if status != 200:
+            print(json.dumps({"ok": False, "phase": "login", "status": status, "body": payload[:256].decode(errors="replace")}))
+            return 1
+        try:
+            login = json.loads(payload)
+            access_token = login["access_token"]
+        except (UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError) as error:
+            print(json.dumps({"ok": False, "phase": "login-response", "error": str(error)}))
+            return 1
+        headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"}
+
+    def one(index: int) -> tuple[int, float]:
+        if args.mode == "ready":
+            status, elapsed, _ = request(args.base_url, "/readyz", "GET", None, {}, args.timeout)
+        else:
+            body = {
+                "node_id": args.node_id,
+                "account_id": args.account_id,
+                "idempotency_key": f"scale-read-{uuid.uuid4().hex}-{index}",
+                "limit": 1,
+                "offset": 0,
+            }
+            status, elapsed, _ = request(args.base_url, "/v1/reads/sessions", "POST", body, headers, args.timeout)
+        return status, elapsed
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor:
+        results = list(executor.map(one, range(args.requests)))
+    elapsed = [item[1] for item in results]
+    success = sum(200 <= status < 300 for status, _ in results)
+    summary = {
+        "ok": success == args.requests,
+        "mode": args.mode,
+        "requests": args.requests,
+        "workers": args.workers,
+        "success": success,
+        "failed": args.requests - success,
+        "p50_ms": round(statistics.median(elapsed) * 1000, 2) if elapsed else 0,
+        "p95_ms": round(percentile(elapsed, 0.95) * 1000, 2),
+        "statuses": {str(status): sum(item[0] == status for item in results) for status, _ in results},
+    }
+    print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
+    return 0 if summary["ok"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/remote-control-smoke.sh

@@ -30,7 +30,7 @@ trap cleanup EXIT
 ) >"$log_file" 2>&1 &
 control_pid=$!
 
-for _ in $(seq 1 60); do
+for _ in $(seq 1 600); do
   if curl -fsS "http://127.0.0.1:${port}/healthz" >/dev/null 2>&1; then break; fi
   sleep 0.1
 done

scripts/revoke-client-certificate.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ "$#" -ne 2 ]; then
+  echo "usage: $0 <client-certificate.pem> <revoked-fingerprint-file>" >&2
+  exit 2
+fi
+certificate=$1
+list=$2
+[ -f "$certificate" ] || { echo "certificate does not exist: $certificate" >&2; exit 1; }
+
+fingerprint=$(openssl x509 -in "$certificate" -outform DER | sha256sum | awk '{print $1}')
+[ "${#fingerprint}" -eq 64 ] || { echo "could not calculate certificate fingerprint" >&2; exit 1; }
+mkdir -p "$(dirname "$list")"
+lock="$list.lock"
+exec 9>"$lock"
+flock -x 9
+if [ -f "$list" ] && grep -Fqx "$fingerprint" "$list"; then
+  printf '%s\n' "$fingerprint"
+  exit 0
+fi
+temporary="${list}.tmp.$$"
+trap 'rm -f "$temporary"' EXIT
+if [ -f "$list" ]; then
+  cat "$list" > "$temporary"
+fi
+printf '%s\n' "$fingerprint" >> "$temporary"
+chmod 600 "$temporary"
+mv -f "$temporary" "$list"
+printf '%s\n' "$fingerprint"