test: capture integrity in live acceptance harness

This commit is contained in:
2026-09-22 11:38:03 +08:00
parent db95a20281
commit 9850a64abf
2 changed files with 22 additions and 12 deletions
+15 -6
View File
@@ -76,6 +76,7 @@ start_server() {
}
stop_server() { [[ -n "$server_pid" ]] && kill "$server_pid" 2>/dev/null || true; server_pid=""; }
sync_json() { curl -fsS -H "Authorization: Bearer $NODE_TOKEN" "http://127.0.0.1:8090/v1/nodes/local-node/data/accounts/$ACCOUNT_ID/sync-status"; }
sync_summary() { sync_json | jq -c '{state,confirmed_sequence,last_success_at}'; }
shard_json() {
python3 - "$live/control-plane-data.json.accounts" <<'PY'
import json, pathlib, sqlite3, sys
@@ -83,15 +84,15 @@ base=pathlib.Path(sys.argv[1])
p=next(base.glob('accounts/*/data.sqlite'))
c=sqlite3.connect(p)
rows=c.execute('select chat_id,title,source,directory_state from conversations').fetchall()
print(json.dumps({'messages':c.execute('select count(*) from messages').fetchone()[0],'conversations':len(rows),'batches':c.execute('select count(*) from ingest_batches').fetchone()[0],'coverage':c.execute('select coverage_state from sync_state').fetchone()[0],'sources':sorted({r[2] for r in rows}),'title_equals_chat_id':sum(r[0]==r[1] for r in rows)}))
print(json.dumps({'messages':c.execute('select count(*) from messages').fetchone()[0],'conversations':len(rows),'batches':c.execute('select count(*) from ingest_batches').fetchone()[0],'coverage':c.execute('select coverage_state from sync_state').fetchone()[0],'integrity':c.execute('pragma integrity_check').fetchone()[0],'sources':sorted({r[2] for r in rows}),'title_equals_chat_id':sum(r[0]==r[1] for r in rows)}))
c.close()
PY
}
run_ps "\$dir=\"$WINDOWS_DIR\"; Get-Process -Name WxAgent.Tray -ErrorAction SilentlyContinue | Stop-Process -Force; Get-ScheduledTask -TaskName \"WxAgent-P4-Audit-*\" -ErrorAction SilentlyContinue | Unregister-ScheduledTask -Confirm:\$false; Remove-Item \"\$dir/data/remote-data-sync-state.json\",\"\$dir/data/remote-data-queue.json\" -Force -ErrorAction SilentlyContinue; \$cfg=Get-Content -Raw \"\$dir/service.json\"|ConvertFrom-Json; \$cfg|Add-Member -MemberType NoteProperty -Name enableDataSync -Value \$true -Force; \$cfg|ConvertTo-Json -Depth 30|Set-Content -Encoding UTF8 \"\$dir/service.json\"" >/dev/null
scp -q "$TRAY_BINARY" "$WINDOWS_HOST:$WINDOWS_DIR/WxAgent.Tray.exe"
remote_hash=$(json_from_ps "[ordered]@{sha256=(Get-FileHash \"$WINDOWS_DIR/WxAgent.Tray.exe\" -Algorithm SHA256).Hash}|ConvertTo-Json -Compress")
echo "{\"event\":\"deployment\",\"source_commit\":\"$(git rev-parse HEAD)\",\"tray_sha256\":$(jq -c .sha256 <<<\"$remote_hash\")}"
remote_hash=$(run_ps "(Get-FileHash \"$WINDOWS_DIR/WxAgent.Tray.exe\" -Algorithm SHA256).Hash" 2>/dev/null | tr -d '\r' | awk '/^[0-9A-Fa-f]{64}$/ {line=$0} END {print line}')
echo "{\"event\":\"deployment\",\"source_commit\":\"$(git rev-parse HEAD)\",\"tray_sha256\":\"$remote_hash\"}"
# Collect while the control plane is deliberately absent.
offline_process=$(start_tray)
@@ -106,13 +107,13 @@ for _ in $(seq 1 30); do
status=$(jq -r '.state // .status // .node_status // "missing"' <(sync_json 2>/dev/null || echo '{}'))
[[ "$(jq -r .queue_pending <<<"$queue")" == 0 && "$status" == "complete" ]] && break
done
echo "{\"event\":\"replay\",\"queue\":$queue,\"sync\":$(sync_json),\"shard\":$(shard_json)}"
echo "{\"event\":\"replay\",\"queue\":$queue,\"sync\":$(sync_summary),\"shard\":$(shard_json)}"
before=$(sync_json)
before=$(sync_summary)
stop_ps='Get-Process -Name WxAgent.Tray -ErrorAction SilentlyContinue | Stop-Process -Force; Start-Sleep -Seconds 15'
run_ps "$stop_ps" >/dev/null 2>&1
restart_process=$(start_tray)
after=$(sync_json)
after=$(sync_summary)
echo "{\"event\":\"agent-restart\",\"process\":$restart_process,\"before\":$before,\"after\":$after}"
run_ps "$stop_ps" >/dev/null 2>&1
@@ -128,3 +129,11 @@ web_token=$(curl -fsS -X POST http://127.0.0.1:8090/v1/auth/login -H 'Content-Ty
conv=$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $web_token" "http://127.0.0.1:8090/v1/data/accounts/$ACCOUNT_ID/conversations")
msgs=$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $web_token" "http://127.0.0.1:8090/v1/data/accounts/$ACCOUNT_ID/messages?chat_id=filehelper&limit=10")
echo "{\"event\":\"authorization-restore\",\"process\":$restore_process,\"conversations\":$conv,\"messages\":$msgs}"
before_shard=$(shard_json)
old_server_pid=$server_pid
stop_server
start_server
for _ in $(seq 1 20); do sleep 2; state=$(sync_summary 2>/dev/null | jq -r .state || true); [[ "$state" == "complete" ]] && break; done
after_shard=$(shard_json)
echo "{\"event\":\"control-plane-restart\",\"old_pid\":$old_server_pid,\"new_pid\":$server_pid,\"before\":$before_shard,\"after\":$after_shard}"
@@ -5,11 +5,12 @@ file=${1:-"$(dirname "$0")/p4-live-operations.jsonl"}
jq -s -e '
map({(.event): .}) | add as $e |
($e["offline-queue"].queue_pending > 0 and $e["offline-queue"].queue_bytes > 0 and $e["offline-queue"].session == 1) and
($e["replay-complete"].queue_pending == 0 and $e["replay-complete"].node_status == "Online" and $e["replay-complete"].coverage == "complete") and
($e["agent-restart"].process.session == 1 and $e["agent-restart"].before.sequence == $e["agent-restart"].after.sequence and $e["agent-restart"].before.messages == $e["agent-restart"].after.messages and $e["agent-restart"].before.batches == $e["agent-restart"].after.batches and $e["agent-restart"].after.node_status == "Online") and
($e["authorization-revoke"].web_revoke_status == 200 and $e["authorization-revoke"].conversations_status == 403 and $e["authorization-revoke"].messages_status == 403 and $e["authorization-revoke"].sync_status == 200) and
($e["authorization-restore"].process.session == 1 and $e["authorization-restore"].conversations_status == 200 and $e["authorization-restore"].messages_status == 200) and
($e["control-plane-before-sigkill"].integrity == "ok" and $e["control-plane-after-restart"].integrity == "ok" and $e["control-plane-before-sigkill"].messages == $e["control-plane-after-restart"].messages and $e["control-plane-before-sigkill"].batches == $e["control-plane-after-restart"].batches)
($e["deployment"].source_commit != null and ($e["deployment"].tray_sha256 | length) == 64) and
($e["offline-queue"].queue.queue_pending > 0 and $e["offline-queue"].queue.queue_bytes > 0 and $e["offline-queue"].process.session == 1) and
($e["replay"].queue.queue_pending == 0 and $e["replay"].sync.state == "complete" and $e["replay"].shard.coverage == "complete" and $e["replay"].shard.integrity == "ok") and
($e["agent-restart"].process.session == 1 and $e["agent-restart"].before.confirmed_sequence == $e["agent-restart"].after.confirmed_sequence and $e["agent-restart"].after.state == "complete") and
($e["authorization-revoke"].revoke == 200 and $e["authorization-revoke"].conversations == 403 and $e["authorization-revoke"].messages == 403 and $e["authorization-revoke"].sync_status == 200) and
($e["authorization-restore"].process.session == 1 and $e["authorization-restore"].conversations == 200 and $e["authorization-restore"].messages == 200) and
($e["control-plane-restart"].before.integrity == "ok" and $e["control-plane-restart"].after.integrity == "ok" and $e["control-plane-restart"].before.messages == $e["control-plane-restart"].after.messages and $e["control-plane-restart"].before.batches == $e["control-plane-restart"].after.batches)
' "$file" >/dev/null
printf 'P4 live operation invariants: PASS\n'