HH-446: restore auditable quality gates from current main (#102)
* HH-446: restore auditable quality gates * test(HH-446): restore help center smoke contract --------- Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
+166
-2
@@ -41,6 +41,9 @@ jobs:
|
||||
run: python -m pip install --require-hashes -r .github/requirements-browser-harness.txt
|
||||
- name: Test Prometheus alert rules
|
||||
run: docker run --rm --entrypoint promtool -v "$PWD/backend/configs:/configs:ro" prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 test rules /configs/prometheus_alerts_test.yml
|
||||
- name: Test quality gate failure contracts
|
||||
working-directory: backend
|
||||
run: python3 scripts/quality_gate_test.py
|
||||
- name: Test
|
||||
working-directory: backend
|
||||
env:
|
||||
@@ -55,7 +58,30 @@ jobs:
|
||||
browser-harness <<'PY'
|
||||
print(page_info())
|
||||
PY
|
||||
go test ./internal/... ./pkg/... ./cmd/...
|
||||
GOCHAT_RACE=false \
|
||||
GOCHAT_COLD_CACHE=true \
|
||||
GOCHAT_MAX_DURATION_SECONDS=480 \
|
||||
GOCHAT_QUALITY_DIR="$GITHUB_WORKSPACE/backend/quality/sqlite/cold-cache" \
|
||||
./scripts/ci_quality_gate.sh
|
||||
- name: Test critical race suites
|
||||
working-directory: backend
|
||||
env:
|
||||
GOCHAT_TEST_DB: sqlite
|
||||
run: |
|
||||
GOCHAT_RACE=true \
|
||||
GOCHAT_COLD_CACHE=true \
|
||||
GOCHAT_TEST_PACKAGES='./internal/ws ./internal/handler/ws ./internal/worker ./internal/channel/...' \
|
||||
GOCHAT_TEST_TIMEOUT=180s \
|
||||
GOCHAT_MAX_DURATION_SECONDS=180 \
|
||||
GOCHAT_QUALITY_DIR="$GITHUB_WORKSPACE/backend/quality/sqlite/race" \
|
||||
./scripts/ci_quality_gate.sh
|
||||
- name: Upload SQLite quality evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: backend-quality-sqlite
|
||||
path: backend/quality/sqlite
|
||||
if-no-files-found: error
|
||||
- name: Build
|
||||
working-directory: backend
|
||||
run: go build ./...
|
||||
@@ -105,6 +131,58 @@ jobs:
|
||||
GOCHAT_TEST_DB: postgres
|
||||
GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
|
||||
run: go test -count=1 -timeout 10m ./internal/... ./pkg/... ./cmd/...
|
||||
- name: Test controlled service race budget
|
||||
working-directory: backend
|
||||
env:
|
||||
GOCHAT_TEST_DB: postgres
|
||||
GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
|
||||
run: |
|
||||
set -euo pipefail
|
||||
evidence="$GITHUB_WORKSPACE/backend/quality/postgres/service-race"
|
||||
mkdir -p "$evidence"
|
||||
start=$SECONDS
|
||||
set +e
|
||||
timeout 180s go test -race -count=1 -json -timeout 170s \
|
||||
-run '^(TestCaptainPreferenceService_(Create|Create_DuplicateAccount|Get|Get_NotFound|Update|Delete)|TestEnsureCaptainAgentBotBindingConcurrentCallsStayUnique|TestTagCreate7_Cov35)$' \
|
||||
./internal/service | tee "$evidence/test.jsonl"
|
||||
test_status=${PIPESTATUS[0]}
|
||||
set -e
|
||||
elapsed=$((SECONDS - start))
|
||||
python3 - "$evidence/test.jsonl" "$evidence/summary.json" "$test_status" "$elapsed" <<'PY'
|
||||
import json, sys
|
||||
|
||||
required = {
|
||||
"TestCaptainPreferenceService_Create",
|
||||
"TestCaptainPreferenceService_Create_DuplicateAccount",
|
||||
"TestCaptainPreferenceService_Get",
|
||||
"TestCaptainPreferenceService_Get_NotFound",
|
||||
"TestCaptainPreferenceService_Update",
|
||||
"TestCaptainPreferenceService_Delete",
|
||||
"TestEnsureCaptainAgentBotBindingConcurrentCallsStayUnique",
|
||||
"TestTagCreate7_Cov35",
|
||||
}
|
||||
results = {}
|
||||
with open(sys.argv[1], encoding="utf-8") as events:
|
||||
for line in events:
|
||||
event = json.loads(line)
|
||||
if event.get("Test") in required and event.get("Action") in {"pass", "fail", "skip"}:
|
||||
results[event["Test"]] = event["Action"]
|
||||
status, elapsed = int(sys.argv[3]), int(sys.argv[4])
|
||||
failures = [name for name in sorted(required) if results.get(name) != "pass"]
|
||||
report = {
|
||||
"status": "passed" if status == 0 and elapsed <= 180 and not failures else "failed",
|
||||
"duration_seconds": elapsed,
|
||||
"budget_seconds": 180,
|
||||
"test_exit_code": status,
|
||||
"required_tests": {name: results.get(name, "missing") for name in sorted(required)},
|
||||
}
|
||||
with open(sys.argv[2], "w", encoding="utf-8") as output:
|
||||
json.dump(report, output, indent=2, sort_keys=True)
|
||||
output.write("\n")
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
if report["status"] != "passed":
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
- name: Test production upload migrations
|
||||
working-directory: backend
|
||||
env:
|
||||
@@ -164,6 +242,13 @@ jobs:
|
||||
if empty:
|
||||
raise SystemExit(f"required E2E suites ran no passing subtests: {', '.join(empty)}")
|
||||
PY
|
||||
- name: Upload PostgreSQL quality evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: backend-quality-postgres
|
||||
path: backend/quality/postgres
|
||||
if-no-files-found: error
|
||||
|
||||
frontend:
|
||||
name: Frontend Build
|
||||
@@ -184,8 +269,85 @@ jobs:
|
||||
- name: Build and verify artifacts
|
||||
run: pnpm --dir frontend test:build
|
||||
|
||||
frontend-smoke:
|
||||
name: Browser core smoke
|
||||
needs: [frontend]
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b
|
||||
env:
|
||||
POSTGRES_DB: gochat_test
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres -d gochat_test"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
redis:
|
||||
image: redis:7.4.2-alpine@sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 1.25.13
|
||||
cache-dependency-path: backend/go.sum
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 20.19.5
|
||||
- uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2
|
||||
id: chrome
|
||||
with:
|
||||
chrome-version: 152.0.7977.54
|
||||
- name: Test browser smoke failure contracts
|
||||
run: node backend/scripts/parity_frontend_browser_smoke.mjs --self-test
|
||||
- run: corepack enable && corepack prepare pnpm@10.2.0 --activate
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- name: Run browser smoke
|
||||
env:
|
||||
CI: "true"
|
||||
GOCHAT_ENV: development
|
||||
GOCHAT_DATABASE_DSN: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
|
||||
GOCHAT_REDIS_DSN: redis://localhost:6379
|
||||
GOCHAT_JWT_SECRET: gochat-smoke-jwt-secret-at-least-32-characters
|
||||
CHATWOOT_DIR: ${{ github.workspace }}/frontend
|
||||
GOCHAT_SMOKE_SEARCH_ENGINE: db
|
||||
GOCHAT_SMOKE_LOG_DIR: ${{ github.workspace }}/backend/.tmp/frontend-smoke
|
||||
GOCHAT_SMOKE_REPORT: ${{ github.workspace }}/docs/parity/frontend-smoke-report.md
|
||||
GOCHAT_SMOKE_CHROME: ${{ steps.chrome.outputs.chrome-path }}
|
||||
run: |
|
||||
mkdir -p "$GOCHAT_SMOKE_LOG_DIR"
|
||||
(cd backend && go run ./cmd/gochat serve >"$GOCHAT_SMOKE_LOG_DIR/gochat.log" 2>&1) &
|
||||
backend_pid=$!
|
||||
(cd frontend && VITE_API_HOST=http://127.0.0.1:3000 pnpm exec vite --host localhost --port 3036 >"$GOCHAT_SMOKE_LOG_DIR/vite.log" 2>&1) &
|
||||
frontend_pid=$!
|
||||
trap 'kill "$backend_pid" "$frontend_pid" 2>/dev/null || true' EXIT
|
||||
curl --fail --silent --show-error --retry 60 --retry-connrefused --retry-delay 1 \
|
||||
http://127.0.0.1:3000/health > /dev/null
|
||||
backend/scripts/parity_frontend_smoke.sh --browser-smoke
|
||||
- name: Upload browser smoke evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: frontend-browser-smoke
|
||||
path: |
|
||||
backend/.tmp/frontend-smoke
|
||||
docs/parity/frontend-smoke-report.md
|
||||
if-no-files-found: error
|
||||
|
||||
security:
|
||||
name: Security (SAST/SCA)
|
||||
needs: [frontend-smoke]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
@@ -221,6 +383,7 @@ jobs:
|
||||
production-smoke:
|
||||
name: Production image smoke
|
||||
if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')
|
||||
needs: [frontend-smoke]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GOCHAT_IMAGE_REF: gochat:production-smoke
|
||||
@@ -308,7 +471,8 @@ jobs:
|
||||
container_id="$("${compose[@]}" ps -q "$service")"
|
||||
test "$(docker inspect --format '{{.State.Health.Status}}' "$container_id")" = healthy
|
||||
done
|
||||
curl -fsS "http://127.0.0.1:$GOCHAT_PORT/health" | grep -q '"status":"healthy"'
|
||||
curl -fsS "http://127.0.0.1:$GOCHAT_PORT/health" > "$RUNNER_TEMP/health.json"
|
||||
python3 backend/scripts/validate_health_json.py "$RUNNER_TEMP/health.json"
|
||||
curl -fsS "http://127.0.0.1:$GOCHAT_PORT/app" | grep -q '/assets/'
|
||||
curl -fsS "http://127.0.0.1:$GOCHAT_PORT/runtime-config.js" | grep -q 'window.__GOCHAT_CONFIG__'
|
||||
curl -fsS -o "$RUNNER_TEMP/favicon-32x32.png" "http://127.0.0.1:$GOCHAT_PORT/favicon-32x32.png"
|
||||
|
||||
Executable
+173
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate Go test coverage, execution evidence, and elapsed time."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def coverage(path):
|
||||
files = defaultdict(lambda: [0, 0])
|
||||
total = [0, 0]
|
||||
for line in Path(path).read_text(encoding="utf-8").splitlines():
|
||||
if not line or line.startswith("mode:"):
|
||||
continue
|
||||
fields = line.split()
|
||||
if len(fields) != 3:
|
||||
raise ValueError(f"invalid coverage line: {line}")
|
||||
file_name = fields[0].split(":", 1)[0]
|
||||
statements, count = int(fields[1]), int(fields[2])
|
||||
total[0] += statements
|
||||
total[1] += statements if count else 0
|
||||
files[file_name][0] += statements
|
||||
files[file_name][1] += statements if count else 0
|
||||
if total[0] == 0:
|
||||
raise ValueError("coverage profile has no statements")
|
||||
return total, files
|
||||
|
||||
|
||||
def test_evidence(path):
|
||||
packages = defaultdict(lambda: {"pass": 0, "fail": 0, "skip": 0})
|
||||
totals = {"pass": 0, "fail": 0, "skip": 0}
|
||||
for line in Path(path).read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
action = event.get("Action")
|
||||
package = event.get("Package", "")
|
||||
if not event.get("Test") or action not in totals:
|
||||
continue
|
||||
packages[package][action] += 1
|
||||
totals[action] += 1
|
||||
return totals, packages
|
||||
|
||||
|
||||
def package_matches(name, spec):
|
||||
recursive = spec.endswith("/...")
|
||||
package = spec.removesuffix("/...")
|
||||
return name == package or recursive and name.startswith(package + "/")
|
||||
|
||||
|
||||
def run(argv=None):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--coverage", required=True)
|
||||
parser.add_argument("--test-json", required=True)
|
||||
parser.add_argument("--duration", required=True)
|
||||
parser.add_argument("--report", required=True)
|
||||
parser.add_argument("--min-coverage", type=float, default=65.0)
|
||||
parser.add_argument("--max-duration", type=float, required=True)
|
||||
parser.add_argument("--test-exit-code", type=int, default=0)
|
||||
parser.add_argument("--critical-package", action="append", default=[])
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
failures = []
|
||||
try:
|
||||
total, files = coverage(args.coverage)
|
||||
total_tests, packages = test_evidence(args.test_json)
|
||||
elapsed = float(Path(args.duration).read_text(encoding="utf-8").strip())
|
||||
except (OSError, ValueError) as error:
|
||||
failures.append(str(error))
|
||||
total, files = [0, 0], {}
|
||||
total_tests, packages, elapsed = {"pass": 0, "fail": 0, "skip": 0}, {}, None
|
||||
|
||||
percent = total[1] / total[0] * 100 if total[0] else 0.0
|
||||
if percent < args.min_coverage:
|
||||
failures.append(f"overall coverage {percent:.2f}% is below {args.min_coverage:.2f}%")
|
||||
if args.test_exit_code:
|
||||
failures.append(f"go test exited with {args.test_exit_code}")
|
||||
if total_tests["fail"]:
|
||||
failures.append(f"test evidence contains {total_tests['fail']} failed test events")
|
||||
if elapsed is not None and elapsed > args.max_duration:
|
||||
failures.append(f"test duration {elapsed:.2f}s exceeds {args.max_duration:.2f}s")
|
||||
if sum(total_tests.values()) == 0:
|
||||
failures.append("test evidence contains no test results")
|
||||
|
||||
critical, critical_coverage = [], []
|
||||
for spec in args.critical_package:
|
||||
matched = {
|
||||
name: values for name, values in packages.items() if package_matches(name, spec)
|
||||
}
|
||||
item = {
|
||||
"package": spec,
|
||||
"matched": sorted(matched),
|
||||
"passed": sum(values["pass"] for values in matched.values()),
|
||||
"failed": sum(values["fail"] for values in matched.values()),
|
||||
"skipped": sum(values["skip"] for values in matched.values()),
|
||||
}
|
||||
critical.append(item)
|
||||
if not matched:
|
||||
failures.append(f"critical suite {spec} produced no package evidence")
|
||||
elif item["passed"] == 0:
|
||||
failures.append(f"critical suite {spec} was entirely skipped")
|
||||
|
||||
matching = [
|
||||
values
|
||||
for name, values in files.items()
|
||||
if package_matches(name.rsplit("/", 1)[0], spec)
|
||||
]
|
||||
statements = sum(values[0] for values in matching)
|
||||
covered = sum(values[1] for values in matching)
|
||||
coverage_item = {
|
||||
"package": spec,
|
||||
"covered_statements": covered,
|
||||
"total_statements": statements,
|
||||
"percent": round(covered / statements * 100, 2) if statements else 0.0,
|
||||
}
|
||||
critical_coverage.append(coverage_item)
|
||||
if not statements:
|
||||
failures.append(f"critical coverage {spec} has no statements")
|
||||
elif coverage_item["percent"] < args.min_coverage:
|
||||
failures.append(
|
||||
f"critical coverage {spec} {coverage_item['percent']:.2f}% "
|
||||
f"is below {args.min_coverage:.2f}%"
|
||||
)
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"status": "failed" if failures else "passed",
|
||||
"thresholds": {
|
||||
"min_coverage_percent": args.min_coverage,
|
||||
"max_duration_seconds": args.max_duration,
|
||||
},
|
||||
"coverage": {
|
||||
"covered_statements": total[1],
|
||||
"total_statements": total[0],
|
||||
"percent": round(percent, 2),
|
||||
"critical": critical_coverage,
|
||||
"files": {
|
||||
name: {
|
||||
"covered_statements": values[1],
|
||||
"total_statements": values[0],
|
||||
"percent": round(values[1] / values[0] * 100, 2) if values[0] else 0.0,
|
||||
}
|
||||
for name, values in sorted(files.items())
|
||||
},
|
||||
},
|
||||
"tests": {**total_tests, "critical": critical},
|
||||
"duration_seconds": elapsed,
|
||||
"test_exit_code": args.test_exit_code,
|
||||
"failures": failures,
|
||||
}
|
||||
report_path = Path(args.report)
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": report["status"],
|
||||
"coverage_percent": report["coverage"]["percent"],
|
||||
"tests": report["tests"],
|
||||
"duration_seconds": elapsed,
|
||||
"failures": failures,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
QUALITY_DIR="${GOCHAT_QUALITY_DIR:-$ROOT/.tmp/quality}"
|
||||
mkdir -p "$QUALITY_DIR"
|
||||
|
||||
RACE="${GOCHAT_RACE:-true}"
|
||||
COLD_CACHE="${GOCHAT_COLD_CACHE:-true}"
|
||||
MIN_COVERAGE="${GOCHAT_MIN_COVERAGE:-65}"
|
||||
MAX_DURATION="${GOCHAT_MAX_DURATION_SECONDS:-300}"
|
||||
COVERAGE_FILE="${GOCHAT_COVERAGE_FILE:-$QUALITY_DIR/coverage.out}"
|
||||
TEST_JSON="${GOCHAT_TEST_JSON:-$QUALITY_DIR/test.jsonl}"
|
||||
DURATION_FILE="${GOCHAT_DURATION_FILE:-$QUALITY_DIR/duration.seconds}"
|
||||
REPORT_FILE="${GOCHAT_QUALITY_REPORT:-$QUALITY_DIR/quality-gate.json}"
|
||||
|
||||
packages=(./internal/... ./pkg/... ./cmd/...)
|
||||
if [[ -n "${GOCHAT_TEST_PACKAGES:-}" ]]; then
|
||||
read -r -a packages <<< "$GOCHAT_TEST_PACKAGES"
|
||||
fi
|
||||
|
||||
if [[ "$COLD_CACHE" == "true" ]]; then
|
||||
go clean -testcache
|
||||
fi
|
||||
|
||||
race_args=()
|
||||
if [[ "$RACE" == "true" ]]; then
|
||||
race_args=(-race)
|
||||
fi
|
||||
|
||||
start="$(date +%s.%N)"
|
||||
set +e
|
||||
go test -json -count=1 "${race_args[@]}" -covermode=atomic \
|
||||
-coverprofile="$COVERAGE_FILE" -timeout "${GOCHAT_TEST_TIMEOUT:-300s}" \
|
||||
"${packages[@]}" | tee "$TEST_JSON" >/dev/null
|
||||
test_status=${PIPESTATUS[0]}
|
||||
set -e
|
||||
end="$(date +%s.%N)"
|
||||
awk -v start="$start" -v end="$end" 'BEGIN { printf "%.3f\n", end-start }' > "$DURATION_FILE"
|
||||
|
||||
set +e
|
||||
python3 "$ROOT/scripts/ci_quality_gate.py" \
|
||||
--coverage "$COVERAGE_FILE" \
|
||||
--test-json "$TEST_JSON" \
|
||||
--duration "$DURATION_FILE" \
|
||||
--report "$REPORT_FILE" \
|
||||
--min-coverage "$MIN_COVERAGE" \
|
||||
--max-duration "$MAX_DURATION" \
|
||||
--test-exit-code "$test_status" \
|
||||
--critical-package github.com/gochat/gochat/internal/ws \
|
||||
--critical-package github.com/gochat/gochat/internal/handler/ws \
|
||||
--critical-package github.com/gochat/gochat/internal/worker \
|
||||
--critical-package github.com/gochat/gochat/internal/channel/...
|
||||
gate_status=$?
|
||||
set -e
|
||||
|
||||
[[ "$test_status" -eq 0 ]] || exit "$test_status"
|
||||
exit "$gate_status"
|
||||
@@ -1,6 +1,13 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdtempSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import {
|
||||
closeSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { createServer } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
@@ -20,6 +27,39 @@ const frontendBaseURL = `http://${shellHost}:${shellPort}`;
|
||||
const apiBaseURL = `http://${apiHost}:${apiPort}`;
|
||||
const enterpriseMode = process.argv.includes('--enterprise');
|
||||
|
||||
function isSuccessfulRequest(request, substring) {
|
||||
return request.url.includes(substring) && request.status >= 200 && request.status < 400;
|
||||
}
|
||||
|
||||
function isFailedAPIRequest(request) {
|
||||
const isBackendAPI = request.url.includes(apiBaseURL);
|
||||
const isShellProxiedAPI = request.url.startsWith(frontendBaseURL) && (
|
||||
request.url.includes('/api/') ||
|
||||
request.url.includes('/enterprise/') ||
|
||||
request.url.includes('/public/') ||
|
||||
request.url.includes('/auth/') ||
|
||||
request.url.includes('/rails/')
|
||||
);
|
||||
if (!isBackendAPI && !isShellProxiedAPI || request.type === 'Preflight') return false;
|
||||
const isNavigationAbort = request.type === 'Document' &&
|
||||
['net::ERR_ABORTED', 'net::ERR_FAILED'].includes(request.errorText);
|
||||
return request.status >= 400 || request.status === 0 && !isNavigationAbort;
|
||||
}
|
||||
|
||||
if (process.argv.includes('--self-test')) {
|
||||
const apiURL = `${apiBaseURL}/api/v1/profile`;
|
||||
const checks = [
|
||||
isSuccessfulRequest({ url: apiURL, status: 200 }, '/api/v1/profile'),
|
||||
!isSuccessfulRequest({ url: apiURL, status: 500 }, '/api/v1/profile'),
|
||||
isFailedAPIRequest({ url: apiURL, status: 0, type: 'Fetch', errorText: 'net::ERR_FAILED' }),
|
||||
!isFailedAPIRequest({ url: apiURL, status: 0, type: 'Document', errorText: 'net::ERR_ABORTED' }),
|
||||
!isFailedAPIRequest({ url: `${frontendBaseURL}/favicon.ico`, status: 404, type: 'Image' }),
|
||||
];
|
||||
if (checks.some(check => !check)) throw new Error('browser smoke failure contract self-test failed');
|
||||
console.log('browser smoke failure contract self-test: ok');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
mkdirSync(logDir, { recursive: true });
|
||||
|
||||
const seed = JSON.parse(readFileSync(path.join(logDir, 'seed.json'), 'utf8'));
|
||||
@@ -38,7 +78,7 @@ const report = {
|
||||
|
||||
function smokeHTML(entrypoint, route) {
|
||||
const config = {
|
||||
apiHost: apiBaseURL,
|
||||
apiHost: '',
|
||||
hostURL: frontendBaseURL,
|
||||
helpCenterURL: '',
|
||||
allowedLoginMethods: ['email'],
|
||||
@@ -78,7 +118,7 @@ function smokeHTML(entrypoint, route) {
|
||||
window.errorLoggingConfig = '';
|
||||
window.analyticsConfig = { token: '' };
|
||||
</script>
|
||||
<script type="module" src="/vite-dev/entrypoints/${entrypoint}.js"></script>
|
||||
<script type="module" src="${viteBaseURL}/app/javascript/entrypoints/${entrypoint}.js"></script>
|
||||
</head>
|
||||
<body class="text-slate-600"><div id="app"></div></body>
|
||||
</html>`;
|
||||
@@ -114,7 +154,7 @@ function widgetSmokeHTML(route) {
|
||||
window.browserConfig = { browser_name: 'chrome' };
|
||||
window.errorLoggingConfig = '';
|
||||
</script>
|
||||
<script type="module" src="/vite-dev/entrypoints/widget.js"></script>
|
||||
<script type="module" src="${viteBaseURL}/app/javascript/entrypoints/widget.js"></script>
|
||||
</head>
|
||||
<body class="text-slate-600"><div id="app"></div></body>
|
||||
</html>`;
|
||||
@@ -393,6 +433,7 @@ class CDPPage {
|
||||
report.requests.push({
|
||||
url: this.requestURLs.get(params.requestId) || params.requestId,
|
||||
status: 0,
|
||||
type: params.type,
|
||||
errorText: params.errorText,
|
||||
});
|
||||
});
|
||||
@@ -456,14 +497,14 @@ class CDPPage {
|
||||
);
|
||||
}
|
||||
|
||||
async waitForCapturedRequest(substring, label, timeout = 30000) {
|
||||
return this.waitForCapturedRequestAfter(substring, 0, label, timeout);
|
||||
async waitForSuccessfulRequest(substring, label, timeout = 30000) {
|
||||
return this.waitForSuccessfulRequestAfter(substring, 0, label, timeout);
|
||||
}
|
||||
|
||||
async waitForCapturedRequestAfter(substring, requestIndex, label, timeout = 30000) {
|
||||
async waitForSuccessfulRequestAfter(substring, requestIndex, label, timeout = 30000) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeout) {
|
||||
if (report.requests.slice(requestIndex).some(request => request.url.includes(substring))) {
|
||||
if (report.requests.slice(requestIndex).some(request => isSuccessfulRequest(request, substring))) {
|
||||
report.checks.push({ label, status: 'passed' });
|
||||
return;
|
||||
}
|
||||
@@ -473,22 +514,7 @@ class CDPPage {
|
||||
}
|
||||
|
||||
assertNoFailedAPIRequests(requestIndex = 0) {
|
||||
const failures = report.requests.slice(requestIndex).filter(request => {
|
||||
const isBackendAPI = request.url.includes(apiBaseURL);
|
||||
const isShellProxiedAPI = request.url.startsWith(frontendBaseURL) && (
|
||||
request.url.includes('/api/') ||
|
||||
request.url.includes('/enterprise/') ||
|
||||
request.url.includes('/public/') ||
|
||||
request.url.includes('/auth/') ||
|
||||
request.url.includes('/rails/')
|
||||
);
|
||||
if (!isBackendAPI && !isShellProxiedAPI) return false;
|
||||
if (request.type === 'Preflight') return false;
|
||||
if (request.status === 0 && ['net::ERR_ABORTED', 'net::ERR_FAILED'].includes(request.errorText)) {
|
||||
return false;
|
||||
}
|
||||
return request.status >= 400 || request.status === 0;
|
||||
});
|
||||
const failures = report.requests.slice(requestIndex).filter(isFailedAPIRequest);
|
||||
if (failures.length > 0) {
|
||||
throw new Error(`Frontend API requests failed: ${failures.map(request => `${request.status} ${request.url}`).join('; ')}`);
|
||||
}
|
||||
@@ -502,17 +528,33 @@ class CDPPage {
|
||||
}
|
||||
|
||||
async function launchChrome() {
|
||||
if (typeof WebSocket !== 'function') {
|
||||
throw new Error('Node.js WebSocket API is unavailable; use --experimental-websocket');
|
||||
}
|
||||
const userDataDir = mkdtempSync(path.join(tmpdir(), 'gochat-chrome-'));
|
||||
const chrome = spawn(chromePath, [
|
||||
const args = [
|
||||
'--headless=new',
|
||||
'--disable-gpu',
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
'--disable-dev-shm-usage',
|
||||
'--enable-logging=stderr',
|
||||
'--remote-debugging-port=0',
|
||||
`--user-data-dir=${userDataDir}`,
|
||||
'about:blank',
|
||||
], { stdio: ['ignore', 'ignore', 'pipe'] });
|
||||
];
|
||||
if (process.env.CI) args.push('--no-sandbox');
|
||||
args.push('about:blank');
|
||||
const chromeLogPath = path.join(logDir, 'chrome.log');
|
||||
const chromeLog = openSync(chromeLogPath, 'w');
|
||||
const chrome = spawn(chromePath, args, { stdio: ['ignore', 'ignore', chromeLog] });
|
||||
closeSync(chromeLog);
|
||||
let chromeFailure = '';
|
||||
chrome.once('error', error => {
|
||||
chromeFailure = `failed to launch Chrome: ${error.message}`;
|
||||
});
|
||||
chrome.once('exit', (code, signal) => {
|
||||
chromeFailure = `Chrome exited before DevTools was ready (code=${code}, signal=${signal})`;
|
||||
});
|
||||
const portFile = path.join(userDataDir, 'DevToolsActivePort');
|
||||
for (let i = 0; i < 80; i += 1) {
|
||||
try {
|
||||
@@ -527,11 +569,12 @@ async function launchChrome() {
|
||||
await page.init();
|
||||
return page;
|
||||
} catch {
|
||||
if (chromeFailure) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 250));
|
||||
}
|
||||
}
|
||||
chrome.kill('SIGTERM');
|
||||
throw new Error('Chrome DevTools did not become ready');
|
||||
throw new Error(`${chromeFailure || 'Chrome DevTools did not become ready'}; see ${chromeLogPath}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
@@ -547,17 +590,17 @@ async function main() {
|
||||
|
||||
await page.navigate(smokePages.dashboard);
|
||||
await page.waitForAppMounted('dashboard app mounted');
|
||||
await page.waitForCapturedRequest('/auth/validate_token', 'dashboard validates auth token');
|
||||
await page.waitForCapturedRequest(`/api/v1/accounts/${seed.account_id}/conversations`, 'dashboard requests conversations');
|
||||
await page.waitForSuccessfulRequest('/auth/validate_token', 'dashboard validates auth token');
|
||||
await page.waitForSuccessfulRequest(`/api/v1/accounts/${seed.account_id}/conversations`, 'dashboard requests conversations');
|
||||
page.assertNoFailedAPIRequests();
|
||||
|
||||
const widgetRequestIndex = report.requests.length;
|
||||
await page.navigate(smokePages.widget);
|
||||
await page.waitForAppMounted('widget app mounted');
|
||||
await page.waitForCapturedRequest('/api/v1/widget/messages', 'widget requests messages');
|
||||
await page.waitForCapturedRequest('/api/v1/widget/inbox_members', 'widget requests inbox members');
|
||||
await page.waitForSuccessfulRequest('/api/v1/widget/messages', 'widget requests messages');
|
||||
await page.waitForSuccessfulRequest('/api/v1/widget/inbox_members', 'widget requests inbox members');
|
||||
await page.eval(`fetch('/api/v1/widget/campaigns?website_token=${encodeURIComponent(widgetConfig.website_channel_config?.website_token || 'gochat-smoke-widget-token')}').then(response => response.ok)`);
|
||||
await page.waitForCapturedRequest('/api/v1/widget/campaigns', 'widget campaigns endpoint works');
|
||||
await page.waitForSuccessfulRequest('/api/v1/widget/campaigns', 'widget campaigns endpoint works');
|
||||
page.assertNoFailedAPIRequests(widgetRequestIndex);
|
||||
|
||||
if (enterpriseMode) {
|
||||
@@ -566,7 +609,7 @@ async function main() {
|
||||
await page.navigate(enterprisePage.url);
|
||||
await page.waitForAppMounted(`${enterprisePage.label} app mounted`);
|
||||
for (const request of enterprisePage.requests) {
|
||||
await page.waitForCapturedRequestAfter(request, requestIndex, `${enterprisePage.label} requests ${request}`);
|
||||
await page.waitForSuccessfulRequestAfter(request, requestIndex, `${enterprisePage.label} requests ${request}`);
|
||||
}
|
||||
page.assertNoFailedAPIRequests(requestIndex);
|
||||
}
|
||||
@@ -586,7 +629,7 @@ async function main() {
|
||||
})(),
|
||||
body: JSON.stringify({ message: 'B12 enterprise browser copilot smoke', assistant_id: ${Number(seed.captain_assistant_id)}, conversation_id: ${Number(seed.conversation_id)} })
|
||||
}).then(response => response.ok)`);
|
||||
await page.waitForCapturedRequestAfter('/captain/copilot_threads', copilotRequestIndex, 'browser context requests Copilot threads');
|
||||
await page.waitForSuccessfulRequestAfter('/captain/copilot_threads', copilotRequestIndex, 'browser context requests Copilot threads');
|
||||
page.assertNoFailedAPIRequests(copilotRequestIndex);
|
||||
}
|
||||
report.finished_at = new Date().toISOString();
|
||||
|
||||
@@ -478,7 +478,7 @@ SEED
|
||||
|
||||
run_browser_smoke() {
|
||||
run_api_smoke
|
||||
wait_url "http://$FRONTEND_HOST:$FRONTEND_PORT/vite-dev/entrypoints/dashboard.js" "Chatwoot Vite"
|
||||
wait_url "http://$FRONTEND_HOST:$FRONTEND_PORT/app/javascript/entrypoints/dashboard.js" "Chatwoot Vite"
|
||||
GOCHAT_ROOT="$ROOT" \
|
||||
CHATWOOT_DIR="$CHATWOOT_DIR" \
|
||||
GOCHAT_SMOKE_LOG_DIR="$LOG_DIR" \
|
||||
@@ -486,7 +486,7 @@ run_browser_smoke() {
|
||||
GOCHAT_SMOKE_API_PORT="$API_PORT" \
|
||||
GOCHAT_SMOKE_FRONTEND_HOST="$FRONTEND_HOST" \
|
||||
GOCHAT_SMOKE_FRONTEND_PORT="$FRONTEND_PORT" \
|
||||
node "$ROOT/scripts/parity_frontend_browser_smoke.mjs"
|
||||
node --experimental-websocket "$ROOT/scripts/parity_frontend_browser_smoke.mjs"
|
||||
BOOT_BACKEND_RESULT="Passed /health during browser smoke" \
|
||||
BOOT_FRONTEND_RESULT="Passed Vite readiness during browser smoke" \
|
||||
AUTH_PROFILE_RESULT="Passed browser login plus API smoke" \
|
||||
@@ -793,7 +793,7 @@ wait_url "http://$API_HOST:$API_PORT/health" "GoChat"
|
||||
|
||||
echo "starting reused Chatwoot frontend..."
|
||||
(cd "$CHATWOOT_DIR" && "${frontend_cmd[@]}") >"$LOG_DIR/chatwoot-vite.log" 2>&1 &
|
||||
wait_url "http://$FRONTEND_HOST:$FRONTEND_PORT/vite-dev/entrypoints/dashboard.js" "Chatwoot Vite"
|
||||
wait_url "http://$FRONTEND_HOST:$FRONTEND_PORT/app/javascript/entrypoints/dashboard.js" "Chatwoot Vite"
|
||||
|
||||
BOOT_BACKEND_RESULT="Passed boot readiness" \
|
||||
BOOT_FRONTEND_RESULT="Passed boot readiness" \
|
||||
|
||||
Executable
+127
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Failure-injection checks for CI quality gates."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
GATE = ROOT / "ci_quality_gate.py"
|
||||
HEALTH = ROOT / "validate_health_json.py"
|
||||
|
||||
|
||||
def run(command):
|
||||
return subprocess.run(command, capture_output=True, text=True)
|
||||
|
||||
|
||||
def write_fixtures(
|
||||
directory,
|
||||
coverage_lines=(
|
||||
"github.com/gochat/gochat/internal/ws/file.go:1.1,1.2 2 1\n"
|
||||
"github.com/gochat/gochat/internal/ws/file.go:2.1,2.2 2 1\n"
|
||||
),
|
||||
test_lines=None,
|
||||
duration="1",
|
||||
):
|
||||
(directory / "coverage.out").write_text("mode: atomic\n" + coverage_lines, encoding="utf-8")
|
||||
events = test_lines or [
|
||||
{"Action": "pass", "Package": "github.com/gochat/gochat/internal/ws", "Test": "TestRelay"}
|
||||
]
|
||||
(directory / "test.jsonl").write_text(
|
||||
"\n".join(json.dumps(event) for event in events) + "\n", encoding="utf-8"
|
||||
)
|
||||
(directory / "duration").write_text(duration, encoding="utf-8")
|
||||
|
||||
|
||||
def gate(directory, budget=5, exit_code=0):
|
||||
return run(
|
||||
[
|
||||
sys.executable,
|
||||
str(GATE),
|
||||
"--coverage",
|
||||
str(directory / "coverage.out"),
|
||||
"--test-json",
|
||||
str(directory / "test.jsonl"),
|
||||
"--duration",
|
||||
str(directory / "duration"),
|
||||
"--report",
|
||||
str(directory / "report.json"),
|
||||
"--min-coverage",
|
||||
"65",
|
||||
"--max-duration",
|
||||
str(budget),
|
||||
"--test-exit-code",
|
||||
str(exit_code),
|
||||
"--critical-package",
|
||||
"github.com/gochat/gochat/internal/ws",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
directory = Path(temp)
|
||||
write_fixtures(directory)
|
||||
assert gate(directory).returncode == 0
|
||||
|
||||
write_fixtures(
|
||||
directory,
|
||||
"github.com/gochat/gochat/internal/ws/file.go:1.1,1.2 2 1\n"
|
||||
"github.com/gochat/gochat/internal/ws/file.go:2.1,2.2 2 0\n",
|
||||
)
|
||||
assert gate(directory).returncode != 0, "coverage regression must fail"
|
||||
|
||||
write_fixtures(
|
||||
directory,
|
||||
test_lines=[
|
||||
{"Action": "skip", "Package": "github.com/gochat/gochat/internal/ws", "Test": "TestRelay"}
|
||||
],
|
||||
)
|
||||
assert gate(directory).returncode != 0, "all-skipped critical suite must fail"
|
||||
|
||||
write_fixtures(
|
||||
directory,
|
||||
"github.com/gochat/gochat/internal/wsevent/file.go:1.1,1.2 2 1\n",
|
||||
[{"Action": "pass", "Package": "github.com/gochat/gochat/internal/wsevent", "Test": "TestBridge"}],
|
||||
)
|
||||
assert gate(directory).returncode != 0, "sibling package must not satisfy the ws gate"
|
||||
|
||||
write_fixtures(directory, duration="6")
|
||||
assert gate(directory).returncode != 0, "duration budget must fail"
|
||||
|
||||
write_fixtures(
|
||||
directory,
|
||||
test_lines=[
|
||||
{"Action": "fail", "Package": "github.com/gochat/gochat/internal/ws", "Test": "TestRelay"}
|
||||
],
|
||||
)
|
||||
assert gate(directory).returncode != 0, "failed test evidence must fail"
|
||||
|
||||
write_fixtures(directory)
|
||||
assert gate(directory, exit_code=2).returncode != 0, "non-zero go test status must fail"
|
||||
|
||||
health = directory / "health.json"
|
||||
health.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "healthy",
|
||||
"timestamp": "2026-08-22T00:00:00Z",
|
||||
"uptime": "1s",
|
||||
"checks": {"database": "healthy"},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert run([sys.executable, str(HEALTH), str(health)]).returncode == 0
|
||||
health.write_text("{broken", encoding="utf-8")
|
||||
assert run([sys.executable, str(HEALTH), str(health)]).returncode != 0, "invalid health JSON must fail"
|
||||
health.write_text(json.dumps({"status": "unhealthy"}), encoding="utf-8")
|
||||
assert run([sys.executable, str(HEALTH), str(health)]).returncode != 0, "unhealthy contract must fail"
|
||||
|
||||
print("quality gate failure-injection checks: ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the stable JSON contract returned by GET /health."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def validate(value):
|
||||
if not isinstance(value, dict):
|
||||
return "health response must be a JSON object"
|
||||
missing = sorted({"status", "timestamp", "uptime", "checks"} - value.keys())
|
||||
if missing:
|
||||
return f"health response missing fields: {', '.join(missing)}"
|
||||
if value["status"] != "healthy":
|
||||
return f"unexpected health status: {value['status']}"
|
||||
try:
|
||||
datetime.fromisoformat(value["timestamp"].replace("Z", "+00:00"))
|
||||
except (AttributeError, ValueError):
|
||||
return "timestamp must be RFC3339"
|
||||
if not isinstance(value["uptime"], str) or not value["uptime"]:
|
||||
return "uptime must be a non-empty string"
|
||||
if not isinstance(value["checks"], dict) or value["checks"].get("database") != "healthy":
|
||||
return "database health check must be healthy"
|
||||
return None
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("path", nargs="?", help="JSON file; stdin when omitted")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
raw = Path(args.path).read_text(encoding="utf-8") if args.path else sys.stdin.read()
|
||||
error = validate(json.loads(raw))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
error = f"invalid health JSON: {exc}"
|
||||
if error:
|
||||
print(error, file=sys.stderr)
|
||||
return 1
|
||||
print("health JSON contract: ok")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user