From 46e87840ee9e450c52cecd1f0ef3527a5535c5e5 Mon Sep 17 00:00:00 2001 From: Jesse Vincent Date: Sat, 4 Jul 2026 20:32:55 -0700 Subject: [PATCH] feat(skills): add spec-vs-cards checker with test harness --- .../scripts/check-cards-against-spec | 136 +++++++++++++++ .../test-check-cards-against-spec.sh | 163 ++++++++++++++++++ 2 files changed, 299 insertions(+) create mode 100755 skills/agentic-end-to-end-testing/scripts/check-cards-against-spec create mode 100755 tests/agentic-e2e-checker/test-check-cards-against-spec.sh diff --git a/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec b/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec new file mode 100755 index 00000000..c0755655 --- /dev/null +++ b/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# check-cards-against-spec — verify scenario cards carry their spec table's +# falsification lines verbatim. See authoring-cards-from-a-spec.md. +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: check-cards-against-spec + +Verifies the spec's "E2E scenario cards" table against the cards directory: + 1. table parses (>=1 row; non-empty Card and Falsification cells) + 2. every row has /.md + 3. every card contains its Falsification line verbatim + (whitespace-normalized, fixed-string, case-sensitive) + 4. every card has **What this covers** (bold inline) and ## headings + Pre-state, Steps, Expected, Cleanup (Sharp edges not required) + 5. extra cards in are reported as warnings, not failures + +Exit: 0 all pass; 1 check failed; 2 no "E2E scenario cards" table; 64 usage. +EOF +} + +[ "${1:-}" = "--help" ] && { usage; exit 0; } +[ $# -eq 2 ] || { usage >&2; exit 64; } +SPEC="$1"; CARDS="$2" +[ -f "$SPEC" ] || { echo "error: spec not found: $SPEC" >&2; exit 64; } +[ -d "$CARDS" ] || { echo "error: cards dir not found: $CARDS" >&2; exit 64; } + +FAILURES=0 +fail() { echo "FAIL: $1"; FAILURES=$((FAILURES + 1)); } +warn() { echo "warn: $1"; } + +# Collapse every whitespace run to one space; trim ends. (Normative per the +# design spec: markdown re-wrapping must not defeat the verbatim check.) +normalize() { tr -s '[:space:]' ' ' | sed -e 's/^ //' -e 's/ $//'; } + +# --- extract the first table under the (case-insensitive) heading ---------- +TABLE="$(awk ' + /^#{1,6}[[:space:]]/ { + h = $0; sub(/^#+[[:space:]]*/, "", h); sub(/[[:space:]]+$/, "", h) + if (tolower(h) == "e2e scenario cards") { insec = 1; next } + if (insec) exit + } + insec && /^[[:space:]]*\|/ { intable = 1; print; next } + insec && intable { exit } +' "$SPEC")" + +if [ -z "$TABLE" ]; then + echo "no scenario table: $SPEC has no \"E2E scenario cards\" heading with a table under it" >&2 + exit 2 +fi + +# --- parse: protect escaped pipes, split rows into cells ------------------- +US=$'\x1f' +CARD_COL=-1; FALS_COL=-1; ROWS=0 +declare -a ROW_CARD ROW_FALS + +lineno=0 +while IFS= read -r line; do + lineno=$((lineno + 1)) + esc="${line//\\|/$US}" + IFS='|' read -r -a cells <<< "$esc" + # drop leading/trailing empty fields produced by the outer pipes + trimmed=() + for c in "${cells[@]}"; do + c="${c//$US/|}" + c="$(printf '%s' "$c" | normalize)" + trimmed+=("$c") + done + # cells[0] is empty (before first |); last may be empty too + if [ "$lineno" -eq 1 ]; then + for i in "${!trimmed[@]}"; do + low="$(printf '%s' "${trimmed[$i]}" | tr '[:upper:]' '[:lower:]')" + [ "$low" = "card" ] && CARD_COL=$i + [ "$low" = "falsification" ] && FALS_COL=$i + done + continue + fi + # separator row: cells of dashes/colons only + joined="$(printf '%s' "${trimmed[*]}" | tr -d ' :-')" + [ -z "$joined" ] && continue + if [ "$CARD_COL" -lt 0 ] || [ "$FALS_COL" -lt 0 ]; then + fail "table header must name Card and Falsification columns" + break + fi + card="${trimmed[$CARD_COL]:-}" + falsif="${trimmed[$FALS_COL]:-}" + card="${card//\`/}" # tolerate `card-name` backticks in the cell + if [ -z "$card" ] || [ -z "$falsif" ]; then + fail "row $lineno: empty Card or Falsification cell" + continue + fi + ROW_CARD[$ROWS]="$card"; ROW_FALS[$ROWS]="$falsif"; ROWS=$((ROWS + 1)) +done <<< "$TABLE" + +[ "$ROWS" -ge 1 ] || fail "scenario table has no data rows" + +# --- checks 2-4 per row ----------------------------------------------------- +i=0 +while [ "$i" -lt "$ROWS" ]; do + card="${ROW_CARD[$i]}"; falsif="${ROW_FALS[$i]}" + f="$CARDS/$card.md" + if [ ! -f "$f" ]; then + fail "missing card file: $f" + i=$((i + 1)); continue + fi + hay="$(normalize < "$f")" + case "$hay" in + *"$falsif"*) : ;; + *) fail "$f: falsification line not present verbatim. + expected (normalized): $falsif" ;; + esac + grep -q '\*\*What this covers\*\*' "$f" || fail "$f: missing **What this covers**" + for sec in Pre-state Steps Expected Cleanup; do + grep -Eiq "^#{2,}[[:space:]]*${sec}" "$f" || fail "$f: missing ## ${sec} section" + done + i=$((i + 1)) +done + +# --- check 5: extra cards are warnings -------------------------------------- +for f in "$CARDS"/*.md; do + [ -e "$f" ] || continue + base="$(basename "$f" .md)" + known=0; i=0 + while [ "$i" -lt "$ROWS" ]; do + [ "${ROW_CARD[$i]}" = "$base" ] && known=1 + i=$((i + 1)) + done + [ "$known" -eq 1 ] || warn "extra card not in spec table: $base" +done + +if [ "$FAILURES" -gt 0 ]; then + echo "$FAILURES check(s) failed" + exit 1 +fi +echo "all checks passed ($ROWS card(s))" diff --git a/tests/agentic-e2e-checker/test-check-cards-against-spec.sh b/tests/agentic-e2e-checker/test-check-cards-against-spec.sh new file mode 100755 index 00000000..5890c84c --- /dev/null +++ b/tests/agentic-e2e-checker/test-check-cards-against-spec.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +CHECKER="$REPO_ROOT/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec" + +FAILURES=0 +TEST_ROOT="$(mktemp -d)" +cleanup() { rm -rf "$TEST_ROOT"; } +trap cleanup EXIT + +pass() { echo " [PASS] $1"; } +fail() { echo " [FAIL] $1"; FAILURES=$((FAILURES + 1)); } + +assert_exit() { # expected_code description -- command... + local expected="$1" desc="$2"; shift 2 + local code=0 + "$@" >"$TEST_ROOT/out.txt" 2>&1 || code=$? + if [ "$code" -eq "$expected" ]; then pass "$desc"; else + fail "$desc (expected exit $expected, got $code)"; sed 's/^/ /' "$TEST_ROOT/out.txt"; fi +} + +assert_out_contains() { # needle description + if grep -Fq -- "$1" "$TEST_ROOT/out.txt"; then pass "$2"; else + fail "$2 (output missing: $1)"; sed 's/^/ /' "$TEST_ROOT/out.txt"; fi +} + +# ---- fixture builders ---------------------------------------------------- + +make_spec() { # dir (spec with 2-row table; row 2 has \| and regex chars) + mkdir -p "$1" + cat > "$1/spec.md" <<'EOF' +# Widget Design + +## Requirements + +Widgets render a table with a TOTAL row. + +## E2E scenario cards + +| Card | Covers | Falsification | +| --- | --- | --- | +| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. | +| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. | +EOF +} + +good_card_1() { + cat <<'EOF' +# widget-show-table: table renders with TOTAL + +**What this covers**: the rendered table. + +## Pre-state +A built widget binary. + +## Steps +1. Run `widget show`. + +## Expected +If stdout's last line is not `TOTAL` followed by the +two-decimal sum (20.85 for the seed +fixture), or the TOTAL row is absent entirely, the scenario FAILS. + +## Cleanup +Nothing to clean. +EOF +} + +good_card_2() { + cat <<'EOF' +# widget-status-flags: status output + +**What this covers**: status flags. + +## Pre-state +A built widget binary. + +## Steps +1. Run `widget status`. + +## Expected +If `widget status` does not print exactly `OK | DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. + +## Cleanup +Nothing to clean. +EOF +} + +make_cards() { # dir + mkdir -p "$1" + good_card_1 > "$1/widget-show-table.md" + good_card_2 > "$1/widget-status-flags.md" +} + +# ---- tests ---------------------------------------------------------------- + +echo "happy path" +make_spec "$TEST_ROOT/t1"; make_cards "$TEST_ROOT/t1/cards" +assert_exit 0 "2 rows, 2 conforming cards -> exit 0" \ + "$CHECKER" "$TEST_ROOT/t1/spec.md" "$TEST_ROOT/t1/cards" + +echo "re-wrapped falsification line still matches (whitespace normalization)" +# good_card_1 already wraps the line across three lines; covered above. Prove +# the inverse too: collapse the card line to one line, still passes. +make_spec "$TEST_ROOT/t2"; make_cards "$TEST_ROOT/t2/cards" +perl -0pi -e 's/\n(two-decimal)/ $1/; s/\n(fixture\))/ $1/' "$TEST_ROOT/t2/cards/widget-show-table.md" 2>/dev/null || \ + sed -i '' -e ':a' -e 'N;$!ba' -e 's/the\ntwo-decimal/the two-decimal/' "$TEST_ROOT/t2/cards/widget-show-table.md" +assert_exit 0 "single-line variant -> exit 0" \ + "$CHECKER" "$TEST_ROOT/t2/spec.md" "$TEST_ROOT/t2/cards" + +echo "escaped pipe in table cell matches literal pipe in card" +# covered by widget-status-flags in the happy path; also prove failure when +# the card drops the pipe phrase entirely: +make_spec "$TEST_ROOT/t3"; make_cards "$TEST_ROOT/t3/cards" +sed -i.bak 's/OK | DEGRADED/OK or DEGRADED/' "$TEST_ROOT/t3/cards/widget-status-flags.md" +assert_exit 1 "reworded falsification -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t3/spec.md" "$TEST_ROOT/t3/cards" +assert_out_contains "widget-status-flags" "failure names the card" + +echo "missing card file" +make_spec "$TEST_ROOT/t4"; make_cards "$TEST_ROOT/t4/cards" +rm "$TEST_ROOT/t4/cards/widget-show-table.md" +assert_exit 1 "missing card -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t4/spec.md" "$TEST_ROOT/t4/cards" +assert_out_contains "widget-show-table.md" "failure names the missing file" + +echo "missing required section" +make_spec "$TEST_ROOT/t5"; make_cards "$TEST_ROOT/t5/cards" +sed -i.bak '/^## Cleanup/,$d' "$TEST_ROOT/t5/cards/widget-show-table.md" +assert_exit 1 "card without Cleanup heading -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t5/spec.md" "$TEST_ROOT/t5/cards" +assert_out_contains "Cleanup" "failure names the section" + +echo "extra card is a warning, not a failure" +make_spec "$TEST_ROOT/t6"; make_cards "$TEST_ROOT/t6/cards" +good_card_1 > "$TEST_ROOT/t6/cards/extra-exploration.md" +assert_exit 0 "extra card -> exit 0" \ + "$CHECKER" "$TEST_ROOT/t6/spec.md" "$TEST_ROOT/t6/cards" +assert_out_contains "extra-exploration" "warning names the extra card" + +echo "no scenario table" +mkdir -p "$TEST_ROOT/t7/cards" +printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md" +assert_exit 2 "table-less spec -> exit 2" \ + "$CHECKER" "$TEST_ROOT/t7/spec.md" "$TEST_ROOT/t7/cards" +assert_out_contains "no scenario table" "diagnostic present" + +echo "heading match is case-insensitive" +make_spec "$TEST_ROOT/t8"; make_cards "$TEST_ROOT/t8/cards" +sed -i.bak 's/^## E2E scenario cards/## E2E Scenario Cards/' "$TEST_ROOT/t8/spec.md" +assert_exit 0 "title-case heading still found" \ + "$CHECKER" "$TEST_ROOT/t8/spec.md" "$TEST_ROOT/t8/cards" + +echo "usage" +assert_exit 64 "no args -> exit 64" "$CHECKER" +assert_exit 0 "--help -> exit 0" "$CHECKER" --help +assert_out_contains "Usage:" "help text present" + +echo +if [ "$FAILURES" -gt 0 ]; then echo "$FAILURES test(s) failed"; exit 1; fi +echo "all tests passed"