#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" CHECKER="$REPO_ROOT/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec" FAILURES=0 TEST_ROOT="$(mktemp -d)" cleanup() { rm -rf "$TEST_ROOT"; } trap cleanup EXIT pass() { echo " [PASS] $1"; } fail() { echo " [FAIL] $1"; FAILURES=$((FAILURES + 1)); } assert_exit() { # expected_code description -- command... local expected="$1" desc="$2"; shift 2 local code=0 "$@" >"$TEST_ROOT/out.txt" 2>&1 || code=$? if [ "$code" -eq "$expected" ]; then pass "$desc"; else fail "$desc (expected exit $expected, got $code)"; sed 's/^/ /' "$TEST_ROOT/out.txt"; fi } assert_out_contains() { # needle description if grep -Fq -- "$1" "$TEST_ROOT/out.txt"; then pass "$2"; else fail "$2 (output missing: $1)"; sed 's/^/ /' "$TEST_ROOT/out.txt"; fi } # ---- fixture builders ---------------------------------------------------- make_spec() { # dir (spec with 2-row table; row 2 has \| and regex chars) mkdir -p "$1" cat > "$1/spec.md" <<'EOF' # Widget Design ## Requirements Widgets render a table with a TOTAL row. ## E2E scenario cards | Card | Covers | Falsification | | --- | --- | --- | | widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. | | widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. | EOF } good_card_1() { cat <<'EOF' # widget-show-table: table renders with TOTAL **What this covers**: the rendered table. ## Pre-state A built widget binary. ## Steps 1. Run `widget show`. ## Expected If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. ## Cleanup Nothing to clean. EOF } good_card_2() { cat <<'EOF' # widget-status-flags: status output **What this covers**: status flags. ## Pre-state A built widget binary. ## Steps 1. Run `widget status`. ## Expected If `widget status` does not print exactly `OK | DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. ## Cleanup Nothing to clean. EOF } make_cards() { # dir mkdir -p "$1" good_card_1 > "$1/widget-show-table.md" good_card_2 > "$1/widget-status-flags.md" } # ---- tests ---------------------------------------------------------------- echo "happy path" make_spec "$TEST_ROOT/t1"; make_cards "$TEST_ROOT/t1/cards" assert_exit 0 "2 rows, 2 conforming cards -> exit 0" \ "$CHECKER" "$TEST_ROOT/t1/spec.md" "$TEST_ROOT/t1/cards" echo "re-wrapped falsification line still matches (whitespace normalization)" # good_card_1 already wraps the line across three lines; covered above. Prove # the inverse too: collapse the card line to one line, still passes. make_spec "$TEST_ROOT/t2"; make_cards "$TEST_ROOT/t2/cards" perl -0pi -e 's/\n(two-decimal)/ $1/; s/\n(fixture\))/ $1/' "$TEST_ROOT/t2/cards/widget-show-table.md" 2>/dev/null || \ sed -i '' -e ':a' -e 'N;$!ba' -e 's/the\ntwo-decimal/the two-decimal/' "$TEST_ROOT/t2/cards/widget-show-table.md" assert_exit 0 "single-line variant -> exit 0" \ "$CHECKER" "$TEST_ROOT/t2/spec.md" "$TEST_ROOT/t2/cards" echo "escaped pipe in table cell matches literal pipe in card" # covered by widget-status-flags in the happy path; also prove failure when # the card drops the pipe phrase entirely: make_spec "$TEST_ROOT/t3"; make_cards "$TEST_ROOT/t3/cards" sed -i.bak 's/OK | DEGRADED/OK or DEGRADED/' "$TEST_ROOT/t3/cards/widget-status-flags.md" assert_exit 1 "reworded falsification -> exit 1" \ "$CHECKER" "$TEST_ROOT/t3/spec.md" "$TEST_ROOT/t3/cards" assert_out_contains "widget-status-flags" "failure names the card" echo "verbatim line outside Expected does not count" make_spec "$TEST_ROOT/t3b"; make_cards "$TEST_ROOT/t3b/cards" cat > "$TEST_ROOT/t3b/cards/widget-show-table.md" <<'EOF' # widget-show-table: table renders with TOTAL **What this covers**: If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. ## Pre-state A built widget binary. ## Steps 1. Run `widget show`. ## Expected The widget prints a friendly banner and exits zero. ## Cleanup Nothing to clean. EOF assert_exit 1 "line only outside Expected -> exit 1" \ "$CHECKER" "$TEST_ROOT/t3b/spec.md" "$TEST_ROOT/t3b/cards" assert_out_contains "widget-show-table" "failure names the card" echo "level-1 heading after Expected does not extend the section (false-PASS regression)" # ## Expected is vague; a later # Appendix (level-1 heading, no intervening # ##+ heading) carries the verbatim falsification line. The Expected section # must end at the level-1 heading, so this must FAIL, not false-PASS. make_spec "$TEST_ROOT/t3c"; make_cards "$TEST_ROOT/t3c/cards" cat > "$TEST_ROOT/t3c/cards/widget-show-table.md" <<'EOF' # widget-show-table: table renders with TOTAL **What this covers**: the rendered table. ## Pre-state A built widget binary. ## Steps 1. Run `widget show`. ## Expected The widget prints something on screen. # Appendix If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. ## Cleanup Nothing to clean. EOF assert_exit 1 "level-1 heading terminates Expected section -> exit 1" \ "$CHECKER" "$TEST_ROOT/t3c/spec.md" "$TEST_ROOT/t3c/cards" assert_out_contains "widget-show-table" "failure names the card" echo "missing card file" make_spec "$TEST_ROOT/t4"; make_cards "$TEST_ROOT/t4/cards" rm "$TEST_ROOT/t4/cards/widget-show-table.md" assert_exit 1 "missing card -> exit 1" \ "$CHECKER" "$TEST_ROOT/t4/spec.md" "$TEST_ROOT/t4/cards" assert_out_contains "widget-show-table.md" "failure names the missing file" echo "missing required section" make_spec "$TEST_ROOT/t5"; make_cards "$TEST_ROOT/t5/cards" sed -i.bak '/^## Cleanup/,$d' "$TEST_ROOT/t5/cards/widget-show-table.md" assert_exit 1 "card without Cleanup heading -> exit 1" \ "$CHECKER" "$TEST_ROOT/t5/spec.md" "$TEST_ROOT/t5/cards" assert_out_contains "Cleanup" "failure names the section" echo "presence grep requires exact Expected heading, not a prefix match" make_spec "$TEST_ROOT/t9"; make_cards "$TEST_ROOT/t9/cards" sed -i.bak 's/^## Expected$/## Expectedly odd heading/' "$TEST_ROOT/t9/cards/widget-show-table.md" assert_exit 1 "prefix-matching heading -> exit 1" \ "$CHECKER" "$TEST_ROOT/t9/spec.md" "$TEST_ROOT/t9/cards" assert_out_contains "missing ## Expected section" "failure names the Expected section" echo "extra card is a warning, not a failure" make_spec "$TEST_ROOT/t6"; make_cards "$TEST_ROOT/t6/cards" good_card_1 > "$TEST_ROOT/t6/cards/extra-exploration.md" assert_exit 0 "extra card -> exit 0" \ "$CHECKER" "$TEST_ROOT/t6/spec.md" "$TEST_ROOT/t6/cards" assert_out_contains "extra-exploration" "warning names the extra card" echo "no scenario table" mkdir -p "$TEST_ROOT/t7/cards" printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md" assert_exit 2 "table-less spec -> exit 2" \ "$CHECKER" "$TEST_ROOT/t7/spec.md" "$TEST_ROOT/t7/cards" assert_out_contains "no scenario table" "diagnostic present" assert_out_contains "heading must be exactly" "diagnostic includes naming hint" echo "heading match is case-insensitive" make_spec "$TEST_ROOT/t8"; make_cards "$TEST_ROOT/t8/cards" sed -i.bak 's/^## E2E scenario cards/## E2E Scenario Cards/' "$TEST_ROOT/t8/spec.md" assert_exit 0 "title-case heading still found" \ "$CHECKER" "$TEST_ROOT/t8/spec.md" "$TEST_ROOT/t8/cards" echo "usage" assert_exit 64 "no args -> exit 64" "$CHECKER" assert_exit 0 "--help -> exit 0" "$CHECKER" --help assert_out_contains "Usage:" "help text present" echo if [ "$FAILURES" -gt 0 ]; then echo "$FAILURES test(s) failed"; exit 1; fi echo "all tests passed"