mirror of
https://github.com/obra/superpowers.git
synced 2026-08-07 22:59:20 +08:00
2a2e0b41c2
Filter backtick and tilde fenced examples before locating the E2E scenario-card table, so illustrative Markdown cannot satisfy the mechanical gate or shadow a later real table.\n\nParse only the documented leading/trailing-pipe row form and require a same-width delimiter row whose cells contain at least three hyphens with optional alignment colons. Missing or malformed delimiters now remain check failures instead of being silently treated as data.\n\nThe process-level harness records RED reproductions for both false-pass forms, fenced-only tables, and fenced examples before real tables, then verifies the focused GREEN behavior and prior contracts.
279 lines
10 KiB
Bash
Executable File
279 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
CHECKER="$REPO_ROOT/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec"
|
|
|
|
FAILURES=0
|
|
TEST_ROOT="$(mktemp -d)"
|
|
cleanup() { rm -rf "$TEST_ROOT"; }
|
|
trap cleanup EXIT
|
|
|
|
pass() { echo " [PASS] $1"; }
|
|
fail() { echo " [FAIL] $1"; FAILURES=$((FAILURES + 1)); }
|
|
|
|
assert_exit() { # expected_code description -- command...
|
|
local expected="$1" desc="$2"; shift 2
|
|
local code=0
|
|
"$@" >"$TEST_ROOT/out.txt" 2>&1 || code=$?
|
|
if [ "$code" -eq "$expected" ]; then pass "$desc"; else
|
|
fail "$desc (expected exit $expected, got $code)"; sed 's/^/ /' "$TEST_ROOT/out.txt"; fi
|
|
}
|
|
|
|
assert_out_contains() { # needle description
|
|
if grep -Fq -- "$1" "$TEST_ROOT/out.txt"; then pass "$2"; else
|
|
fail "$2 (output missing: $1)"; sed 's/^/ /' "$TEST_ROOT/out.txt"; fi
|
|
}
|
|
|
|
# ---- fixture builders ----------------------------------------------------
|
|
|
|
make_spec() { # dir (spec with 2-row table; row 2 has \| and regex chars)
|
|
mkdir -p "$1"
|
|
cat > "$1/spec.md" <<'EOF'
|
|
# Widget Design
|
|
|
|
## Requirements
|
|
|
|
Widgets render a table with a TOTAL row.
|
|
|
|
## E2E scenario cards
|
|
|
|
| Card | Covers | Falsification |
|
|
| --- | --- | --- |
|
|
| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. |
|
|
| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. |
|
|
EOF
|
|
}
|
|
|
|
good_card_1() {
|
|
cat <<'EOF'
|
|
# widget-show-table: table renders with TOTAL
|
|
|
|
**What this covers**: the rendered table.
|
|
|
|
## Pre-state
|
|
A built widget binary.
|
|
|
|
## Steps
|
|
1. Run `widget show`.
|
|
|
|
## Expected
|
|
If stdout's last line is not `TOTAL` followed by the
|
|
two-decimal sum (20.85 for the seed
|
|
fixture), or the TOTAL row is absent entirely, the scenario FAILS.
|
|
|
|
## Cleanup
|
|
Nothing to clean.
|
|
EOF
|
|
}
|
|
|
|
good_card_2() {
|
|
cat <<'EOF'
|
|
# widget-status-flags: status output
|
|
|
|
**What this covers**: status flags.
|
|
|
|
## Pre-state
|
|
A built widget binary.
|
|
|
|
## Steps
|
|
1. Run `widget status`.
|
|
|
|
## Expected
|
|
If `widget status` does not print exactly `OK | DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS.
|
|
|
|
## Cleanup
|
|
Nothing to clean.
|
|
EOF
|
|
}
|
|
|
|
make_cards() { # dir
|
|
mkdir -p "$1"
|
|
good_card_1 > "$1/widget-show-table.md"
|
|
good_card_2 > "$1/widget-status-flags.md"
|
|
}
|
|
|
|
# ---- tests ----------------------------------------------------------------
|
|
|
|
echo "happy path"
|
|
make_spec "$TEST_ROOT/t1"; make_cards "$TEST_ROOT/t1/cards"
|
|
assert_exit 0 "2 rows, 2 conforming cards -> exit 0" \
|
|
"$CHECKER" "$TEST_ROOT/t1/spec.md" "$TEST_ROOT/t1/cards"
|
|
|
|
echo "re-wrapped falsification line still matches (whitespace normalization)"
|
|
# good_card_1 already wraps the line across three lines; covered above. Prove
|
|
# the inverse too: collapse the card line to one line, still passes.
|
|
make_spec "$TEST_ROOT/t2"; make_cards "$TEST_ROOT/t2/cards"
|
|
perl -0pi -e 's/\n(two-decimal)/ $1/; s/\n(fixture\))/ $1/' "$TEST_ROOT/t2/cards/widget-show-table.md" 2>/dev/null || \
|
|
sed -i '' -e ':a' -e 'N;$!ba' -e 's/the\ntwo-decimal/the two-decimal/' "$TEST_ROOT/t2/cards/widget-show-table.md"
|
|
assert_exit 0 "single-line variant -> exit 0" \
|
|
"$CHECKER" "$TEST_ROOT/t2/spec.md" "$TEST_ROOT/t2/cards"
|
|
|
|
echo "escaped pipe in table cell matches literal pipe in card"
|
|
# covered by widget-status-flags in the happy path; also prove failure when
|
|
# the card drops the pipe phrase entirely:
|
|
make_spec "$TEST_ROOT/t3"; make_cards "$TEST_ROOT/t3/cards"
|
|
sed -i.bak 's/OK | DEGRADED/OK or DEGRADED/' "$TEST_ROOT/t3/cards/widget-status-flags.md"
|
|
assert_exit 1 "reworded falsification -> exit 1" \
|
|
"$CHECKER" "$TEST_ROOT/t3/spec.md" "$TEST_ROOT/t3/cards"
|
|
assert_out_contains "widget-status-flags" "failure names the card"
|
|
|
|
echo "verbatim line outside Expected does not count"
|
|
make_spec "$TEST_ROOT/t3b"; make_cards "$TEST_ROOT/t3b/cards"
|
|
cat > "$TEST_ROOT/t3b/cards/widget-show-table.md" <<'EOF'
|
|
# widget-show-table: table renders with TOTAL
|
|
|
|
**What this covers**: If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS.
|
|
|
|
## Pre-state
|
|
A built widget binary.
|
|
|
|
## Steps
|
|
1. Run `widget show`.
|
|
|
|
## Expected
|
|
The widget prints a friendly banner and exits zero.
|
|
|
|
## Cleanup
|
|
Nothing to clean.
|
|
EOF
|
|
assert_exit 1 "line only outside Expected -> exit 1" \
|
|
"$CHECKER" "$TEST_ROOT/t3b/spec.md" "$TEST_ROOT/t3b/cards"
|
|
assert_out_contains "widget-show-table" "failure names the card"
|
|
|
|
echo "level-1 heading after Expected does not extend the section (false-PASS regression)"
|
|
# ## Expected is vague; a later # Appendix (level-1 heading, no intervening
|
|
# ##+ heading) carries the verbatim falsification line. The Expected section
|
|
# must end at the level-1 heading, so this must FAIL, not false-PASS.
|
|
make_spec "$TEST_ROOT/t3c"; make_cards "$TEST_ROOT/t3c/cards"
|
|
cat > "$TEST_ROOT/t3c/cards/widget-show-table.md" <<'EOF'
|
|
# widget-show-table: table renders with TOTAL
|
|
|
|
**What this covers**: the rendered table.
|
|
|
|
## Pre-state
|
|
A built widget binary.
|
|
|
|
## Steps
|
|
1. Run `widget show`.
|
|
|
|
## Expected
|
|
The widget prints something on screen.
|
|
|
|
# Appendix
|
|
|
|
If stdout's last line is not `TOTAL` followed by the
|
|
two-decimal sum (20.85 for the seed
|
|
fixture), or the TOTAL row is absent entirely, the scenario FAILS.
|
|
|
|
## Cleanup
|
|
Nothing to clean.
|
|
EOF
|
|
assert_exit 1 "level-1 heading terminates Expected section -> exit 1" \
|
|
"$CHECKER" "$TEST_ROOT/t3c/spec.md" "$TEST_ROOT/t3c/cards"
|
|
assert_out_contains "widget-show-table" "failure names the card"
|
|
|
|
echo "missing card file"
|
|
make_spec "$TEST_ROOT/t4"; make_cards "$TEST_ROOT/t4/cards"
|
|
rm "$TEST_ROOT/t4/cards/widget-show-table.md"
|
|
assert_exit 1 "missing card -> exit 1" \
|
|
"$CHECKER" "$TEST_ROOT/t4/spec.md" "$TEST_ROOT/t4/cards"
|
|
assert_out_contains "widget-show-table.md" "failure names the missing file"
|
|
|
|
echo "missing required section"
|
|
make_spec "$TEST_ROOT/t5"; make_cards "$TEST_ROOT/t5/cards"
|
|
sed -i.bak '/^## Cleanup/,$d' "$TEST_ROOT/t5/cards/widget-show-table.md"
|
|
assert_exit 1 "card without Cleanup heading -> exit 1" \
|
|
"$CHECKER" "$TEST_ROOT/t5/spec.md" "$TEST_ROOT/t5/cards"
|
|
assert_out_contains "Cleanup" "failure names the section"
|
|
|
|
echo "presence grep requires exact Expected heading, not a prefix match"
|
|
make_spec "$TEST_ROOT/t9"; make_cards "$TEST_ROOT/t9/cards"
|
|
sed -i.bak 's/^## Expected$/## Expectedly odd heading/' "$TEST_ROOT/t9/cards/widget-show-table.md"
|
|
assert_exit 1 "prefix-matching heading -> exit 1" \
|
|
"$CHECKER" "$TEST_ROOT/t9/spec.md" "$TEST_ROOT/t9/cards"
|
|
assert_out_contains "missing ## Expected section" "failure names the Expected section"
|
|
|
|
echo "extra card is a warning, not a failure"
|
|
make_spec "$TEST_ROOT/t6"; make_cards "$TEST_ROOT/t6/cards"
|
|
good_card_1 > "$TEST_ROOT/t6/cards/extra-exploration.md"
|
|
assert_exit 0 "extra card -> exit 0" \
|
|
"$CHECKER" "$TEST_ROOT/t6/spec.md" "$TEST_ROOT/t6/cards"
|
|
assert_out_contains "extra-exploration" "warning names the extra card"
|
|
|
|
echo "scenario table requires a delimiter row"
|
|
make_spec "$TEST_ROOT/t10"; make_cards "$TEST_ROOT/t10/cards"
|
|
sed -i.bak '/^| --- | --- | --- |$/d' "$TEST_ROOT/t10/spec.md"
|
|
assert_exit 1 "header followed by data without delimiter -> exit 1" \
|
|
"$CHECKER" "$TEST_ROOT/t10/spec.md" "$TEST_ROOT/t10/cards"
|
|
|
|
echo "scenario table requires a valid delimiter row"
|
|
make_spec "$TEST_ROOT/t11"; make_cards "$TEST_ROOT/t11/cards"
|
|
sed -i.bak 's/^| --- | --- | --- |$/| -- | --- | --- |/' "$TEST_ROOT/t11/spec.md"
|
|
assert_exit 1 "delimiter cells require at least three hyphens -> exit 1" \
|
|
"$CHECKER" "$TEST_ROOT/t11/spec.md" "$TEST_ROOT/t11/cards"
|
|
|
|
echo "scenario table inside a fenced example does not count"
|
|
mkdir -p "$TEST_ROOT/t12/cards"
|
|
make_cards "$TEST_ROOT/t12/cards"
|
|
cat > "$TEST_ROOT/t12/spec.md" <<'EOF'
|
|
# Widget Design
|
|
|
|
## E2E scenario cards
|
|
|
|
```markdown
|
|
| Card | Covers | Falsification |
|
|
| --- | --- | --- |
|
|
| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. |
|
|
| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. |
|
|
```
|
|
EOF
|
|
assert_exit 2 "fenced-only table -> exit 2" \
|
|
"$CHECKER" "$TEST_ROOT/t12/spec.md" "$TEST_ROOT/t12/cards"
|
|
|
|
echo "fenced example before a real table is ignored"
|
|
mkdir -p "$TEST_ROOT/t13/cards"
|
|
make_cards "$TEST_ROOT/t13/cards"
|
|
cat > "$TEST_ROOT/t13/spec.md" <<'EOF'
|
|
# Widget Design
|
|
|
|
## E2E scenario cards
|
|
|
|
~~~markdown
|
|
| Card | Covers | Falsification |
|
|
| --- | --- | --- |
|
|
| fake-card | Example only | If the example is absent, the scenario FAILS. |
|
|
~~~
|
|
|
|
| Card | Covers | Falsification |
|
|
| --- | --- | --- |
|
|
| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. |
|
|
| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. |
|
|
EOF
|
|
assert_exit 0 "real table after fenced example -> exit 0" \
|
|
"$CHECKER" "$TEST_ROOT/t13/spec.md" "$TEST_ROOT/t13/cards"
|
|
|
|
echo "no scenario table"
|
|
mkdir -p "$TEST_ROOT/t7/cards"
|
|
printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md"
|
|
assert_exit 2 "table-less spec -> exit 2" \
|
|
"$CHECKER" "$TEST_ROOT/t7/spec.md" "$TEST_ROOT/t7/cards"
|
|
assert_out_contains "no scenario table" "diagnostic present"
|
|
assert_out_contains "heading must be exactly" "diagnostic includes naming hint"
|
|
|
|
echo "heading match is case-insensitive"
|
|
make_spec "$TEST_ROOT/t8"; make_cards "$TEST_ROOT/t8/cards"
|
|
sed -i.bak 's/^## E2E scenario cards/## E2E Scenario Cards/' "$TEST_ROOT/t8/spec.md"
|
|
assert_exit 0 "title-case heading still found" \
|
|
"$CHECKER" "$TEST_ROOT/t8/spec.md" "$TEST_ROOT/t8/cards"
|
|
|
|
echo "usage"
|
|
assert_exit 64 "no args -> exit 64" "$CHECKER"
|
|
assert_exit 0 "--help -> exit 0" "$CHECKER" --help
|
|
assert_out_contains "Usage:" "help text present"
|
|
|
|
echo
|
|
if [ "$FAILURES" -gt 0 ]; then echo "$FAILURES test(s) failed"; exit 1; fi
|
|
echo "all tests passed"
|