diff --git a/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec b/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec index 81846092..4b491564 100755 --- a/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec +++ b/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec @@ -38,7 +38,7 @@ normalize() { tr -s '[:space:]' ' ' | sed -e 's/^ //' -e 's/ $//'; } # any ##+ level; section ends at the next heading or EOF). expected_section() { awk ' - /^#{2,}[[:space:]]/ { + /^#{1,6}[[:space:]]/ { low = tolower($0) if (low ~ /^#+[[:space:]]*expected[[:space:]]*$/) { insec = 1; next } if (insec) exit @@ -60,6 +60,7 @@ TABLE="$(awk ' if [ -z "$TABLE" ]; then echo "no scenario table: $SPEC has no \"E2E scenario cards\" heading with a table under it" >&2 + echo "(heading must be exactly \"E2E scenario cards\" — no numbering or extra words)" >&2 exit 2 fi @@ -125,7 +126,7 @@ while [ "$i" -lt "$ROWS" ]; do esac grep -q '\*\*What this covers\*\*' "$f" || fail "$f: missing **What this covers**" for sec in Pre-state Steps Expected Cleanup; do - grep -Eiq "^#{2,}[[:space:]]*${sec}" "$f" || fail "$f: missing ## ${sec} section" + grep -Eiq "^#{2,}[[:space:]]*${sec}[[:space:]]*$" "$f" || fail "$f: missing ## ${sec} section" done i=$((i + 1)) done diff --git a/tests/agentic-e2e-checker/test-check-cards-against-spec.sh b/tests/agentic-e2e-checker/test-check-cards-against-spec.sh index 516eb22f..44385ef6 100755 --- a/tests/agentic-e2e-checker/test-check-cards-against-spec.sh +++ b/tests/agentic-e2e-checker/test-check-cards-against-spec.sh @@ -142,6 +142,38 @@ assert_exit 1 "line only outside Expected -> exit 1" \ "$CHECKER" "$TEST_ROOT/t3b/spec.md" "$TEST_ROOT/t3b/cards" assert_out_contains "widget-show-table" "failure names the card" +echo "level-1 heading after Expected does not extend the section (false-PASS regression)" +# ## Expected is vague; a later # Appendix (level-1 heading, no intervening +# ##+ heading) carries the verbatim falsification line. The Expected section +# must end at the level-1 heading, so this must FAIL, not false-PASS. +make_spec "$TEST_ROOT/t3c"; make_cards "$TEST_ROOT/t3c/cards" +cat > "$TEST_ROOT/t3c/cards/widget-show-table.md" <<'EOF' +# widget-show-table: table renders with TOTAL + +**What this covers**: the rendered table. + +## Pre-state +A built widget binary. + +## Steps +1. Run `widget show`. + +## Expected +The widget prints something on screen. + +# Appendix + +If stdout's last line is not `TOTAL` followed by the +two-decimal sum (20.85 for the seed +fixture), or the TOTAL row is absent entirely, the scenario FAILS. + +## Cleanup +Nothing to clean. +EOF +assert_exit 1 "level-1 heading terminates Expected section -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t3c/spec.md" "$TEST_ROOT/t3c/cards" +assert_out_contains "widget-show-table" "failure names the card" + echo "missing card file" make_spec "$TEST_ROOT/t4"; make_cards "$TEST_ROOT/t4/cards" rm "$TEST_ROOT/t4/cards/widget-show-table.md" @@ -156,6 +188,13 @@ assert_exit 1 "card without Cleanup heading -> exit 1" \ "$CHECKER" "$TEST_ROOT/t5/spec.md" "$TEST_ROOT/t5/cards" assert_out_contains "Cleanup" "failure names the section" +echo "presence grep requires exact Expected heading, not a prefix match" +make_spec "$TEST_ROOT/t9"; make_cards "$TEST_ROOT/t9/cards" +sed -i.bak 's/^## Expected$/## Expectedly odd heading/' "$TEST_ROOT/t9/cards/widget-show-table.md" +assert_exit 1 "prefix-matching heading -> exit 1" \ + "$CHECKER" "$TEST_ROOT/t9/spec.md" "$TEST_ROOT/t9/cards" +assert_out_contains "missing ## Expected section" "failure names the Expected section" + echo "extra card is a warning, not a failure" make_spec "$TEST_ROOT/t6"; make_cards "$TEST_ROOT/t6/cards" good_card_1 > "$TEST_ROOT/t6/cards/extra-exploration.md" @@ -169,6 +208,7 @@ printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md" assert_exit 2 "table-less spec -> exit 2" \ "$CHECKER" "$TEST_ROOT/t7/spec.md" "$TEST_ROOT/t7/cards" assert_out_contains "no scenario table" "diagnostic present" +assert_out_contains "heading must be exactly" "diagnostic includes naming hint" echo "heading match is case-insensitive" make_spec "$TEST_ROOT/t8"; make_cards "$TEST_ROOT/t8/cards"