From 9f4f46639c7981ac25a4d0a3a554c9262adb5a77 Mon Sep 17 00:00:00 2001 From: Drew Ritter Date: Thu, 6 Aug 2026 16:51:54 -0700 Subject: [PATCH] fix(e2e): preserve evidence pipeline failures --- .../recording-a-proof-movie.md | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/skills/agentic-end-to-end-testing/recording-a-proof-movie.md b/skills/agentic-end-to-end-testing/recording-a-proof-movie.md index 2441f51b..b179b6a9 100644 --- a/skills/agentic-end-to-end-testing/recording-a-proof-movie.md +++ b/skills/agentic-end-to-end-testing/recording-a-proof-movie.md @@ -31,14 +31,18 @@ markers. Use `bash`, not `zsh` — zsh's read-only `$status` injects a spurious error *after* a passing run and pollutes the movie. ```bash +run_log=/run.log bash -o pipefail -c ' - printf "MANUAL_E2E_KIND=\n"; - printf "STARTED_AT="; date -u +%Y-%m-%dT%H:%M:%SZ; - ; # e.g. xcodebuild test-without-building ... -resultBundlePath ... - rc=$?; - printf "FINISHED_AT="; date -u +%Y-%m-%dT%H:%M:%SZ; - printf "EXIT_STATUS=%s\n" "$rc"; exit "$rc" -' 2>&1 | tee /run.log + { + printf "MANUAL_E2E_KIND=\n"; + printf "STARTED_AT="; date -u +%Y-%m-%dT%H:%M:%SZ; + ; # e.g. xcodebuild test-without-building ... -resultBundlePath ... + rc=$?; + printf "FINISHED_AT="; date -u +%Y-%m-%dT%H:%M:%SZ; + printf "EXIT_STATUS=%s\n" "$rc"; + exit "$rc" + } 2>&1 | tee "$1" +' bash "$run_log" ``` ## Snapshot external state before and after @@ -47,10 +51,13 @@ If the run touches a remote host or a shared tmux, snapshot it identically pre- and post-run and diff. Equal snapshots prove the run left no residue. ```bash -ssh 'date -Is; tmux list-sessions -F "#{session_name}|#{session_windows}|attached=#{session_attached}"; \ - ps -eo pid=,args= | awk "// {print}"; find /tmp -maxdepth 1 -name "" | wc -l' \ - | tee /pre-snapshot.txt -# ... run gate ... then repeat with SNAPSHOT_KIND=post => post-snapshot.txt ; assert they match +host= +snapshot_path=/pre-snapshot.txt +bash -o pipefail -c 'ssh "$1" "$2" | tee "$3"' bash \ + "$host" \ + 'date -Is; tmux list-sessions -F "#{session_name}|#{session_windows}|attached=#{session_attached}"; ps -eo pid=,args= | awk "// {print}"; find /tmp -maxdepth 1 -name "" | wc -l' \ + "$snapshot_path" +# ... run gate ... then repeat for post-snapshot.txt. Begin comparison only after both capture commands succeed. ``` ## Render the reel from the log