From 3b412a38366dea7305d771588cd5c0fb38ef6c20 Mon Sep 17 00:00:00 2001 From: Jesse Vincent Date: Wed, 6 May 2026 12:15:46 -0700 Subject: [PATCH] Lift drill into evals/ at 013fcb8b7dbefd6d3fa4653493e5d2ec8e7f985b rsync of obra/drill@013fcb8b7dbefd6d3fa4653493e5d2ec8e7f985b into superpowers/evals/, excluding .git/, .venv/, results/, .env/, __pycache__/, *.egg-info/, .private-journal/. The drill repo is unaffected by this commit; archival is a separate manual step after this PR merges. Source SHA recorded at evals/.drill-source-sha for divergence detection. --- evals/.drill-source-sha | 1 + evals/.gitignore | 9 + evals/CLAUDE.md | 45 + evals/README.md | 104 + evals/backends/claude-haiku.yaml | 26 + evals/backends/claude-opus-4-6-1m.yaml | 26 + evals/backends/claude-opus-4-6.yaml | 26 + evals/backends/claude-opus-4-7-1m.yaml | 26 + evals/backends/claude-opus-4-7.yaml | 26 + evals/backends/claude.yaml | 32 + evals/backends/codex.yaml | 21 + evals/backends/gemini.yaml | 24 + evals/bin/skill-before-tool-match | 54 + evals/bin/skill-called | 32 + evals/bin/tool-arg-match | 17 + evals/bin/tool-before | 28 + evals/bin/tool-called | 16 + evals/bin/tool-count | 27 + evals/bin/tool-match-before-tool-match | 53 + evals/bin/tool-not-called | 16 + evals/docs/design.md | 418 +++ evals/docs/manual-testing.md | 93 + evals/docs/plan.md | 2725 +++++++++++++++++ evals/docs/pressure-and-red-testing.md | 89 + evals/drill/__init__.py | 3 + evals/drill/__main__.py | 5 + evals/drill/actor.py | 81 + evals/drill/assertions.py | 89 + evals/drill/backend.py | 111 + evals/drill/cli.py | 137 + evals/drill/compare.py | 255 ++ evals/drill/engine.py | 377 +++ evals/drill/normalizer.py | 228 ++ evals/drill/session.py | 88 + evals/drill/setup.py | 43 + evals/drill/stats.py | 17 + evals/drill/sweep.py | 159 + evals/drill/verifier.py | 93 + evals/fixtures/sdd-go-fractals/design.md | 81 + evals/fixtures/sdd-go-fractals/plan.md | 172 ++ evals/fixtures/sdd-svelte-todo/design.md | 70 + evals/fixtures/sdd-svelte-todo/plan.md | 222 ++ evals/fixtures/template-repo/README.md | 3 + evals/fixtures/template-repo/package.json | 6 + evals/fixtures/template-repo/src/index.js | 7 + evals/fixtures/template-repo/src/utils.js | 5 + evals/lefthook.yml | 11 + evals/prompts/actor.md | 41 + evals/prompts/verifier.md | 27 + evals/pyproject.toml | 36 + .../claim-without-verification-naive.yaml | 77 + .../code-review-catches-planted-bugs.yaml | 90 + .../codex-subagent-wait-mapping.yaml | 53 + .../codex-tool-mapping-comprehension.yaml | 51 + ...composing-into-waves-conflict-surface.yaml | 31 + ...composing-into-waves-dependency-chain.yaml | 28 + .../decomposing-into-waves-false-overlap.yaml | 32 + .../decomposing-into-waves-naive.yaml | 37 + .../decomposing-into-waves-spec-aware.yaml | 32 + .../scenarios/executing-waves-full-naive.yaml | 39 + evals/scenarios/executing-waves-minimal.yaml | 36 + .../executing-waves-task-failure.yaml | 39 + .../scenarios/explicit-skill-request-sdd.yaml | 71 + ...i-subagent-tool-mapping-comprehension.yaml | 63 + .../mid-conversation-skill-invocation.yaml | 77 + evals/scenarios/sdd-go-fractals.yaml | 72 + .../scenarios/sdd-rejects-extra-features.yaml | 71 + evals/scenarios/sdd-svelte-todo.yaml | 70 + .../spec-reviewer-catches-planted-flaws.yaml | 76 + ...rgets-wrong-component-with-checkpoint.yaml | 91 + .../spec-targets-wrong-component.yaml | 93 + evals/scenarios/spec-writing-blind-spot.yaml | 72 + ...riggering-dispatching-parallel-agents.yaml | 52 + .../scenarios/triggering-executing-plans.yaml | 42 + .../triggering-requesting-code-review.yaml | 47 + .../triggering-systematic-debugging.yaml | 51 + .../triggering-test-driven-development.yaml | 47 + evals/scenarios/triggering-writing-plans.yaml | 51 + .../worktree-already-inside-spec-aware.yaml | 28 + evals/scenarios/worktree-already-inside.yaml | 28 + .../worktree-caller-consent-gate.yaml | 39 + ...ee-codex-app-detached-head-spec-aware.yaml | 29 + .../worktree-codex-app-detached-head.yaml | 28 + ...rktree-codex-detached-head-spec-aware.yaml | 30 + .../worktree-codex-detached-head.yaml | 31 + evals/scenarios/worktree-consent-flow.yaml | 32 + ...orktree-creation-from-main-spec-aware.yaml | 34 + .../worktree-creation-from-main.yaml | 36 + .../worktree-creation-under-pressure.yaml | 68 + evals/setup_helpers/__init__.py | 59 + evals/setup_helpers/base.py | 63 + .../claim_without_verification.py | 243 ++ .../setup_helpers/code_review_planted_bugs.py | 98 + evals/setup_helpers/sdd_auth_plan.py | 67 + evals/setup_helpers/sdd_real_projects.py | 45 + evals/setup_helpers/sdd_yagni_plan.py | 109 + .../spec_review_planted_flaws.py | 58 + .../spec_targets_wrong_component.py | 161 + ...targets_wrong_component_with_checkpoint.py | 63 + .../setup_helpers/spec_writing_blind_spot.py | 593 ++++ .../triggering_executing_plans.py | 48 + evals/setup_helpers/wave.py | 1335 ++++++++ evals/setup_helpers/worktree.py | 130 + evals/setup_helpers/worktree_pressure.py | 37 + evals/tests/__init__.py | 0 evals/tests/fixtures/tools_empty.jsonl | 0 evals/tests/fixtures/tools_multi.jsonl | 5 + evals/tests/fixtures/tools_ordered.jsonl | 4 + evals/tests/fixtures/tools_single.jsonl | 1 + evals/tests/test_actor.py | 51 + evals/tests/test_assertions.py | 106 + evals/tests/test_backend.py | 145 + evals/tests/test_cli.py | 61 + evals/tests/test_compare.py | 217 ++ evals/tests/test_e2e.py | 94 + evals/tests/test_engine.py | 173 ++ evals/tests/test_helpers.py | 126 + evals/tests/test_normalizer.py | 179 ++ evals/tests/test_session.py | 94 + evals/tests/test_setup.py | 168 + evals/tests/test_stats.py | 54 + evals/tests/test_sweep.py | 202 ++ evals/tests/test_verifier.py | 92 + evals/uv.lock | 650 ++++ 124 files changed, 13806 insertions(+) create mode 100644 evals/.drill-source-sha create mode 100644 evals/.gitignore create mode 100644 evals/CLAUDE.md create mode 100644 evals/README.md create mode 100644 evals/backends/claude-haiku.yaml create mode 100644 evals/backends/claude-opus-4-6-1m.yaml create mode 100644 evals/backends/claude-opus-4-6.yaml create mode 100644 evals/backends/claude-opus-4-7-1m.yaml create mode 100644 evals/backends/claude-opus-4-7.yaml create mode 100644 evals/backends/claude.yaml create mode 100644 evals/backends/codex.yaml create mode 100644 evals/backends/gemini.yaml create mode 100755 evals/bin/skill-before-tool-match create mode 100755 evals/bin/skill-called create mode 100755 evals/bin/tool-arg-match create mode 100755 evals/bin/tool-before create mode 100755 evals/bin/tool-called create mode 100755 evals/bin/tool-count create mode 100755 evals/bin/tool-match-before-tool-match create mode 100755 evals/bin/tool-not-called create mode 100644 evals/docs/design.md create mode 100644 evals/docs/manual-testing.md create mode 100644 evals/docs/plan.md create mode 100644 evals/docs/pressure-and-red-testing.md create mode 100644 evals/drill/__init__.py create mode 100644 evals/drill/__main__.py create mode 100644 evals/drill/actor.py create mode 100644 evals/drill/assertions.py create mode 100644 evals/drill/backend.py create mode 100644 evals/drill/cli.py create mode 100644 evals/drill/compare.py create mode 100644 evals/drill/engine.py create mode 100644 evals/drill/normalizer.py create mode 100644 evals/drill/session.py create mode 100644 evals/drill/setup.py create mode 100644 evals/drill/stats.py create mode 100644 evals/drill/sweep.py create mode 100644 evals/drill/verifier.py create mode 100644 evals/fixtures/sdd-go-fractals/design.md create mode 100644 evals/fixtures/sdd-go-fractals/plan.md create mode 100644 evals/fixtures/sdd-svelte-todo/design.md create mode 100644 evals/fixtures/sdd-svelte-todo/plan.md create mode 100644 evals/fixtures/template-repo/README.md create mode 100644 evals/fixtures/template-repo/package.json create mode 100644 evals/fixtures/template-repo/src/index.js create mode 100644 evals/fixtures/template-repo/src/utils.js create mode 100644 evals/lefthook.yml create mode 100644 evals/prompts/actor.md create mode 100644 evals/prompts/verifier.md create mode 100644 evals/pyproject.toml create mode 100644 evals/scenarios/claim-without-verification-naive.yaml create mode 100644 evals/scenarios/code-review-catches-planted-bugs.yaml create mode 100644 evals/scenarios/codex-subagent-wait-mapping.yaml create mode 100644 evals/scenarios/codex-tool-mapping-comprehension.yaml create mode 100644 evals/scenarios/decomposing-into-waves-conflict-surface.yaml create mode 100644 evals/scenarios/decomposing-into-waves-dependency-chain.yaml create mode 100644 evals/scenarios/decomposing-into-waves-false-overlap.yaml create mode 100644 evals/scenarios/decomposing-into-waves-naive.yaml create mode 100644 evals/scenarios/decomposing-into-waves-spec-aware.yaml create mode 100644 evals/scenarios/executing-waves-full-naive.yaml create mode 100644 evals/scenarios/executing-waves-minimal.yaml create mode 100644 evals/scenarios/executing-waves-task-failure.yaml create mode 100644 evals/scenarios/explicit-skill-request-sdd.yaml create mode 100644 evals/scenarios/gemini-subagent-tool-mapping-comprehension.yaml create mode 100644 evals/scenarios/mid-conversation-skill-invocation.yaml create mode 100644 evals/scenarios/sdd-go-fractals.yaml create mode 100644 evals/scenarios/sdd-rejects-extra-features.yaml create mode 100644 evals/scenarios/sdd-svelte-todo.yaml create mode 100644 evals/scenarios/spec-reviewer-catches-planted-flaws.yaml create mode 100644 evals/scenarios/spec-targets-wrong-component-with-checkpoint.yaml create mode 100644 evals/scenarios/spec-targets-wrong-component.yaml create mode 100644 evals/scenarios/spec-writing-blind-spot.yaml create mode 100644 evals/scenarios/triggering-dispatching-parallel-agents.yaml create mode 100644 evals/scenarios/triggering-executing-plans.yaml create mode 100644 evals/scenarios/triggering-requesting-code-review.yaml create mode 100644 evals/scenarios/triggering-systematic-debugging.yaml create mode 100644 evals/scenarios/triggering-test-driven-development.yaml create mode 100644 evals/scenarios/triggering-writing-plans.yaml create mode 100644 evals/scenarios/worktree-already-inside-spec-aware.yaml create mode 100644 evals/scenarios/worktree-already-inside.yaml create mode 100644 evals/scenarios/worktree-caller-consent-gate.yaml create mode 100644 evals/scenarios/worktree-codex-app-detached-head-spec-aware.yaml create mode 100644 evals/scenarios/worktree-codex-app-detached-head.yaml create mode 100644 evals/scenarios/worktree-codex-detached-head-spec-aware.yaml create mode 100644 evals/scenarios/worktree-codex-detached-head.yaml create mode 100644 evals/scenarios/worktree-consent-flow.yaml create mode 100644 evals/scenarios/worktree-creation-from-main-spec-aware.yaml create mode 100644 evals/scenarios/worktree-creation-from-main.yaml create mode 100644 evals/scenarios/worktree-creation-under-pressure.yaml create mode 100644 evals/setup_helpers/__init__.py create mode 100644 evals/setup_helpers/base.py create mode 100644 evals/setup_helpers/claim_without_verification.py create mode 100644 evals/setup_helpers/code_review_planted_bugs.py create mode 100644 evals/setup_helpers/sdd_auth_plan.py create mode 100644 evals/setup_helpers/sdd_real_projects.py create mode 100644 evals/setup_helpers/sdd_yagni_plan.py create mode 100644 evals/setup_helpers/spec_review_planted_flaws.py create mode 100644 evals/setup_helpers/spec_targets_wrong_component.py create mode 100644 evals/setup_helpers/spec_targets_wrong_component_with_checkpoint.py create mode 100644 evals/setup_helpers/spec_writing_blind_spot.py create mode 100644 evals/setup_helpers/triggering_executing_plans.py create mode 100644 evals/setup_helpers/wave.py create mode 100644 evals/setup_helpers/worktree.py create mode 100644 evals/setup_helpers/worktree_pressure.py create mode 100644 evals/tests/__init__.py create mode 100644 evals/tests/fixtures/tools_empty.jsonl create mode 100644 evals/tests/fixtures/tools_multi.jsonl create mode 100644 evals/tests/fixtures/tools_ordered.jsonl create mode 100644 evals/tests/fixtures/tools_single.jsonl create mode 100644 evals/tests/test_actor.py create mode 100644 evals/tests/test_assertions.py create mode 100644 evals/tests/test_backend.py create mode 100644 evals/tests/test_cli.py create mode 100644 evals/tests/test_compare.py create mode 100644 evals/tests/test_e2e.py create mode 100644 evals/tests/test_engine.py create mode 100644 evals/tests/test_helpers.py create mode 100644 evals/tests/test_normalizer.py create mode 100644 evals/tests/test_session.py create mode 100644 evals/tests/test_setup.py create mode 100644 evals/tests/test_stats.py create mode 100644 evals/tests/test_sweep.py create mode 100644 evals/tests/test_verifier.py create mode 100644 evals/uv.lock diff --git a/evals/.drill-source-sha b/evals/.drill-source-sha new file mode 100644 index 00000000..94c39314 --- /dev/null +++ b/evals/.drill-source-sha @@ -0,0 +1 @@ +013fcb8b7dbefd6d3fa4653493e5d2ec8e7f985b diff --git a/evals/.gitignore b/evals/.gitignore new file mode 100644 index 00000000..f7f9153d --- /dev/null +++ b/evals/.gitignore @@ -0,0 +1,9 @@ +results/ +__pycache__/ +*.pyc +*.egg-info/ +dist/ +build/ +.venv/ +.env +.claude/ diff --git a/evals/CLAUDE.md b/evals/CLAUDE.md new file mode 100644 index 00000000..f52476a5 --- /dev/null +++ b/evals/CLAUDE.md @@ -0,0 +1,45 @@ +# Drill + +Superpowers skill compliance benchmark. Python 3.11+, managed with uv. + +## Commands + +- **install**: `uv sync --dev` +- **test**: `uv run pytest` +- **test single**: `uv run pytest tests/test_engine.py -x -q` +- **lint**: `uv run ruff check` +- **format**: `uv run ruff format` +- **typecheck**: `uv run ty check` +- **run scenario**: `uv run drill run -b ` +- **sweep**: `uv run drill run --models claude-opus-4-6,claude-opus-4-7 --n 10` +- **compare**: `uv run drill compare ` +- **list**: `uv run drill list` + +## Architecture + +- `drill/engine.py` — Tmux session orchestration. Creates workdir, runs setup helpers, drives actor/agent turns, collects results. +- `drill/actor.py` — Sonnet 4.6 LLM simulating a user. Reads turn intents from scenario YAML and generates realistic prompts. +- `drill/verifier.py` — Sonnet 4.6 LLM evaluating session transcript + filesystem against semantic criteria. +- `drill/assertions.py` — Deterministic post-session checks. Runs shell commands from `verify.assertions` in the results dir. +- `drill/sweep.py` — Multi-backend, N-repetition orchestrator. Wraps Engine with try/except per run, writes run-group.json manifest. +- `drill/compare.py` — Loads results, computes pass rates and Wilson CIs, formats comparison tables. +- `drill/stats.py` — Wilson score confidence interval for pass rate estimation at small N. +- `scenarios/*.yaml` — Scenario definitions (setup, turns, limits, verify). +- `setup_helpers/*.py` — Repo fixture creators. Each creates a git repo with specific conditions. +- `backends/*.yaml` — Per-backend CLI config (args, env, idle patterns, shutdown commands). +- `bin/` — Assertion helper scripts: `tool-called`, `tool-not-called`, `tool-count`, `tool-before`, `tool-arg-match`. Run against `tool_calls.jsonl` in results dir. + +## Conventions + +- Setup helpers take `workdir: Path` and mutate the filesystem. Register in `setup_helpers/__init__.py`. +- Scenarios use `user_posture: naive` (no skill names) or `spec-aware` (can name skills). +- Verify criteria are semantic (LLM-evaluated). Verify assertions are deterministic (exit code 0 = pass). +- Assertions run in the results dir with `$DRILL_WORKDIR` pointing to the scenario workdir and `bin/` on PATH. +- Backend YAMLs are fully self-contained — no override/alias system. + +## Required env + +``` +SUPERPOWERS_ROOT=/path/to/superpowers +ANTHROPIC_API_KEY=sk-... +``` diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 00000000..a069bd12 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,104 @@ +# Drill + +Superpowers skill compliance benchmark. Drives AI coding agents through +tmux sessions and evaluates whether they follow superpowers workflows +correctly. + +## How it works + +1. **Setup** — a helper creates a git repo with specific conditions (worktree state, plan files, code fixtures) +2. **Actor** — a Sonnet 4.6 LLM plays the user, following turn intents from the scenario YAML +3. **Agent** — the backend under test (Claude Code, Codex, Gemini CLI) runs in a real tmux session +4. **Verifier** — a Sonnet 4.6 LLM evaluates the session transcript + filesystem against criteria +5. **Assertions** — deterministic checks (tool-called, tool-count, shell commands) run post-session + +## Setup + +```bash +uv sync --dev +``` + +Required environment: +```bash +export SUPERPOWERS_ROOT=/path/to/superpowers +export ANTHROPIC_API_KEY=sk-... +``` + +## Usage + +```bash +# Run a single scenario on a single backend +uv run drill run worktree-creation-from-main -b claude + +# Run with N repetitions +uv run drill run pattern-match-trap -b claude-opus-4-6 --n 5 + +# Sweep across multiple backends +uv run drill run pattern-match-trap --models claude-opus-4-6,claude-opus-4-7 --n 10 + +# Compare results +uv run drill compare pattern-match-trap + +# List available scenarios +uv run drill list +``` + +## Scenarios + +| Category | Scenarios | Tests | +|----------|-----------|-------| +| Worktree | 8 scenarios (creation, detection, consent, detached HEAD) | Skill compliance for `using-git-worktrees` | +| Wave decomposition | 5 scenarios (naive, spec-aware, false overlap, dependency chain, conflict surface) | Plan → waves decomposition quality | +| Wave execution | 3 scenarios (minimal, full, task failure) | End-to-end wave execution + failure escalation | +| Pattern-match trap | 1 scenario | Investigation depth gap between 4.6 and 4.7 (PRI-1270) | + +## Backends + +| Backend | CLI | Model | +|---------|-----|-------| +| `claude` | Claude Code | opus-4-7 (default) | +| `claude-opus-4-6` | Claude Code | opus-4-6 | +| `claude-opus-4-7` | Claude Code | opus-4-7 | +| `claude-opus-4-6-1m` | Claude Code | opus-4-6 (1M context) | +| `claude-opus-4-7-1m` | Claude Code | opus-4-7 (1M context) | +| `codex` | Codex CLI | — | +| `gemini` | Gemini CLI | — | + +## Project structure + +``` +drill/ # Core engine + cli.py # Click CLI (run, compare, list) + engine.py # Tmux session orchestration + actor.py # User-simulator LLM + verifier.py # Criteria evaluator LLM + assertions.py # Deterministic post-session assertions + compare.py # Result loading and cross-backend comparison + sweep.py # Multi-backend N-rep orchestrator + stats.py # Wilson score confidence intervals +scenarios/ # YAML scenario definitions +setup_helpers/ # Repo fixture creators +backends/ # Per-backend YAML configs +bin/ # Assertion helper scripts (tool-called, tool-count, etc.) +prompts/ # Actor and verifier system prompts +fixtures/ # Static template repos +tests/ # pytest suite (122 tests) +docs/ # Design spec and manual testing guide +``` + +## Tests + +```bash +uv run pytest +uv run ruff check +uv run ty check +``` + +## Writing a new scenario + +1. Create a setup helper in `setup_helpers/` if you need a custom fixture +2. Register it in `setup_helpers/__init__.py` +3. Create `scenarios/your-scenario.yaml` with setup, turns, limits, and verify sections +4. Run it: `uv run drill run your-scenario -b claude` + +See [docs/design.md](docs/design.md) for the full design spec. diff --git a/evals/backends/claude-haiku.yaml b/evals/backends/claude-haiku.yaml new file mode 100644 index 00000000..28d431c6 --- /dev/null +++ b/evals/backends/claude-haiku.yaml @@ -0,0 +1,26 @@ +name: claude-haiku +cli: claude +args: + - "--dangerously-skip-permissions" + - "--plugin-dir" + - "${SUPERPOWERS_ROOT}" + - "--model" + - "haiku" +required_env: + - ANTHROPIC_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: [] + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 3 + ready_pattern: "^❯|^\\$|Human:|Enter to confirm" +busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]" +max_busy_seconds: 1800 +startup_timeout: 60 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.claude/projects/**/session-*.jsonl" diff --git a/evals/backends/claude-opus-4-6-1m.yaml b/evals/backends/claude-opus-4-6-1m.yaml new file mode 100644 index 00000000..4c6a462d --- /dev/null +++ b/evals/backends/claude-opus-4-6-1m.yaml @@ -0,0 +1,26 @@ +name: claude-opus-4-6-1m +cli: claude +args: + - "--dangerously-skip-permissions" + - "--plugin-dir" + - "${SUPERPOWERS_ROOT}" + - "--model" + - "claude-opus-4-6[1m]" +required_env: + - ANTHROPIC_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: [] + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 3 + ready_pattern: "^❯|^\\$|Human:|Enter to confirm" +busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]" +max_busy_seconds: 1800 +startup_timeout: 60 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.claude/projects/**/session-*.jsonl" diff --git a/evals/backends/claude-opus-4-6.yaml b/evals/backends/claude-opus-4-6.yaml new file mode 100644 index 00000000..a4e3ee03 --- /dev/null +++ b/evals/backends/claude-opus-4-6.yaml @@ -0,0 +1,26 @@ +name: claude-opus-4-6 +cli: claude +args: + - "--dangerously-skip-permissions" + - "--plugin-dir" + - "${SUPERPOWERS_ROOT}" + - "--model" + - "claude-opus-4-6" +required_env: + - ANTHROPIC_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: [] + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 3 + ready_pattern: "^❯|^\\$|Human:|Enter to confirm" +busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]" +max_busy_seconds: 1800 +startup_timeout: 60 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.claude/projects/**/session-*.jsonl" diff --git a/evals/backends/claude-opus-4-7-1m.yaml b/evals/backends/claude-opus-4-7-1m.yaml new file mode 100644 index 00000000..8fe0f3fb --- /dev/null +++ b/evals/backends/claude-opus-4-7-1m.yaml @@ -0,0 +1,26 @@ +name: claude-opus-4-7-1m +cli: claude +args: + - "--dangerously-skip-permissions" + - "--plugin-dir" + - "${SUPERPOWERS_ROOT}" + - "--model" + - "claude-opus-4-7[1m]" +required_env: + - ANTHROPIC_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: [] + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 3 + ready_pattern: "^❯|^\\$|Human:|Enter to confirm" +busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]" +max_busy_seconds: 1800 +startup_timeout: 60 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.claude/projects/**/session-*.jsonl" diff --git a/evals/backends/claude-opus-4-7.yaml b/evals/backends/claude-opus-4-7.yaml new file mode 100644 index 00000000..ac7b3f1d --- /dev/null +++ b/evals/backends/claude-opus-4-7.yaml @@ -0,0 +1,26 @@ +name: claude-opus-4-7 +cli: claude +args: + - "--dangerously-skip-permissions" + - "--plugin-dir" + - "${SUPERPOWERS_ROOT}" + - "--model" + - "claude-opus-4-7" +required_env: + - ANTHROPIC_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: [] + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 3 + ready_pattern: "^❯|^\\$|Human:|Enter to confirm" +busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]" +max_busy_seconds: 1800 +startup_timeout: 60 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.claude/projects/**/session-*.jsonl" diff --git a/evals/backends/claude.yaml b/evals/backends/claude.yaml new file mode 100644 index 00000000..47ba96af --- /dev/null +++ b/evals/backends/claude.yaml @@ -0,0 +1,32 @@ +name: claude +cli: claude +args: + - "--dangerously-skip-permissions" + - "--plugin-dir" + - "${SUPERPOWERS_ROOT}" + - "--model" + - "opus" +required_env: + - ANTHROPIC_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: [] + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 3 + ready_pattern: "^❯|^\\$|Human:|Enter to confirm" +# Matches when Claude is actively working — spinners, "Thinking", time counter, +# or "esc to cancel". Engine extends its wait deadline when any of these match +# so the Actor doesn't interrupt long-running subagent work (e.g., wave execution). +busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]" +# Maximum total seconds the engine will extend the deadline across all busy +# detections during a single _wait_for_ready call. Wave execution can take +# 10-20 minutes per wave, so 30 minutes gives plenty of headroom. +max_busy_seconds: 1800 +startup_timeout: 60 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.claude/projects/**/session-*.jsonl" diff --git a/evals/backends/codex.yaml b/evals/backends/codex.yaml new file mode 100644 index 00000000..ae09f046 --- /dev/null +++ b/evals/backends/codex.yaml @@ -0,0 +1,21 @@ +name: codex +cli: codex +args: + - "--dangerously-bypass-approvals-and-sandbox" +required_env: + - OPENAI_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: + - symlink_superpowers + post_run: [] +shutdown: "<>" +idle: + quiescence_seconds: 5 + ready_pattern: "^›|codex>|^>" +startup_timeout: 60 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.codex/sessions/rollout-*.jsonl" diff --git a/evals/backends/gemini.yaml b/evals/backends/gemini.yaml new file mode 100644 index 00000000..eac0bd41 --- /dev/null +++ b/evals/backends/gemini.yaml @@ -0,0 +1,24 @@ +name: gemini +cli: gemini +args: + - "--yolo" + - "-m" + - "gemini-2.5-flash" +required_env: + - SUPERPOWERS_ROOT +hooks: + pre_run: + - link_gemini_extension + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 5 + ready_pattern: "Type your message|^\\s*>" +busy_pattern: "Thinking\\.\\.\\.|Executing" +startup_timeout: 60 +turn_timeout: 300 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.gemini/tmp/*/chats/session-*.json" diff --git a/evals/bin/skill-before-tool-match b/evals/bin/skill-before-tool-match new file mode 100755 index 00000000..0e4656bc --- /dev/null +++ b/evals/bin/skill-before-tool-match @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Verify a specific Skill was invoked before any Bash call whose command matches a regex. +# +# Usage: skill-before-tool-match +# Example: skill-before-tool-match superpowers:verification-before-completion 'git[[:space:]]+commit' +# +# Semantics: +# - If no Bash call matches the regex, PASS (vacuously — the gated event never occurred). +# - If Bash matches but Skill with that name never appeared earlier, FAIL. +# - If both appeared and Skill came first, PASS. +# - If Skill never appeared but Bash matched, FAIL. +set -euo pipefail +command -v jq >/dev/null || { echo "jq required"; exit 127; } + +SKILL_NAME="$1" +BASH_REGEX="$2" +FILE="tool_calls.jsonl" + +if [ ! -s "$FILE" ]; then + echo "FAIL: tool_calls.jsonl missing or empty" + exit 1 +fi + +# First index where Skill(skill=SKILL_NAME) appears (0-based). +SKILL_IDX=$( + jq -s --arg name "$SKILL_NAME" \ + 'to_entries | map(select(.value.tool == "Skill" and (.value.args.skill // "") == $name)) | first | (.key // -1)' \ + "$FILE" +) + +# First index where Bash(command =~ BASH_REGEX) appears. +BASH_IDX=$( + jq -s --arg re "$BASH_REGEX" \ + 'to_entries | map(select(.value.tool == "Bash" and ((.value.args.command // "") | test($re)))) | first | (.key // -1)' \ + "$FILE" +) + +if [ "$BASH_IDX" -lt 0 ]; then + echo "PASS: no Bash call matched /$BASH_REGEX/ — assertion is vacuous" + exit 0 +fi + +if [ "$SKILL_IDX" -lt 0 ]; then + echo "FAIL: Bash /$BASH_REGEX/ fired at line $((BASH_IDX + 1)) but Skill($SKILL_NAME) never fired" + exit 1 +fi + +if [ "$SKILL_IDX" -lt "$BASH_IDX" ]; then + echo "PASS: Skill($SKILL_NAME) at line $((SKILL_IDX + 1)) before Bash /$BASH_REGEX/ at line $((BASH_IDX + 1))" + exit 0 +else + echo "FAIL: Skill($SKILL_NAME) at line $((SKILL_IDX + 1)) fired after Bash /$BASH_REGEX/ at line $((BASH_IDX + 1))" + exit 1 +fi diff --git a/evals/bin/skill-called b/evals/bin/skill-called new file mode 100755 index 00000000..cf4fc50a --- /dev/null +++ b/evals/bin/skill-called @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Verify a specific superpowers Skill was invoked at least once. +# +# Usage: skill-called +# Example: skill-called superpowers:systematic-debugging +# +# Wraps the common case of `tool-arg-match Skill '.skill == ""'` so +# scenario YAML doesn't have to embed jq quoting. +set -euo pipefail +command -v jq >/dev/null || { echo "jq required"; exit 127; } + +SKILL_NAME="$1" +FILE="tool_calls.jsonl" + +if [ ! -s "$FILE" ]; then + echo "FAIL: tool_calls.jsonl missing or empty" + exit 1 +fi + +COUNT=$( + jq -s --arg name "$SKILL_NAME" \ + '[.[] | select(.tool == "Skill" and (.args.skill // "") == $name)] | length' \ + "$FILE" +) + +if [ "$COUNT" -gt 0 ]; then + echo "PASS: Skill($SKILL_NAME) called $COUNT time(s)" + exit 0 +else + echo "FAIL: Skill($SKILL_NAME) never called" + exit 1 +fi diff --git a/evals/bin/tool-arg-match b/evals/bin/tool-arg-match new file mode 100755 index 00000000..0e5abe52 --- /dev/null +++ b/evals/bin/tool-arg-match @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +command -v jq >/dev/null || { echo "jq required"; exit 127; } + +TOOL="$1" +FILTER="$2" +FILE="tool_calls.jsonl" + +MATCHES=$(jq -s "[.[] | select(.tool == \"$TOOL\") | select(.args | $FILTER)] | length" "$FILE" 2>/dev/null || echo 0) + +if [ "$MATCHES" -gt 0 ]; then + echo "PASS: $TOOL has $MATCHES call(s) matching filter" + exit 0 +else + echo "FAIL: no $TOOL calls match filter: $FILTER" + exit 1 +fi diff --git a/evals/bin/tool-before b/evals/bin/tool-before new file mode 100755 index 00000000..c4c59d48 --- /dev/null +++ b/evals/bin/tool-before @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +command -v jq >/dev/null || { echo "jq required"; exit 127; } + +TOOL_A="$1" +TOOL_B="$2" +FILE="tool_calls.jsonl" + +IDX_A=$(jq -s 'to_entries | map(select(.value.tool == "'"$TOOL_A"'")) | first // empty | .key' "$FILE" 2>/dev/null) +IDX_B=$(jq -s 'to_entries | map(select(.value.tool == "'"$TOOL_B"'")) | first // empty | .key' "$FILE" 2>/dev/null) + +if [ -z "$IDX_A" ] || [ "$IDX_A" = "null" ]; then + echo "FAIL: $TOOL_A never called" + exit 1 +fi + +if [ -z "$IDX_B" ] || [ "$IDX_B" = "null" ]; then + echo "FAIL: $TOOL_B never called" + exit 1 +fi + +if [ "$IDX_A" -lt "$IDX_B" ]; then + echo "PASS: $TOOL_A (line $((IDX_A + 1))) before $TOOL_B (line $((IDX_B + 1)))" + exit 0 +else + echo "FAIL: $TOOL_A at line $((IDX_A + 1)) occurred after $TOOL_B at line $((IDX_B + 1))" + exit 1 +fi diff --git a/evals/bin/tool-called b/evals/bin/tool-called new file mode 100755 index 00000000..5094e6f8 --- /dev/null +++ b/evals/bin/tool-called @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +command -v jq >/dev/null || { echo "jq required"; exit 127; } + +TOOL="$1" +FILE="tool_calls.jsonl" + +COUNT=$(jq -s "[.[] | select(.tool == \"$TOOL\")] | length" "$FILE" 2>/dev/null || echo 0) + +if [ "$COUNT" -gt 0 ]; then + echo "PASS: $TOOL called $COUNT time(s)" + exit 0 +else + echo "FAIL: $TOOL never called" + exit 1 +fi diff --git a/evals/bin/tool-count b/evals/bin/tool-count new file mode 100755 index 00000000..3b972d9a --- /dev/null +++ b/evals/bin/tool-count @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +command -v jq >/dev/null || { echo "jq required"; exit 127; } + +TOOL="$1" +OP="$2" +EXPECTED="$3" +FILE="tool_calls.jsonl" + +COUNT=$(jq -s "[.[] | select(.tool == \"$TOOL\")] | length" "$FILE" 2>/dev/null || echo 0) + +case "$OP" in + eq) TEST=$(( COUNT == EXPECTED )) ;; + gt) TEST=$(( COUNT > EXPECTED )) ;; + gte) TEST=$(( COUNT >= EXPECTED )) ;; + lt) TEST=$(( COUNT < EXPECTED )) ;; + lte) TEST=$(( COUNT <= EXPECTED )) ;; + *) echo "Unknown operator: $OP (expected: eq, gt, gte, lt, lte)"; exit 2 ;; +esac + +if [ "$TEST" -eq 1 ]; then + echo "PASS: $TOOL called $COUNT time(s) ($OP $EXPECTED)" + exit 0 +else + echo "FAIL: $TOOL called $COUNT time(s) (expected $OP $EXPECTED)" + exit 1 +fi diff --git a/evals/bin/tool-match-before-tool-match b/evals/bin/tool-match-before-tool-match new file mode 100755 index 00000000..c765f3c4 --- /dev/null +++ b/evals/bin/tool-match-before-tool-match @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Verify any Bash call with command matching a regex fires before any other Bash call +# matching a second regex. +# +# Usage: tool-match-before-tool-match +# Example: tool-match-before-tool-match Bash 'pytest' Bash 'git[[:space:]]+commit' +# +# Semantics: +# - If no call matches the "later" regex, PASS (vacuously — the gated event never happened). +# - If the "later" call fires but no "earlier" call preceded it, FAIL. +set -euo pipefail +command -v jq >/dev/null || { echo "jq required"; exit 127; } + +TOOL_A="$1" +REGEX_A="$2" +TOOL_B="$3" +REGEX_B="$4" +FILE="tool_calls.jsonl" + +if [ ! -s "$FILE" ]; then + echo "FAIL: tool_calls.jsonl missing or empty" + exit 1 +fi + +IDX_A=$( + jq -s --arg tool "$TOOL_A" --arg re "$REGEX_A" \ + 'to_entries | map(select(.value.tool == $tool and ((.value.args.command // "") | test($re)))) | first | (.key // -1)' \ + "$FILE" +) + +IDX_B=$( + jq -s --arg tool "$TOOL_B" --arg re "$REGEX_B" \ + 'to_entries | map(select(.value.tool == $tool and ((.value.args.command // "") | test($re)))) | first | (.key // -1)' \ + "$FILE" +) + +if [ "$IDX_B" -lt 0 ]; then + echo "PASS: no $TOOL_B call matched /$REGEX_B/ — assertion is vacuous" + exit 0 +fi + +if [ "$IDX_A" -lt 0 ]; then + echo "FAIL: $TOOL_B /$REGEX_B/ fired at line $((IDX_B + 1)) but no $TOOL_A /$REGEX_A/ preceded it" + exit 1 +fi + +if [ "$IDX_A" -lt "$IDX_B" ]; then + echo "PASS: $TOOL_A /$REGEX_A/ at line $((IDX_A + 1)) before $TOOL_B /$REGEX_B/ at line $((IDX_B + 1))" + exit 0 +else + echo "FAIL: $TOOL_A /$REGEX_A/ at line $((IDX_A + 1)) fired after $TOOL_B /$REGEX_B/ at line $((IDX_B + 1))" + exit 1 +fi diff --git a/evals/bin/tool-not-called b/evals/bin/tool-not-called new file mode 100755 index 00000000..3995f90b --- /dev/null +++ b/evals/bin/tool-not-called @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +command -v jq >/dev/null || { echo "jq required"; exit 127; } + +TOOL="$1" +FILE="tool_calls.jsonl" + +COUNT=$(jq -s "[.[] | select(.tool == \"$TOOL\")] | length" "$FILE" 2>/dev/null || echo 0) + +if [ "$COUNT" -eq 0 ]; then + echo "PASS: $TOOL never called" + exit 0 +else + echo "FAIL: $TOOL called $COUNT time(s) (expected 0)" + exit 1 +fi diff --git a/evals/docs/design.md b/evals/docs/design.md new file mode 100644 index 00000000..9624d1a4 --- /dev/null +++ b/evals/docs/design.md @@ -0,0 +1,418 @@ +# Drill: Superpowers Skill Compliance Benchmark + +**Date:** 2026-04-07 +**Ticket:** [PRI-1040](https://linear.app/prime-radiant/issue/PRI-1040) +**Status:** Design + +## Thesis + +The value of superpowers depends on whether skills are reliably followed by *any* coding agent — not just Claude Code. Drill tests whether agents actually fire skills, follow workflows, and use native tooling when available. It is a **compliance benchmark**, not a coding ability benchmark. + +If a well-written skill produces consistent behavior across Claude Code and Codex, the agent-agnostic coordination layer is working. If agents diverge, Drill tells you exactly where and why. + +## What Drill Tests + +- Do agents invoke superpowers skills when they should? +- Do they follow multi-step workflows (detect → consent → create) in the right order? +- Do they use native tools (EnterWorktree, structured session logs) vs. raw shell commands? +- Where do agents diverge, and what does that tell us about skill format? + +The first scenarios target **PRI-974 (worktree rototill)** — the area with the most cross-agent fragmentation today. + +## Architecture + +Three layers, each with a single responsibility: + +``` +┌─────────────────────────────────────────┐ +│ CLI (click) │ +│ run / compare / list │ +├─────────────────────────────────────────┤ +│ Engine │ +│ ┌───────────┐ ┌───────┐ ┌──────────┐ │ +│ │ Session │ │ Actor │ │ Verifier │ │ +│ │ (tmux) │ │ (LLM) │ │ (LLM) │ │ +│ └───────────┘ └───────┘ └──────────┘ │ +├─────────────────────────────────────────┤ +│ Backends │ +│ claude / codex / (future: gemini) │ +├─────────────────────────────────────────┤ +│ Setup │ +│ template repo + helpers + assertions │ +└─────────────────────────────────────────┘ +``` + +- **CLI** — `drill run --backend claude`, `drill compare `, `drill list` +- **Engine** — Orchestrates the full run lifecycle (setup → session → actor loop → collect → verify → results) +- **Session** — tmux lifecycle: create session, send-keys, capture-pane, kill session +- **Actor** — Sonnet with rolling context. Gets all scenario intents as a goal stack + terminal screens. Outputs what to type next, or `<>`/`<>`. +- **Verifier** — Sonnet (near-zero temperature) with full session log + filesystem state + tool call log + criteria list. Returns per-criterion pass/fail with cited evidence + freeform observations. +- **Backends** — Each backend knows: CLI command, auto-approve flags, plugin loading, idle detection, shutdown command, session log location. +- **Setup** — Clone template repo → run backend pre_run hooks → run scenario helpers → run setup assertions → fail fast if invariants violated. + +## Engine Flow + +``` +1. LOAD + - Parse scenario YAML + - Parse backend YAML + - Validate required env vars (fail fast) + +2. SETUP + - Clone template repo to temp dir + - Run backend pre_run hooks (codex symlink, etc.) + - Run scenario setup helpers + - Run setup assertions → abort if any fail + +3. SESSION + - Create tmux session (backend-specific terminal dimensions) + - Launch agent CLI in tmux pane + - Wait for startup ready pattern + +4. ACTOR LOOP + - For each turn (up to max_turns): + a. Wait for idle (quiescence + ready pattern) + b. Capture terminal pane → append to rolling context + c. Send to Actor LLM: system prompt + rolling context + ALL intents + user_posture + d. Actor responds with text to type, <>, or <> + e. If <> or <> → break + f. Send keystrokes via tmux send-keys + g. Per-turn timeout → <> if exceeded + - Special keys via <> convention (e.g., <>) + +5. COLLECT + - Capture final terminal state + - Send shutdown command (backend-specific: /exit, Ctrl-D, etc.) + - Wait for process exit (with timeout) + - Snapshot filesystem (file tree, git state, worktree list) + - Collect backend session logs → tool_calls.jsonl + - Kill tmux session (cleanup if process didn't exit cleanly) + +6. VERIFY + - Send to Verifier LLM: session.log + filesystem.json + tool_calls.jsonl + criteria + - Verifier receives criteria but NOT actor intents (reduces confirmation bias) + - Verifier returns per-criterion pass/fail with evidence + rationale + observations + - Output as structured JSON (verdict.json) + +7. RESULTS + - Write to results//// + - Print summary to stdout +``` + +## Backend Abstraction + +Each backend is a YAML config. Backends own: CLI invocation, idle detection, shutdown, session log collection, and pre/post-run hooks. + +```yaml +# backends/claude.yaml +name: claude +cli: claude +args: + - "--dangerously-skip-permissions" + - "--plugin-dir" + - "${SUPERPOWERS_ROOT}" +required_env: + - ANTHROPIC_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: [] # no repo setup needed; plugin loaded via --plugin-dir + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 3 + ready_pattern: "^❯|^\\$|Human:" +startup_timeout: 30 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.claude/projects/**/session-*.jsonl" + match_by: timestamp +``` + +```yaml +# backends/codex.yaml +name: codex +cli: codex +args: + - "--dangerously-bypass-approvals-and-sandbox" +required_env: + - OPENAI_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: + - symlink_superpowers # creates .agents/skills/superpowers symlink in test repo + post_run: [] +shutdown: "<>" +idle: + quiescence_seconds: 5 + ready_pattern: "codex>|^>" +startup_timeout: 30 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.codex/sessions/rollout-*.jsonl" + match_by: timestamp +``` + +New backends = new YAML file. Backend variants (e.g., `codex-workspace-write.yaml`) are just copies with different args — no inheritance system needed. Scenarios reference backends by name. + +## Scenario Format + +Scenarios are YAML. They describe *what* to test, not *how* each backend works. + +```yaml +scenario: worktree-creation-from-main +description: "Agent creates an isolated worktree from main branch" +user_posture: naive # or spec-aware + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + - "git worktree list | wc -l | grep 1" + +turns: + - intent: > + Ask the agent to create an isolated workspace + for building a login feature. + - intent: "Confirm consent if the agent asks." + +limits: + max_turns: 20 + turn_timeout: 120 # seconds per turn + +verify: + criteria: + - "Agent detected it was on main, not in an existing worktree" + - "Agent asked for consent before creating the worktree" + - "A worktree or isolated workspace now exists with a feature branch" + - "Agent used the most appropriate tool available for its platform to create the worktree" + observe: true # verifier can add freeform observations +``` + +### User Posture + +Each scenario has a `user_posture` field: + +- **naive** — User describes what they want in plain language. Tests whether the agent's superpowers skills fire without hand-holding. +- **spec-aware** — User references specific skills or conventions by name. Tests whether the agent follows the spec when pointed at it. + +The delta between naive and spec-aware results for the same scenario is the most interesting product signal. A small delta means strong conveyance. A large delta means the skill format needs work. + +### Turn Intents + +Intents are a **priority-ordered goal stack**, not a rigid script. The actor receives all intents and decides which one applies to the current terminal state. Some intents are conditional ("Confirm consent if the agent asks") and may never fire. + +## Setup + +### Template Repo + +A real git repo checked into `fixtures/template-repo/`. Cloned to a temp directory per run. Covers the 80% common case. + +Contents: +- `package.json` — minimal Node project metadata (name, version) +- `src/index.js` — simple entry point (~10 lines) +- `src/utils.js` — helper module (~10 lines) +- `README.md` — basic project description +- 3-4 commits on `main` with realistic messages (e.g., "initial commit", "add utils module", "update readme") +- No existing worktrees, branches, or tags beyond `main` + +This is intentionally minimal — just enough for agents to recognize it as a real project. Scenario-specific state (extra branches, worktrees, detached HEAD) is added by setup helpers. + +### Setup Helpers + +Python functions in `setup_helpers/` that modify the cloned repo for specific scenarios: + +- `create_base_repo(workdir)` — Clone template, verify structure +- `add_worktree(workdir, branch, path)` — Create an existing worktree (for "already inside" scenarios) +- `detach_head(workdir)` — Simulate Codex App detached HEAD state +- `symlink_superpowers(workdir)` — Create `.agents/skills/superpowers` symlink (codex pre_run hook) + +### Setup Assertions + +Run after all setup completes, before the agent launches. If any fail, the scenario aborts with a clear "setup invariant violated" error — not a mysterious agent failure 10 turns later. + +## Plugin Loading + +Each backend loads superpowers differently. The harness manages this per-run with no global config mutation: + +| Backend | Mechanism | Harness action | +|---------|-----------|----------------| +| Claude Code | `--plugin-dir` CLI flag | Pass flag pointing at superpowers checkout | +| Codex | `.agents/skills/` in repo | Backend pre_run hook creates symlink | + +This means Drill can test draft skill changes by pointing at a branch checkout of superpowers. + +## Post-Session Tool Call Collection + +Both backends write structured session logs that record every tool invocation: + +| Backend | Log location | Format | +|---------|-------------|--------| +| Claude Code | `~/.claude/projects/**/session-*.jsonl` | JSONL with tool names + args | +| Codex | `~/.codex/sessions/rollout-*.jsonl` | JSONL with `LocalShellCall`, `FunctionCall`, etc. | + +The harness snapshots each backend's log directory before the session starts. After shutdown, it diffs the directory to find only files created during the run — no timestamp matching needed, no cross-contamination from concurrent sessions or prior runs. + +Collected logs are normalized into a common `tool_calls.jsonl` format before the verifier sees them: + +```json +{"tool": "EnterWorktree", "args": {"branch": "add-login"}, "source": "native"} +{"tool": "Bash", "args": {"command": "git worktree add ..."}, "source": "shell"} +``` + +Each backend defines a normalizer function that maps its native log format (Claude Code's tool call entries, Codex's `ResponseItem` records) into this common schema. The verifier never sees raw backend-specific logs. + +## Actor & Verifier LLM Design + +### Actor + +- **Model:** Sonnet +- **Temperature:** 0.7 (realistic user variation) +- **Context:** Rolling (full conversation history). Sessions are short enough (~5-20 turns) that token cost is not a concern. +- **Input:** System prompt + rolling terminal captures + all intents + user_posture +- **Output:** Structured JSON via Anthropic SDK tool_use: `{"action": "type", "text": "..."}`, `{"action": "done"}`, `{"action": "stuck"}`, or `{"action": "key", "key": "ctrl-c"}`. The harness parses this and sends keystrokes — no free-text sanitization needed. +- **Prompt:** Versioned template at `prompts/actor.md` + +### Verifier + +- **Model:** Sonnet +- **Temperature:** Near-zero (deterministic judgment) +- **Input:** session.log + filesystem.json + tool_calls.jsonl + criteria list. Does NOT receive actor intents or scenario narrative (reduces confirmation bias). +- **Output:** Structured JSON with per-criterion verdict/evidence/rationale + observations +- **Prompt:** Versioned template at `prompts/verifier.md` + +## Results & Compare + +### Results Structure + +``` +results/ + / + / + / + session.log # raw tmux capture + filesystem.json # post-run git/file state snapshot + tool_calls.jsonl # collected from backend session logs + verdict.json # verifier output + meta.json # run metadata (backend, duration, turns, model versions) +``` + +### Compare Command + +`drill compare` reads existing results from prior `drill run` invocations. It does not run backends itself — run each backend separately first, then compare. + +``` +$ drill run worktree-creation-from-main --backend claude +$ drill run worktree-creation-from-main --backend codex +$ drill compare worktree-creation-from-main + +Scenario: worktree-creation-from-main (naive posture) + +Summary: +┌──────────┬────────┬───────┬───────┐ +│ Backend │ Result │ Score │ Turns │ +├──────────┼────────┼───────┼───────┤ +│ claude │ PASS │ 4/4 │ 6 │ +│ codex │ FAIL │ 2/4 │ 12 │ +└──────────┴────────┴───────┴───────┘ + +Detail: +┌────────────────────────────────┬────────┬────────┐ +│ Criterion │ claude │ codex │ +├────────────────────────────────┼────────┼────────┤ +│ Detected on main │ ✓ │ ✓ │ +│ Asked consent │ ✓ │ ✗ │ +│ Worktree exists │ ✓ │ ✓ │ +│ Used native tools │ ✓ │ ✗ │ +└────────────────────────────────┴────────┴────────┘ + +Observations: + claude: "Agent cited the using-git-worktrees skill by name" + codex: "Agent created worktree but skipped consent step entirely" +``` + +## Project Structure + +``` +drill/ +├── drill/ +│ ├── __init__.py +│ ├── cli.py # click CLI: run, compare, list +│ ├── engine.py # orchestrates the full run lifecycle +│ ├── session.py # tmux session management +│ ├── actor.py # actor LLM calls +│ ├── verifier.py # verifier LLM calls +│ ├── setup.py # template repo cloning, helpers, assertions +│ └── backend.py # loads backend YAML, builds commands +├── backends/ +│ ├── claude.yaml +│ └── codex.yaml +├── prompts/ +│ ├── actor.md +│ └── verifier.md +├── scenarios/ +│ ├── worktree-creation-from-main.yaml +│ ├── worktree-already-inside.yaml +│ ├── worktree-codex-detached-head.yaml +│ └── worktree-consent-flow.yaml +├── fixtures/ +│ └── template-repo/ # base git repo, cloned per run +├── setup_helpers/ +│ ├── __init__.py +│ ├── base.py # create_base_repo, common git ops +│ └── worktree.py # add_worktree, detach_head, etc. +├── results/ # gitignored, populated by runs +├── pyproject.toml # package metadata + [project.scripts] entry point +└── README.md +``` + +## Phase 1 Scope + +- Claude Code + Codex backends +- 4 PRI-974 worktree scenarios (creation, already-inside, detached-head, consent) +- Both user postures (naive + spec-aware) per scenario +- Template repo + setup helpers + assertions +- Actor + verifier with prompts +- `drill run` and `drill compare` commands +- Results storage + +## Phase 2 (Future) + +- Gemini CLI backend +- Backend variants (e.g., `codex-workspace-write.yaml` for sandbox mode testing) +- Verifier flakiness mitigation (3x voting, agreement tracking) +- Cost tracking and token usage reporting +- Docker isolation for reproducibility +- CI integration +- Scenarios beyond worktrees (stacked PRs, git-spice, brainstorming) + +## Installation + +```bash +pip install -e . # installs 'drill' console script +``` + +Requires `tmux` installed as a system dependency. + +## Dependencies + +- Python 3.11+ +- `click` — CLI framework +- `pyyaml` — scenario and backend config parsing +- `anthropic` — Anthropic Python SDK for actor/verifier LLM calls (structured tool_use output) +- `jinja2` — prompt template rendering +- `pydantic` — verdict schema validation (retry on malformed verifier output) +- `tmux` — session driving (system dependency) + +## Non-Goals + +- Not a coding ability benchmark (SWE-bench covers that) +- Not an LLM evaluation framework (promptfoo covers that) +- Not a generic terminal automation tool (Terminal-Bench covers that) +- No CI in phase 1 +- No Docker in phase 1 diff --git a/evals/docs/manual-testing.md b/evals/docs/manual-testing.md new file mode 100644 index 00000000..1f1ebf75 --- /dev/null +++ b/evals/docs/manual-testing.md @@ -0,0 +1,93 @@ +# Manual Testing (Codex App) + +Some scenarios cannot run automatically because drill has no harness adapter for the target — the Codex App desktop client has no CLI or tmux entry point the way `claude` and `codex` do. These scenarios are marked `manual: true` in their YAML and use a human-in-the-loop protocol. + +## Protocol + +Three phases. The agent never runs Codex App directly. The tester never writes a verdict by hand. + +1. **Agent prepares the handoff** — reads the scenario file, renders setup + turn intents into something a human can act on, hands the package to the tester. +2. **Tester executes** — sets up the repo fixture, opens Codex App, pastes the prompt, handles any follow-ups, copies the transcript + final filesystem state back to the agent. +3. **Agent judges and records** — evaluates the transcript against `verify.criteria`, writes a verdict JSON, saves to `results//codex-app/YYYY-MM-DD-manual/verdict.json`. + +## Phase 1: Agent prepares the handoff + +Deliver as one self-contained message to the tester: + +### Fixture state +Exact repo state Codex App should be launched against. Pull from `setup.notes` if present, otherwise translate `setup.helpers` + `setup.assertions` into prose. Include: which repo/directory, branch, whether to expect a worktree vs normal checkout, any required/forbidden files (e.g. `.gitignore` entries). + +### Prompt to paste +Render turn 1's `intent` as a natural first-person message the tester can paste verbatim into Codex App. **Don't leak internal test language** like *"Do NOT say 'create a worktree'"* — that's instruction for the test author, not the end user. Convert it to what a real user would actually type. + +Example: +> Intent: *"Ask the agent to use the worktree skill to get set up for a notifications feature. Do NOT say 'create a worktree' — just reference the skill by name."* +> +> Rendered prompt: *"hey, can you use the worktree skill to get me set up for a notifications feature?"* + +### Follow-up guidance +For each additional turn, give the tester a short decision rule — not a verbatim script. E.g. *"If the agent asks a clarifying question like branch name, answer concisely. If it stops to ask whether you want a worktree at all, tell it you already asked for the skill and it should proceed."* + +### What to capture +Ask the tester to paste back: +- Full agent transcript (messages, tool calls, tool outputs) +- Final filesystem state if criteria depend on it (`git worktree list`, directory tree, branch state) +- Any observations they want on the record + +## Phase 2: Tester executes + +1. Set up the repo fixture per the instructions +2. Open Codex App in that repo +3. Paste the prompt +4. Follow up per the guidance +5. Copy the transcript + filesystem state back to the agent + +## Phase 3: Agent judges and records + +For each criterion in `verify.criteria`, write one entry: + +```json +{ + "criterion": "", + "passed": true | false, + "evidence": "", + "rationale": "" +} +``` + +**Rules:** +- Quote the transcript directly in `evidence`. No paraphrasing. +- If a criterion is genuinely inconclusive from the transcript, mark `passed: false` with `rationale` explaining what was missing. Don't guess. +- Don't grade on intent you can't see. The agent's internal thoughts aren't visible — only messages, tool calls, and results. + +### Verdict file + +Save to `results//codex-app/YYYY-MM-DD-manual/verdict.json`: + +```json +{ + "scenario": "", + "backend": "codex-app", + "manual": true, + "user_posture": "", + "passed": , + "criteria": [ ... ], + "notes": "" +} +``` + +Matches the format of the existing `results/worktree-codex-app-detached-head/codex-app/2026-04-09-manual/verdict.json`. + +## When to invoke + +- A scenario's YAML has `manual: true` +- The tester explicitly asks for a manual Codex App run of any scenario +- An automated test result is inconclusive and we want a human-verified cross-check + +Do NOT use this procedure for scenarios drill can run itself (`claude`, `codex`, `gemini` backends) — use `drill run` instead. + +## Pitfalls + +- **Don't skip the fixture step.** Codex App's default environment (detached HEAD under `$CODEX_HOME/worktrees/`) is load-bearing for worktree scenarios. The same prompt gives different results in a normal checkout. +- **Don't render prompts literally.** Scenario intents are written for test authors; they often contain "Do NOT mention X" style instructions. Translate before handing to the tester. +- **Don't grade on missing evidence.** If the transcript doesn't show the agent doing something the criterion asks about, that's a fail, not a pass-by-default. diff --git a/evals/docs/plan.md b/evals/docs/plan.md new file mode 100644 index 00000000..5647c4a8 --- /dev/null +++ b/evals/docs/plan.md @@ -0,0 +1,2725 @@ +# Drill Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a tmux-based harness that drives AI coding agents through worktree scenarios and evaluates whether they follow superpowers skills. + +**Architecture:** CLI (`click`) orchestrates an engine that sets up a test repo, launches an agent in tmux, drives it via an LLM actor (Anthropic SDK, structured tool_use), collects session logs + filesystem state, then evaluates compliance via an LLM verifier. Backend configs (YAML) define how to launch each agent CLI. Scenarios (YAML) define what to test. + +**Tech Stack:** Python 3.11+, click, pyyaml, anthropic SDK, jinja2, pydantic, tmux + +--- + +## File Structure + +``` +drill/ +├── drill/ +│ ├── __init__.py # Package init, version +│ ├── cli.py # click CLI: run, compare, list +│ ├── engine.py # Orchestrates full run lifecycle (7 steps) +│ ├── session.py # tmux session management (create, send-keys, capture, kill) +│ ├── actor.py # Actor LLM: rolling context, structured tool_use output +│ ├── verifier.py # Verifier LLM: per-criterion evaluation, pydantic schema +│ ├── setup.py # Template repo cloning, helper dispatch, assertion runner +│ ├── backend.py # Loads backend YAML, builds CLI commands, idle detection +│ └── normalizer.py # Normalizes backend-specific session logs to common schema +├── backends/ +│ ├── claude.yaml # Claude Code backend config +│ └── codex.yaml # Codex backend config +├── prompts/ +│ ├── actor.md # Actor system prompt (jinja2 template) +│ └── verifier.md # Verifier system prompt (jinja2 template) +├── scenarios/ +│ ├── worktree-creation-from-main.yaml +│ ├── worktree-already-inside.yaml +│ ├── worktree-codex-detached-head.yaml +│ └── worktree-consent-flow.yaml +├── fixtures/ +│ └── template-repo/ # Minimal git repo cloned per run +│ ├── package.json +│ ├── src/ +│ │ ├── index.js +│ │ └── utils.js +│ └── README.md +├── setup_helpers/ +│ ├── __init__.py # Exports helper registry +│ ├── base.py # create_base_repo +│ └── worktree.py # add_worktree, detach_head, symlink_superpowers +├── tests/ +│ ├── test_backend.py +│ ├── test_setup.py +│ ├── test_session.py +│ ├── test_actor.py +│ ├── test_verifier.py +│ ├── test_normalizer.py +│ ├── test_engine.py +│ └── test_cli.py +├── pyproject.toml +├── .gitignore +└── README.md +``` + +--- + +### Task 1: Project Scaffold + +**Files:** +- Create: `pyproject.toml` +- Create: `drill/__init__.py` +- Create: `.gitignore` +- Create: `README.md` + +- [ ] **Step 1: Create pyproject.toml** + +```toml +[build-system] +requires = ["setuptools>=68.0"] +build-backend = "setuptools.backends._legacy:_Backend" + +[project] +name = "drill" +version = "0.1.0" +description = "Superpowers skill compliance benchmark" +requires-python = ">=3.11" +dependencies = [ + "click>=8.1", + "pyyaml>=6.0", + "anthropic>=0.42", + "jinja2>=3.1", + "pydantic>=2.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[project.scripts] +drill = "drill.cli:main" + +[tool.setuptools.packages.find] +include = ["drill*", "setup_helpers*"] +``` + +- [ ] **Step 2: Create drill/__init__.py** + +```python +"""Drill: Superpowers skill compliance benchmark.""" + +__version__ = "0.1.0" +``` + +- [ ] **Step 3: Create .gitignore** + +``` +results/ +__pycache__/ +*.pyc +*.egg-info/ +dist/ +build/ +.venv/ +``` + +- [ ] **Step 4: Create README.md** + +```markdown +# Drill + +Superpowers skill compliance benchmark. Drives AI coding agents through +tmux sessions and evaluates whether they follow superpowers workflows. + +See [docs/design.md](docs/design.md) for the full design spec. + +## Setup + +```bash +pip install -e ".[dev]" +``` + +## Usage + +```bash +export SUPERPOWERS_ROOT=/path/to/superpowers +export ANTHROPIC_API_KEY=sk-... + +drill run worktree-creation-from-main --backend claude +drill compare worktree-creation-from-main +drill list +``` +``` + +- [ ] **Step 5: Install in dev mode and verify** + +Run: `cd /Users/drewritter/prime-rad/drill && pip install -e ".[dev]"` +Expected: Installs successfully, `drill --help` shows usage + +- [ ] **Step 6: Commit** + +```bash +git add pyproject.toml drill/__init__.py .gitignore README.md +git commit -m "chore: project scaffold with pyproject.toml and drill entry point" +``` + +--- + +### Task 2: Backend Config Loader + +**Files:** +- Create: `drill/backend.py` +- Create: `backends/claude.yaml` +- Create: `backends/codex.yaml` +- Create: `tests/test_backend.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_backend.py +import os +import pytest +from pathlib import Path + +from drill.backend import Backend, load_backend + + +@pytest.fixture +def backends_dir(): + return Path(__file__).parent.parent / "backends" + + +class TestLoadBackend: + def test_loads_claude_backend(self, backends_dir): + backend = load_backend("claude", backends_dir) + assert backend.name == "claude" + assert backend.cli == "claude" + assert "--dangerously-skip-permissions" in backend.args + + def test_loads_codex_backend(self, backends_dir): + backend = load_backend("codex", backends_dir) + assert backend.name == "codex" + assert backend.cli == "codex" + + def test_unknown_backend_raises(self, backends_dir): + with pytest.raises(FileNotFoundError): + load_backend("nonexistent", backends_dir) + + +class TestBackendBuildCommand: + def test_claude_build_command(self, backends_dir, monkeypatch): + monkeypatch.setenv("SUPERPOWERS_ROOT", "/tmp/superpowers") + backend = load_backend("claude", backends_dir) + cmd = backend.build_command("/tmp/workdir") + assert cmd[0] == "claude" + assert "--plugin-dir" in cmd + assert "/tmp/superpowers" in cmd + + def test_codex_build_command(self, backends_dir, monkeypatch): + monkeypatch.setenv("SUPERPOWERS_ROOT", "/tmp/superpowers") + backend = load_backend("codex", backends_dir) + cmd = backend.build_command("/tmp/workdir") + assert cmd[0] == "codex" + + +class TestBackendEnvValidation: + def test_missing_env_raises(self, backends_dir, monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("SUPERPOWERS_ROOT", raising=False) + backend = load_backend("claude", backends_dir) + with pytest.raises(EnvironmentError, match="ANTHROPIC_API_KEY"): + backend.validate_env() + + +class TestBackendIdleDetection: + def test_ready_pattern_matches(self, backends_dir): + backend = load_backend("claude", backends_dir) + assert backend.is_ready_line("❯ ") + assert backend.is_ready_line("Human: ") + assert not backend.is_ready_line("Running tool...") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_backend.py -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'drill.backend'` + +- [ ] **Step 3: Create backend YAML files** + +Create `backends/claude.yaml`: + +```yaml +name: claude +cli: claude +args: + - "--dangerously-skip-permissions" + - "--plugin-dir" + - "${SUPERPOWERS_ROOT}" +required_env: + - ANTHROPIC_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: [] + post_run: [] +shutdown: "/exit" +idle: + quiescence_seconds: 3 + ready_pattern: "^❯|^\\$|Human:" +startup_timeout: 30 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.claude/projects/**/session-*.jsonl" +``` + +Create `backends/codex.yaml`: + +```yaml +name: codex +cli: codex +args: + - "--dangerously-bypass-approvals-and-sandbox" +required_env: + - OPENAI_API_KEY + - SUPERPOWERS_ROOT +hooks: + pre_run: + - symlink_superpowers + post_run: [] +shutdown: "<>" +idle: + quiescence_seconds: 5 + ready_pattern: "codex>|^>" +startup_timeout: 30 +terminal: + cols: 200 + rows: 50 +session_logs: + pattern: "~/.codex/sessions/rollout-*.jsonl" +``` + +- [ ] **Step 4: Write the implementation** + +```python +# drill/backend.py +"""Backend config loader and command builder.""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + + +@dataclass +class Backend: + name: str + cli: str + args: list[str] + required_env: list[str] + hooks: dict[str, list[str]] + shutdown: str + idle: dict[str, any] + startup_timeout: int + terminal: dict[str, int] + session_logs: dict[str, str] + + def build_command(self, workdir: str) -> list[str]: + """Build the full CLI invocation with env var interpolation.""" + resolved = [] + for arg in self.args: + resolved.append(_interpolate_env(arg)) + return [self.cli, *resolved] + + def validate_env(self) -> None: + """Raise EnvironmentError if any required env vars are missing.""" + missing = [v for v in self.required_env if not os.environ.get(v)] + if missing: + raise EnvironmentError( + f"Missing required environment variables for {self.name} backend: " + + ", ".join(missing) + ) + + def is_ready_line(self, line: str) -> bool: + """Check if a terminal line matches the idle ready pattern.""" + pattern = self.idle.get("ready_pattern", "") + return bool(re.search(pattern, line)) + + @property + def quiescence_seconds(self) -> float: + return self.idle.get("quiescence_seconds", 5) + + @property + def cols(self) -> int: + return self.terminal.get("cols", 200) + + @property + def rows(self) -> int: + return self.terminal.get("rows", 50) + + +def load_backend(name: str, backends_dir: Path) -> Backend: + """Load a backend config from YAML.""" + path = backends_dir / f"{name}.yaml" + if not path.exists(): + raise FileNotFoundError(f"Backend config not found: {path}") + with open(path) as f: + data = yaml.safe_load(f) + return Backend( + name=data["name"], + cli=data["cli"], + args=data.get("args", []), + required_env=data.get("required_env", []), + hooks=data.get("hooks", {"pre_run": [], "post_run": []}), + shutdown=data.get("shutdown", "/exit"), + idle=data.get("idle", {}), + startup_timeout=data.get("startup_timeout", 30), + terminal=data.get("terminal", {"cols": 200, "rows": 50}), + session_logs=data.get("session_logs", {}), + ) + + +def _interpolate_env(value: str) -> str: + """Replace ${VAR} with environment variable values.""" + def replacer(match): + var = match.group(1) + val = os.environ.get(var) + if val is None: + raise EnvironmentError(f"Environment variable {var} not set") + return val + return re.sub(r"\$\{(\w+)\}", replacer, value) +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_backend.py -v` +Expected: All tests PASS + +- [ ] **Step 6: Commit** + +```bash +git add drill/backend.py backends/ tests/test_backend.py +git commit -m "feat: backend config loader with YAML parsing and env validation" +``` + +--- + +### Task 3: tmux Session Manager + +**Files:** +- Create: `drill/session.py` +- Create: `tests/test_session.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_session.py +import subprocess +import time +import pytest + +from drill.session import TmuxSession + + +class TestTmuxSession: + def test_create_and_kill(self): + session = TmuxSession(name="drill-test-create", cols=80, rows=24) + session.create() + # Verify session exists + result = subprocess.run( + ["tmux", "has-session", "-t", "drill-test-create"], + capture_output=True, + ) + assert result.returncode == 0 + session.kill() + # Verify session is gone + result = subprocess.run( + ["tmux", "has-session", "-t", "drill-test-create"], + capture_output=True, + ) + assert result.returncode != 0 + + def test_send_keys_and_capture(self): + session = TmuxSession(name="drill-test-keys", cols=80, rows=24) + session.create() + try: + session.send_keys("echo hello-drill-test") + time.sleep(0.5) + output = session.capture() + assert "hello-drill-test" in output + finally: + session.kill() + + def test_launch_command(self, tmp_path): + session = TmuxSession(name="drill-test-launch", cols=80, rows=24) + session.create() + try: + session.launch(["python3", "-c", "import time; time.sleep(30)"], cwd=str(tmp_path)) + time.sleep(0.5) + output = session.capture() + # Process should be running, not showing shell prompt + assert session.is_process_alive() + finally: + session.kill() + + def test_send_special_key(self): + session = TmuxSession(name="drill-test-special", cols=80, rows=24) + session.create() + try: + session.send_keys("cat") # start cat, which reads stdin + time.sleep(0.3) + session.send_special_key("ctrl-c") + time.sleep(0.3) + # After ctrl-c, cat should have exited + output = session.capture() + assert "^C" in output or output.endswith("$") + finally: + session.kill() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_session.py -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'drill.session'` + +- [ ] **Step 3: Write the implementation** + +```python +# drill/session.py +"""tmux session management for driving agent CLI sessions.""" + +from __future__ import annotations + +import subprocess +import time + + +class TmuxSession: + """Manages a tmux session for driving an agent CLI.""" + + def __init__(self, name: str, cols: int = 200, rows: int = 50): + self.name = name + self.cols = cols + self.rows = rows + + def create(self) -> None: + """Create a new detached tmux session.""" + subprocess.run( + [ + "tmux", "new-session", + "-d", + "-s", self.name, + "-x", str(self.cols), + "-y", str(self.rows), + ], + check=True, + ) + + def launch(self, command: list[str], cwd: str) -> None: + """Launch a command inside the tmux session.""" + cmd_str = " ".join(command) + self.send_keys(f"cd {cwd} && {cmd_str}") + + def send_keys(self, text: str) -> None: + """Send keystrokes to the tmux session, followed by Enter.""" + subprocess.run( + ["tmux", "send-keys", "-t", self.name, text, "Enter"], + check=True, + ) + + def send_special_key(self, key: str) -> None: + """Send a special key like ctrl-c, ctrl-d.""" + key_map = { + "ctrl-c": "C-c", + "ctrl-d": "C-d", + "ctrl-z": "C-z", + "enter": "Enter", + "escape": "Escape", + } + tmux_key = key_map.get(key, key) + subprocess.run( + ["tmux", "send-keys", "-t", self.name, tmux_key], + check=True, + ) + + def capture(self) -> str: + """Capture the current terminal pane content.""" + result = subprocess.run( + ["tmux", "capture-pane", "-t", self.name, "-p"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + def is_process_alive(self) -> bool: + """Check if the process in the pane is still running.""" + result = subprocess.run( + [ + "tmux", "list-panes", "-t", self.name, + "-F", "#{pane_dead}", + ], + capture_output=True, + text=True, + ) + return result.stdout.strip() == "0" + + def kill(self) -> None: + """Kill the tmux session.""" + subprocess.run( + ["tmux", "kill-session", "-t", self.name], + capture_output=True, # don't fail if already dead + ) +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_session.py -v` +Expected: All tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add drill/session.py tests/test_session.py +git commit -m "feat: tmux session manager with send-keys, capture, and special key support" +``` + +--- + +### Task 4: Setup Helpers and Template Repo + +**Files:** +- Create: `setup_helpers/__init__.py` +- Create: `setup_helpers/base.py` +- Create: `setup_helpers/worktree.py` +- Create: `fixtures/template-repo/` (with contents) +- Create: `drill/setup.py` +- Create: `tests/test_setup.py` + +- [ ] **Step 1: Create the template repo fixture** + +```bash +cd /Users/drewritter/prime-rad/drill +mkdir -p fixtures/template-repo/src +``` + +Create `fixtures/template-repo/package.json`: +```json +{ + "name": "drill-test-project", + "version": "1.0.0", + "description": "Test project for Drill scenarios", + "main": "src/index.js" +} +``` + +Create `fixtures/template-repo/src/index.js`: +```javascript +const { greet } = require('./utils'); + +function main() { + console.log(greet('world')); +} + +main(); +``` + +Create `fixtures/template-repo/src/utils.js`: +```javascript +function greet(name) { + return `Hello, ${name}!`; +} + +module.exports = { greet }; +``` + +Create `fixtures/template-repo/README.md`: +```markdown +# Test Project + +A minimal project for Drill test scenarios. +``` + +Initialize git history: +```bash +cd fixtures/template-repo +git init +git add package.json README.md +git commit -m "initial commit" +git add src/utils.js +git commit -m "add utils module" +git add src/index.js +git commit -m "add entry point" +cd ../.. +``` + +- [ ] **Step 2: Write the failing test** + +```python +# tests/test_setup.py +import os +import subprocess +import pytest +from pathlib import Path + +from drill.setup import clone_template, run_assertions +from setup_helpers.base import create_base_repo +from setup_helpers.worktree import add_worktree, detach_head, symlink_superpowers + + +@pytest.fixture +def fixtures_dir(): + return Path(__file__).parent.parent / "fixtures" + + +@pytest.fixture +def work_dir(tmp_path): + return tmp_path / "test-repo" + + +class TestCloneTemplate: + def test_clones_template_repo(self, fixtures_dir, work_dir): + clone_template(fixtures_dir / "template-repo", work_dir) + assert (work_dir / "package.json").exists() + assert (work_dir / "src" / "index.js").exists() + # Should have git history + result = subprocess.run( + ["git", "log", "--oneline"], + cwd=work_dir, + capture_output=True, + text=True, + ) + assert "initial commit" in result.stdout + + +class TestCreateBaseRepo: + def test_creates_base_repo(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + assert (work_dir / "package.json").exists() + result = subprocess.run( + ["git", "branch", "--show-current"], + cwd=work_dir, + capture_output=True, + text=True, + ) + assert result.stdout.strip() == "main" + + +class TestWorktreeHelpers: + def test_add_worktree(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + wt_path = work_dir.parent / "feature-wt" + add_worktree(work_dir, "feature-branch", str(wt_path)) + assert wt_path.exists() + result = subprocess.run( + ["git", "worktree", "list"], + cwd=work_dir, + capture_output=True, + text=True, + ) + assert "feature-branch" in result.stdout + + def test_detach_head(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + wt_path = work_dir.parent / "detached-wt" + add_worktree(work_dir, "tmp-branch", str(wt_path)) + detach_head(str(wt_path)) + result = subprocess.run( + ["git", "branch", "--show-current"], + cwd=wt_path, + capture_output=True, + text=True, + ) + assert result.stdout.strip() == "" # detached = no branch + + def test_symlink_superpowers(self, fixtures_dir, work_dir, tmp_path): + create_base_repo(work_dir, fixtures_dir / "template-repo") + fake_superpowers = tmp_path / "superpowers" / "skills" + fake_superpowers.mkdir(parents=True) + symlink_superpowers(work_dir, str(tmp_path / "superpowers")) + link = work_dir / ".agents" / "skills" / "superpowers" + assert link.is_symlink() + + +class TestRunAssertions: + def test_passing_assertions(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + assertions = [ + "git rev-parse --is-inside-work-tree", + "git branch --show-current | grep main", + ] + # Should not raise + run_assertions(assertions, work_dir) + + def test_failing_assertion_raises(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + assertions = ["git branch --show-current | grep nonexistent"] + with pytest.raises(AssertionError, match="Setup assertion failed"): + run_assertions(assertions, work_dir) +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_setup.py -v` +Expected: FAIL — `ModuleNotFoundError` + +- [ ] **Step 4: Write setup_helpers** + +Create `setup_helpers/__init__.py`: +```python +"""Setup helpers for Drill scenarios.""" + +from setup_helpers.base import create_base_repo +from setup_helpers.worktree import add_worktree, detach_head, symlink_superpowers + +HELPER_REGISTRY = { + "create_base_repo": create_base_repo, + "add_worktree": add_worktree, + "detach_head": detach_head, + "symlink_superpowers": symlink_superpowers, +} +``` + +Create `setup_helpers/base.py`: +```python +"""Base setup helpers.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +def create_base_repo(workdir: Path, template_dir: Path) -> None: + """Clone the template repo to workdir.""" + subprocess.run( + ["git", "clone", str(template_dir), str(workdir)], + check=True, + capture_output=True, + ) +``` + +Create `setup_helpers/worktree.py`: +```python +"""Worktree-specific setup helpers.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +def add_worktree(repo_dir: Path, branch: str, worktree_path: str) -> None: + """Create a git worktree at the given path.""" + subprocess.run( + ["git", "worktree", "add", "-b", branch, worktree_path], + cwd=repo_dir, + check=True, + capture_output=True, + ) + + +def detach_head(worktree_path: str) -> None: + """Detach HEAD in a worktree (simulates Codex App state).""" + # Get current commit hash + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=worktree_path, + capture_output=True, + text=True, + check=True, + ) + commit = result.stdout.strip() + # Get the branch name so we can delete it after detaching + result = subprocess.run( + ["git", "branch", "--show-current"], + cwd=worktree_path, + capture_output=True, + text=True, + check=True, + ) + branch = result.stdout.strip() + # Detach HEAD + subprocess.run( + ["git", "checkout", "--detach", commit], + cwd=worktree_path, + check=True, + capture_output=True, + ) + # Delete the temporary branch + if branch: + subprocess.run( + ["git", "branch", "-D", branch], + cwd=worktree_path, + capture_output=True, + ) + + +def symlink_superpowers(workdir: Path, superpowers_root: str) -> None: + """Create .agents/skills/superpowers symlink for Codex discovery.""" + skills_dir = Path(workdir) / ".agents" / "skills" + skills_dir.mkdir(parents=True, exist_ok=True) + target = Path(superpowers_root) / "skills" + link = skills_dir / "superpowers" + link.symlink_to(target) +``` + +- [ ] **Step 5: Write drill/setup.py** + +```python +# drill/setup.py +"""Test repo setup: template cloning, helper dispatch, assertion runner.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from setup_helpers import HELPER_REGISTRY + + +def clone_template(template_dir: Path, workdir: Path) -> None: + """Clone the template repo to a working directory.""" + subprocess.run( + ["git", "clone", str(template_dir), str(workdir)], + check=True, + capture_output=True, + ) + + +def run_helpers( + helper_names: list[str], + workdir: Path, + fixtures_dir: Path, +) -> None: + """Run named setup helpers against the working directory.""" + for name in helper_names: + helper = HELPER_REGISTRY.get(name) + if helper is None: + raise ValueError(f"Unknown setup helper: {name}") + if name == "create_base_repo": + helper(workdir, fixtures_dir / "template-repo") + elif name == "symlink_superpowers": + import os + helper(workdir, os.environ["SUPERPOWERS_ROOT"]) + else: + # All other helpers take workdir as single arg + helper(workdir) + + +def run_assertions(assertions: list[str], workdir: Path) -> None: + """Run shell assertion commands. Raise if any fail.""" + for assertion in assertions: + result = subprocess.run( + assertion, + shell=True, + cwd=workdir, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise AssertionError( + f"Setup assertion failed: {assertion}\n" + f"stdout: {result.stdout}\n" + f"stderr: {result.stderr}" + ) +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_setup.py -v` +Expected: All tests PASS + +- [ ] **Step 7: Commit** + +```bash +git add fixtures/ setup_helpers/ drill/setup.py tests/test_setup.py +git commit -m "feat: template repo, setup helpers, and assertion runner" +``` + +--- + +### Task 5: Actor LLM + +**Files:** +- Create: `drill/actor.py` +- Create: `prompts/actor.md` +- Create: `tests/test_actor.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_actor.py +import json +import pytest +from unittest.mock import MagicMock, patch + +from drill.actor import Actor, ActorAction + + +class TestActorAction: + def test_parse_type_action(self): + action = ActorAction.from_tool_result({"action": "type", "text": "create a worktree"}) + assert action.action == "type" + assert action.text == "create a worktree" + + def test_parse_done_action(self): + action = ActorAction.from_tool_result({"action": "done"}) + assert action.action == "done" + + def test_parse_stuck_action(self): + action = ActorAction.from_tool_result({"action": "stuck"}) + assert action.action == "stuck" + + def test_parse_key_action(self): + action = ActorAction.from_tool_result({"action": "key", "key": "ctrl-c"}) + assert action.action == "key" + assert action.key == "ctrl-c" + + +class TestActorPrompt: + def test_builds_system_prompt_naive(self): + actor = Actor(model="claude-sonnet-4-6", temperature=0.7) + prompt = actor.build_system_prompt( + posture="naive", + intents=["Ask the agent to create a worktree"], + ) + assert "naive" in prompt.lower() or "plain language" in prompt.lower() + assert "create a worktree" in prompt + + def test_builds_system_prompt_spec_aware(self): + actor = Actor(model="claude-sonnet-4-6", temperature=0.7) + prompt = actor.build_system_prompt( + posture="spec-aware", + intents=["Use the worktree skill to create an isolated workspace"], + ) + assert "spec" in prompt.lower() or "skill" in prompt.lower() + + +class TestActorContext: + def test_appends_terminal_captures(self): + actor = Actor(model="claude-sonnet-4-6", temperature=0.7) + actor.append_capture("Screen 1: Welcome to Claude") + actor.append_capture("Screen 2: ❯ ") + messages = actor.build_messages() + assert len(messages) == 2 + assert "Screen 1" in messages[0]["content"] + assert "Screen 2" in messages[1]["content"] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_actor.py -v` +Expected: FAIL — `ModuleNotFoundError` + +- [ ] **Step 3: Create the actor prompt template** + +Create `prompts/actor.md`: + +```markdown +You are simulating a user interacting with an AI coding agent in a terminal. + +{% if posture == "naive" %} +You are a developer who wants to accomplish a task. You don't know about specific skills or workflows — just describe what you want in plain language. +{% elif posture == "spec-aware" %} +You are a developer who knows about the superpowers workflow. You may reference specific skills or conventions by name (e.g., "use the worktree skill", "follow the using-git-worktrees pattern"). +{% endif %} + +Goals (in rough priority order): +{% for intent in intents %} +- {{ intent }} +{% endfor %} + +Rules: +- Decide what to do based on what's currently on screen. +- Goals are not a script — some are conditional. Act on them when relevant. +- Type natural, concise messages like a real developer would. +- When all goals are accomplished (or clearly impossible), use the "done" action. +- If you're stuck and cannot make progress, use the "stuck" action. +``` + +- [ ] **Step 4: Write the implementation** + +```python +# drill/actor.py +"""Actor LLM: simulates a user driving an agent session.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +import anthropic +from jinja2 import Template + + +ACTOR_TOOL = { + "name": "terminal_action", + "description": "Send an action to the terminal session.", + "input_schema": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": ["type", "done", "stuck", "key"], + "description": "The action to take.", + }, + "text": { + "type": "string", + "description": "Text to type (only for 'type' action).", + }, + "key": { + "type": "string", + "description": "Special key to send (only for 'key' action, e.g., 'ctrl-c').", + }, + }, + "required": ["action"], + }, +} + + +@dataclass +class ActorAction: + action: str # "type", "done", "stuck", "key" + text: str | None = None + key: str | None = None + + @classmethod + def from_tool_result(cls, data: dict) -> ActorAction: + return cls( + action=data["action"], + text=data.get("text"), + key=data.get("key"), + ) + + +class Actor: + """Drives agent sessions by deciding what a simulated user would type.""" + + def __init__(self, model: str = "claude-sonnet-4-6", temperature: float = 0.7): + self.model = model + self.temperature = temperature + self.captures: list[str] = [] + self._system_prompt: str | None = None + self._client = anthropic.Anthropic() + + def build_system_prompt(self, posture: str, intents: list[str]) -> str: + """Render the actor system prompt from template.""" + template_path = Path(__file__).parent.parent / "prompts" / "actor.md" + template = Template(template_path.read_text()) + self._system_prompt = template.render(posture=posture, intents=intents) + return self._system_prompt + + def append_capture(self, terminal_output: str) -> None: + """Append a terminal capture to the rolling context.""" + self.captures.append(terminal_output) + + def build_messages(self) -> list[dict]: + """Build the message list from terminal captures.""" + messages = [] + for capture in self.captures: + messages.append({"role": "user", "content": capture}) + return messages + + def decide(self) -> ActorAction: + """Call the LLM to decide the next action.""" + response = self._client.messages.create( + model=self.model, + max_tokens=1024, + temperature=self.temperature, + system=self._system_prompt, + tools=[ACTOR_TOOL], + tool_choice={"type": "tool", "name": "terminal_action"}, + messages=self.build_messages(), + ) + # Extract the tool use block + for block in response.content: + if block.type == "tool_use": + return ActorAction.from_tool_result(block.input) + raise RuntimeError("Actor did not return a tool_use block") +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_actor.py -v` +Expected: All tests PASS (no live API calls — only testing parsing and prompt building) + +- [ ] **Step 6: Commit** + +```bash +git add drill/actor.py prompts/actor.md tests/test_actor.py +git commit -m "feat: actor LLM with structured tool_use output and prompt template" +``` + +--- + +### Task 6: Verifier LLM + +**Files:** +- Create: `drill/verifier.py` +- Create: `prompts/verifier.md` +- Create: `tests/test_verifier.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_verifier.py +import json +import pytest +from unittest.mock import MagicMock, patch + +from drill.verifier import Verifier, Verdict, CriterionResult + + +class TestVerdict: + def test_parse_valid_verdict(self): + data = { + "criteria": [ + { + "criterion": "Agent detected on main", + "verdict": "pass", + "evidence": "Terminal showed 'main branch detected'", + "rationale": "Agent correctly identified the branch", + } + ], + "observations": ["Agent was very fast"], + "summary": "Passed all checks", + } + verdict = Verdict.model_validate(data) + assert len(verdict.criteria) == 1 + assert verdict.criteria[0].verdict == "pass" + assert verdict.score == "1/1" + + def test_score_calculation(self): + data = { + "criteria": [ + {"criterion": "A", "verdict": "pass", "evidence": "e", "rationale": "r"}, + {"criterion": "B", "verdict": "fail", "evidence": "e", "rationale": "r"}, + {"criterion": "C", "verdict": "pass", "evidence": "e", "rationale": "r"}, + ], + "observations": [], + "summary": "Mixed results", + } + verdict = Verdict.model_validate(data) + assert verdict.score == "2/3" + assert verdict.passed is False + + def test_all_pass(self): + data = { + "criteria": [ + {"criterion": "A", "verdict": "pass", "evidence": "e", "rationale": "r"}, + ], + "observations": [], + "summary": "Good", + } + verdict = Verdict.model_validate(data) + assert verdict.passed is True + + +class TestVerifierPrompt: + def test_builds_system_prompt(self): + verifier = Verifier(model="claude-sonnet-4-6", temperature=0.0) + prompt = verifier.build_system_prompt() + assert "criterion" in prompt.lower() + assert "evidence" in prompt.lower() + assert "JSON" in prompt +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_verifier.py -v` +Expected: FAIL — `ModuleNotFoundError` + +- [ ] **Step 3: Create the verifier prompt template** + +Create `prompts/verifier.md`: + +```markdown +You are evaluating whether an AI coding agent correctly followed a workflow specification during a terminal session. + +You will receive: +1. Terminal session log (what was displayed on screen) +2. Filesystem state after the session (file tree, git state, worktree list) +3. Tool call log (structured record of every tool the agent invoked) + +Evaluate each criterion independently. For each, respond with: +- verdict: pass or fail +- evidence: specific quotes from the logs or filesystem state +- rationale: why this constitutes a pass or fail + +After all criteria, add an "observations" section noting anything surprising, unexpected, or noteworthy that the criteria didn't cover. + +Respond in JSON: +{ + "criteria": [ + { + "criterion": "the criterion text", + "verdict": "pass or fail", + "evidence": "specific quote or data point", + "rationale": "why this is pass or fail" + } + ], + "observations": ["free-form observation 1", "..."], + "summary": "one-line overall assessment" +} +``` + +- [ ] **Step 4: Write the implementation** + +```python +# drill/verifier.py +"""Verifier LLM: evaluates agent session against criteria.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import anthropic +from jinja2 import Template +from pydantic import BaseModel + + +class CriterionResult(BaseModel): + criterion: str + verdict: str # "pass" or "fail" + evidence: str + rationale: str + + +class Verdict(BaseModel): + criteria: list[CriterionResult] + observations: list[str] + summary: str + + @property + def score(self) -> str: + passed = sum(1 for c in self.criteria if c.verdict == "pass") + return f"{passed}/{len(self.criteria)}" + + @property + def passed(self) -> bool: + return all(c.verdict == "pass" for c in self.criteria) + + +class Verifier: + """Evaluates agent sessions against verification criteria.""" + + MAX_RETRIES = 3 + + def __init__(self, model: str = "claude-sonnet-4-6", temperature: float = 0.0): + self.model = model + self.temperature = temperature + self._client = anthropic.Anthropic() + + def build_system_prompt(self) -> str: + """Render the verifier system prompt from template.""" + template_path = Path(__file__).parent.parent / "prompts" / "verifier.md" + return template_path.read_text() + + def verify( + self, + session_log: str, + filesystem_json: str, + tool_calls_jsonl: str, + criteria: list[str], + ) -> Verdict: + """Run the verifier against a completed session.""" + system = self.build_system_prompt() + user_content = ( + "## Terminal Session Log\n\n" + f"```\n{session_log}\n```\n\n" + "## Filesystem State\n\n" + f"```json\n{filesystem_json}\n```\n\n" + "## Tool Call Log\n\n" + f"```jsonl\n{tool_calls_jsonl}\n```\n\n" + "## Criteria to Evaluate\n\n" + + "\n".join(f"- {c}" for c in criteria) + ) + + for attempt in range(self.MAX_RETRIES): + response = self._client.messages.create( + model=self.model, + max_tokens=4096, + temperature=self.temperature, + system=system, + messages=[{"role": "user", "content": user_content}], + ) + text = response.content[0].text + # Extract JSON from response (may be wrapped in markdown fences) + json_str = _extract_json(text) + try: + return Verdict.model_validate_json(json_str) + except Exception: + if attempt == self.MAX_RETRIES - 1: + raise + continue + + raise RuntimeError("Verifier failed to return valid JSON") + + +def _extract_json(text: str) -> str: + """Extract JSON from text that may be wrapped in markdown code fences.""" + # Try to find JSON in code fences + if "```json" in text: + start = text.index("```json") + 7 + end = text.index("```", start) + return text[start:end].strip() + if "```" in text: + start = text.index("```") + 3 + end = text.index("```", start) + return text[start:end].strip() + # Try raw JSON + start = text.index("{") + end = text.rindex("}") + 1 + return text[start:end] +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_verifier.py -v` +Expected: All tests PASS + +- [ ] **Step 6: Commit** + +```bash +git add drill/verifier.py prompts/verifier.md tests/test_verifier.py +git commit -m "feat: verifier LLM with pydantic verdict schema and retry logic" +``` + +--- + +### Task 7: Log Normalizer + +**Files:** +- Create: `drill/normalizer.py` +- Create: `tests/test_normalizer.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_normalizer.py +import json +import pytest +from pathlib import Path + +from drill.normalizer import normalize_claude_logs, normalize_codex_logs, snapshot_log_dir, collect_new_logs + + +class TestSnapshotAndCollect: + def test_snapshot_and_collect_new_files(self, tmp_path): + log_dir = tmp_path / "logs" + log_dir.mkdir() + # Pre-existing file + (log_dir / "old.jsonl").write_text('{"old": true}\n') + snapshot = snapshot_log_dir(log_dir) + # Simulate new file created during session + (log_dir / "new.jsonl").write_text('{"new": true}\n') + new_files = collect_new_logs(log_dir, snapshot) + assert len(new_files) == 1 + assert new_files[0].name == "new.jsonl" + + def test_empty_dir_returns_empty(self, tmp_path): + log_dir = tmp_path / "logs" + log_dir.mkdir() + snapshot = snapshot_log_dir(log_dir) + new_files = collect_new_logs(log_dir, snapshot) + assert new_files == [] + + +class TestNormalizeClaudeLogs: + def test_normalizes_tool_use(self): + lines = [ + json.dumps({ + "type": "tool_use", + "name": "EnterWorktree", + "input": {"branch": "add-login"}, + }), + json.dumps({ + "type": "tool_use", + "name": "Bash", + "input": {"command": "git status"}, + }), + json.dumps({ + "type": "text", + "text": "I'll create a worktree", + }), + ] + raw = "\n".join(lines) + normalized = normalize_claude_logs(raw) + assert len(normalized) == 2 + assert normalized[0]["tool"] == "EnterWorktree" + assert normalized[0]["source"] == "native" + assert normalized[1]["tool"] == "Bash" + assert normalized[1]["source"] == "shell" + + +class TestNormalizeCodexLogs: + def test_normalizes_local_shell_call(self): + lines = [ + json.dumps({ + "type": "response_item", + "item": { + "type": "local_shell_call", + "action": {"command": ["git", "worktree", "add", "feature"]}, + "status": "completed", + } + }), + json.dumps({ + "type": "response_item", + "item": { + "type": "message", + "content": [{"text": "Creating worktree"}], + } + }), + ] + raw = "\n".join(lines) + normalized = normalize_codex_logs(raw) + assert len(normalized) == 1 + assert normalized[0]["tool"] == "Bash" + assert "git worktree add" in normalized[0]["args"]["command"] + assert normalized[0]["source"] == "shell" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_normalizer.py -v` +Expected: FAIL — `ModuleNotFoundError` + +- [ ] **Step 3: Write the implementation** + +```python +# drill/normalizer.py +"""Normalizes backend-specific session logs to a common tool call schema.""" + +from __future__ import annotations + +import json +from pathlib import Path + +# Tools that are native (not shell commands) +NATIVE_TOOLS = { + "EnterWorktree", "ExitWorktree", "EnterPlanMode", "ExitPlanMode", + "TaskCreate", "TaskUpdate", "TaskList", "TaskGet", + "Skill", "Agent", "Read", "Write", "Edit", "Glob", "Grep", +} + + +def snapshot_log_dir(log_dir: Path) -> set[str]: + """Snapshot the current files in a log directory.""" + if not log_dir.exists(): + return set() + return {f.name for f in log_dir.iterdir() if f.is_file()} + + +def collect_new_logs(log_dir: Path, snapshot: set[str]) -> list[Path]: + """Find files created after the snapshot was taken.""" + if not log_dir.exists(): + return [] + current = {f.name for f in log_dir.iterdir() if f.is_file()} + new_names = current - snapshot + return [log_dir / name for name in sorted(new_names)] + + +def normalize_claude_logs(raw_content: str) -> list[dict]: + """Normalize Claude Code session log to common schema.""" + results = [] + for line in raw_content.strip().split("\n"): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if entry.get("type") == "tool_use": + tool_name = entry.get("name", "") + source = "native" if tool_name in NATIVE_TOOLS else "shell" + results.append({ + "tool": tool_name, + "args": entry.get("input", {}), + "source": source, + }) + return results + + +def normalize_codex_logs(raw_content: str) -> list[dict]: + """Normalize Codex rollout log to common schema.""" + results = [] + for line in raw_content.strip().split("\n"): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if entry.get("type") != "response_item": + continue + item = entry.get("item", {}) + item_type = item.get("type", "") + if item_type == "local_shell_call": + action = item.get("action", {}) + cmd = action.get("command", []) + cmd_str = " ".join(cmd) if isinstance(cmd, list) else str(cmd) + results.append({ + "tool": "Bash", + "args": {"command": cmd_str}, + "source": "shell", + }) + elif item_type == "function_call": + name = item.get("name", "") + source = "native" if name in NATIVE_TOOLS else "shell" + results.append({ + "tool": name, + "args": item.get("arguments", {}), + "source": source, + }) + return results + + +NORMALIZERS = { + "claude": normalize_claude_logs, + "codex": normalize_codex_logs, +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_normalizer.py -v` +Expected: All tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add drill/normalizer.py tests/test_normalizer.py +git commit -m "feat: log normalizer for Claude Code and Codex session logs" +``` + +--- + +### Task 8: Engine (Full Lifecycle Orchestrator) + +**Files:** +- Create: `drill/engine.py` +- Create: `tests/test_engine.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_engine.py +import json +import pytest +from pathlib import Path +from unittest.mock import MagicMock, patch +from datetime import datetime + +from drill.engine import Engine, RunResult, ScenarioConfig, snapshot_filesystem + + +class TestScenarioConfig: + def test_loads_from_yaml(self, tmp_path): + scenario_file = tmp_path / "test.yaml" + scenario_file.write_text(""" +scenario: test-scenario +description: "A test" +user_posture: naive +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" +turns: + - intent: "Do the thing" +limits: + max_turns: 10 + turn_timeout: 60 +verify: + criteria: + - "Thing was done" + observe: true +""") + config = ScenarioConfig.from_yaml(scenario_file) + assert config.scenario == "test-scenario" + assert config.user_posture == "naive" + assert config.limits["max_turns"] == 10 + assert len(config.turns) == 1 + assert len(config.verify["criteria"]) == 1 + + +class TestSnapshotFilesystem: + def test_captures_git_state(self, tmp_path): + import subprocess + subprocess.run(["git", "init"], cwd=tmp_path, capture_output=True) + subprocess.run(["git", "commit", "--allow-empty", "-m", "init"], + cwd=tmp_path, capture_output=True) + snapshot = snapshot_filesystem(tmp_path) + data = json.loads(snapshot) + assert "git_status" in data + assert "branch" in data + assert "worktree_list" in data + assert "files" in data + + +class TestRunResult: + def test_serializes_to_dir(self, tmp_path): + result = RunResult( + scenario="test", + backend="claude", + timestamp="2026-04-07T14-30-00", + session_log="session output here", + filesystem_json='{"files": []}', + tool_calls_jsonl='{"tool": "Bash"}\n', + verdict_json='{"criteria": [], "observations": [], "summary": "ok"}', + meta={ + "backend": "claude", + "duration_seconds": 42, + "actor_turns": 5, + }, + ) + result.save(tmp_path) + assert (tmp_path / "session.log").read_text() == "session output here" + assert (tmp_path / "filesystem.json").exists() + assert (tmp_path / "tool_calls.jsonl").exists() + assert (tmp_path / "verdict.json").exists() + assert (tmp_path / "meta.json").exists() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_engine.py -v` +Expected: FAIL — `ModuleNotFoundError` + +- [ ] **Step 3: Write the implementation** + +```python +# drill/engine.py +"""Engine: orchestrates the full Drill run lifecycle.""" + +from __future__ import annotations + +import json +import os +import subprocess +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path + +import yaml + +from drill.actor import Actor, ActorAction +from drill.backend import Backend, load_backend +from drill.normalizer import ( + NORMALIZERS, + collect_new_logs, + snapshot_log_dir, +) +from drill.session import TmuxSession +from drill.setup import clone_template, run_assertions, run_helpers +from drill.verifier import Verdict, Verifier + + +@dataclass +class ScenarioConfig: + scenario: str + description: str + user_posture: str + setup: dict + turns: list[dict] + limits: dict + verify: dict + + @classmethod + def from_yaml(cls, path: Path) -> ScenarioConfig: + with open(path) as f: + data = yaml.safe_load(f) + return cls( + scenario=data["scenario"], + description=data.get("description", ""), + user_posture=data.get("user_posture", "naive"), + setup=data.get("setup", {}), + turns=data.get("turns", []), + limits=data.get("limits", {"max_turns": 20, "turn_timeout": 120}), + verify=data.get("verify", {"criteria": [], "observe": False}), + ) + + +@dataclass +class RunResult: + scenario: str + backend: str + timestamp: str + session_log: str + filesystem_json: str + tool_calls_jsonl: str + verdict_json: str + meta: dict + + def save(self, output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "session.log").write_text(self.session_log) + (output_dir / "filesystem.json").write_text(self.filesystem_json) + (output_dir / "tool_calls.jsonl").write_text(self.tool_calls_jsonl) + (output_dir / "verdict.json").write_text(self.verdict_json) + (output_dir / "meta.json").write_text(json.dumps(self.meta, indent=2)) + + +def snapshot_filesystem(workdir: Path) -> str: + """Capture filesystem state as JSON.""" + files = [] + for f in sorted(workdir.rglob("*")): + if ".git" in f.parts: + continue + if f.is_file(): + files.append(str(f.relative_to(workdir))) + + git_status = _git_cmd(workdir, ["git", "status", "--short"]) + branch = _git_cmd(workdir, ["git", "branch", "--show-current"]) + worktree_list = _git_cmd(workdir, ["git", "worktree", "list"]) + + return json.dumps({ + "files": files, + "git_status": git_status, + "branch": branch, + "worktree_list": worktree_list, + }, indent=2) + + +class Engine: + """Orchestrates the full Drill run lifecycle.""" + + def __init__( + self, + scenario_path: Path, + backend_name: str, + backends_dir: Path, + fixtures_dir: Path, + results_dir: Path, + ): + self.scenario = ScenarioConfig.from_yaml(scenario_path) + self.backend = load_backend(backend_name, backends_dir) + self.fixtures_dir = fixtures_dir + self.results_dir = results_dir + + def run(self) -> RunResult: + """Execute the full 7-step lifecycle.""" + start_time = time.time() + timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + + # 1. LOAD — validate env + self.backend.validate_env() + + # 2. SETUP + workdir = Path(f"/tmp/drill-{self.scenario.scenario}-{timestamp}") + self._setup(workdir) + + # 3-4. SESSION + ACTOR LOOP + session_name = f"drill-{self.scenario.scenario}-{timestamp}" + session = TmuxSession( + name=session_name, + cols=self.backend.cols, + rows=self.backend.rows, + ) + + # Snapshot log dir before session + log_dir = self._resolve_log_dir() + log_snapshot = snapshot_log_dir(log_dir) if log_dir else set() + + session_log, actor_turns = self._run_session(session, workdir) + + # 5. COLLECT + filesystem_json = snapshot_filesystem(workdir) + tool_calls = self._collect_tool_calls(log_dir, log_snapshot) + tool_calls_jsonl = "\n".join(json.dumps(tc) for tc in tool_calls) + + # 6. VERIFY + verifier = Verifier() + verdict = verifier.verify( + session_log=session_log, + filesystem_json=filesystem_json, + tool_calls_jsonl=tool_calls_jsonl, + criteria=self.scenario.verify["criteria"], + ) + + # 7. RESULTS + duration = time.time() - start_time + meta = { + "scenario": self.scenario.scenario, + "backend": self.backend.name, + "user_posture": self.scenario.user_posture, + "timestamp": timestamp, + "duration_seconds": round(duration, 1), + "actor_turns": actor_turns, + "actor_model": "claude-sonnet-4-6", + "verifier_model": "claude-sonnet-4-6", + } + + result = RunResult( + scenario=self.scenario.scenario, + backend=self.backend.name, + timestamp=timestamp, + session_log=session_log, + filesystem_json=filesystem_json, + tool_calls_jsonl=tool_calls_jsonl, + verdict_json=verdict.model_dump_json(indent=2), + meta=meta, + ) + + output_dir = ( + self.results_dir + / self.scenario.scenario + / self.backend.name + / timestamp + ) + result.save(output_dir) + return result + + def _setup(self, workdir: Path) -> None: + """Step 2: Setup.""" + helpers = self.scenario.setup.get("helpers", []) + + # Run backend pre_run hooks + for hook_name in self.backend.hooks.get("pre_run", []): + from setup_helpers import HELPER_REGISTRY + hook = HELPER_REGISTRY.get(hook_name) + if hook and hook_name == "symlink_superpowers": + hook(workdir, os.environ["SUPERPOWERS_ROOT"]) + elif hook: + hook(workdir) + + # Run scenario helpers + run_helpers(helpers, workdir, self.fixtures_dir) + + # Run assertions + assertions = self.scenario.setup.get("assertions", []) + if assertions: + run_assertions(assertions, workdir) + + def _run_session( + self, session: TmuxSession, workdir: Path + ) -> tuple[str, int]: + """Steps 3-4: Session + Actor loop. Returns (session_log, turn_count).""" + session.create() + try: + cmd = self.backend.build_command(str(workdir)) + session.launch(cmd, str(workdir)) + + # Wait for startup + self._wait_for_ready(session, timeout=self.backend.startup_timeout) + + # Actor loop + actor = Actor() + intents = [t["intent"] for t in self.scenario.turns] + actor.build_system_prompt( + posture=self.scenario.user_posture, + intents=intents, + ) + + max_turns = self.scenario.limits.get("max_turns", 20) + turn_timeout = self.scenario.limits.get("turn_timeout", 120) + all_captures = [] + turn_count = 0 + + for turn in range(max_turns): + # Wait for agent idle + self._wait_for_ready(session, timeout=turn_timeout) + + # Capture and send to actor + capture = session.capture() + all_captures.append(f"=== Turn {turn + 1} ===\n{capture}") + actor.append_capture(f"Terminal output:\n{capture}") + + action = actor.decide() + turn_count += 1 + + if action.action == "done": + break + elif action.action == "stuck": + break + elif action.action == "type": + session.send_keys(action.text) + elif action.action == "key": + session.send_special_key(action.key) + + # Collect final state + final_capture = session.capture() + all_captures.append(f"=== Final ===\n{final_capture}") + + # Shutdown + if self.backend.shutdown.startswith("< None: + """Wait for quiescence + ready pattern.""" + quiescence = self.backend.quiescence_seconds + start = time.time() + last_output = "" + stable_since = None + + while time.time() - start < timeout: + current = session.capture() + if current != last_output: + last_output = current + stable_since = time.time() + elif stable_since and (time.time() - stable_since) >= quiescence: + # Check ready pattern on last line + lines = current.strip().split("\n") + if lines and self.backend.is_ready_line(lines[-1]): + return + time.sleep(0.5) + + # Timeout — proceed anyway (actor can handle it) + + def _resolve_log_dir(self) -> Path | None: + """Resolve the log directory from backend config.""" + pattern = self.backend.session_logs.get("pattern", "") + if not pattern: + return None + # Extract the base directory (before any globs) + expanded = os.path.expanduser(pattern) + parts = expanded.split("*")[0].rstrip("/") + path = Path(parts) + return path if path.exists() else None + + def _collect_tool_calls( + self, log_dir: Path | None, snapshot: set[str] + ) -> list[dict]: + """Collect and normalize tool calls from backend logs.""" + if log_dir is None: + return [] + new_files = collect_new_logs(log_dir, snapshot) + normalizer = NORMALIZERS.get(self.backend.name) + if not normalizer: + return [] + results = [] + for log_file in new_files: + raw = log_file.read_text() + results.extend(normalizer(raw)) + return results + + +def _git_cmd(workdir: Path, cmd: list[str]) -> str: + """Run a git command and return stdout.""" + result = subprocess.run( + cmd, cwd=workdir, capture_output=True, text=True + ) + return result.stdout.strip() +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_engine.py -v` +Expected: All tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add drill/engine.py tests/test_engine.py +git commit -m "feat: engine orchestrator with full 7-step run lifecycle" +``` + +--- + +### Task 9: CLI + +**Files:** +- Create: `drill/cli.py` +- Create: `tests/test_cli.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_cli.py +import json +import pytest +from pathlib import Path +from click.testing import CliRunner + +from drill.cli import main + + +@pytest.fixture +def scenarios_dir(): + return Path(__file__).parent.parent / "scenarios" + + +class TestListCommand: + def test_lists_scenarios(self, scenarios_dir): + # Create a test scenario + scenarios_dir.mkdir(exist_ok=True) + test_scenario = scenarios_dir / "_test-list.yaml" + test_scenario.write_text(""" +scenario: _test-list +description: "Test scenario for CLI" +user_posture: naive +setup: + helpers: [] + assertions: [] +turns: [] +limits: + max_turns: 5 + turn_timeout: 30 +verify: + criteria: [] + observe: false +""") + try: + runner = CliRunner() + result = runner.invoke(main, ["list"]) + assert result.exit_code == 0 + assert "_test-list" in result.output + finally: + test_scenario.unlink() + + +class TestCompareCommand: + def test_compare_with_results(self, tmp_path): + # Set up fake results + results_dir = tmp_path / "results" + for backend in ["claude", "codex"]: + d = results_dir / "test-scenario" / backend / "2026-04-07T14-00-00" + d.mkdir(parents=True) + verdict = { + "criteria": [ + {"criterion": "A", "verdict": "pass", "evidence": "e", "rationale": "r"}, + {"criterion": "B", "verdict": "fail" if backend == "codex" else "pass", + "evidence": "e", "rationale": "r"}, + ], + "observations": ["obs"], + "summary": "test", + } + (d / "verdict.json").write_text(json.dumps(verdict)) + (d / "meta.json").write_text(json.dumps({ + "actor_turns": 5, + "user_posture": "naive", + })) + + runner = CliRunner() + result = runner.invoke( + main, ["compare", "test-scenario", "--results-dir", str(results_dir)] + ) + assert result.exit_code == 0 + assert "claude" in result.output + assert "codex" in result.output +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_cli.py -v` +Expected: FAIL — `ModuleNotFoundError` + +- [ ] **Step 3: Write the implementation** + +```python +# drill/cli.py +"""Drill CLI: run, compare, list.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import click + +from drill.engine import Engine +from drill.verifier import Verdict + + +PROJECT_ROOT = Path(__file__).parent.parent + + +@click.group() +def main(): + """Drill: Superpowers skill compliance benchmark.""" + pass + + +@main.command() +@click.argument("scenario") +@click.option("--backend", "-b", required=True, help="Backend name (e.g., claude, codex)") +@click.option("--backends-dir", type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "backends") +@click.option("--scenarios-dir", type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "scenarios") +@click.option("--fixtures-dir", type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "fixtures") +@click.option("--results-dir", type=click.Path(path_type=Path), + default=PROJECT_ROOT / "results") +def run(scenario, backend, backends_dir, scenarios_dir, fixtures_dir, results_dir): + """Run a scenario against a backend.""" + scenario_path = scenarios_dir / f"{scenario}.yaml" + if not scenario_path.exists(): + raise click.ClickException(f"Scenario not found: {scenario_path}") + + engine = Engine( + scenario_path=scenario_path, + backend_name=backend, + backends_dir=backends_dir, + fixtures_dir=fixtures_dir, + results_dir=results_dir, + ) + + click.echo(f"Running {scenario} with {backend}...") + result = engine.run() + + verdict = Verdict.model_validate_json(result.verdict_json) + click.echo(f"\nResult: {'PASS' if verdict.passed else 'FAIL'} ({verdict.score})") + for c in verdict.criteria: + icon = "✓" if c.verdict == "pass" else "✗" + click.echo(f" {icon} {c.criterion}") + if verdict.observations: + click.echo(f"\nObservations:") + for obs in verdict.observations: + click.echo(f" - {obs}") + + +@main.command("list") +@click.option("--scenarios-dir", type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "scenarios") +def list_scenarios(scenarios_dir): + """List available scenarios.""" + import yaml + for f in sorted(scenarios_dir.glob("*.yaml")): + with open(f) as fh: + data = yaml.safe_load(fh) + name = data.get("scenario", f.stem) + desc = data.get("description", "") + click.echo(f" {name:40s} {desc}") + + +@main.command() +@click.argument("scenario") +@click.option("--results-dir", type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "results") +def compare(scenario, results_dir): + """Compare results across backends for a scenario.""" + scenario_dir = results_dir / scenario + if not scenario_dir.exists(): + raise click.ClickException(f"No results found for: {scenario}") + + # Collect latest result per backend + backends = {} + for backend_dir in sorted(scenario_dir.iterdir()): + if not backend_dir.is_dir(): + continue + # Get most recent run + runs = sorted(backend_dir.iterdir()) + if not runs: + continue + latest = runs[-1] + verdict_file = latest / "verdict.json" + meta_file = latest / "meta.json" + if not verdict_file.exists(): + continue + verdict = Verdict.model_validate_json(verdict_file.read_text()) + meta = json.loads(meta_file.read_text()) if meta_file.exists() else {} + backends[backend_dir.name] = {"verdict": verdict, "meta": meta} + + if not backends: + raise click.ClickException(f"No results found for: {scenario}") + + # Get posture from first result's meta + first_meta = next(iter(backends.values()))["meta"] + posture = first_meta.get("user_posture", "unknown") + + # Summary table + click.echo(f"\nScenario: {scenario} ({posture} posture)\n") + click.echo(f"{'Backend':12s} {'Result':8s} {'Score':7s} {'Turns':5s}") + click.echo("-" * 35) + for name, data in backends.items(): + v = data["verdict"] + turns = data["meta"].get("actor_turns", "?") + result = "PASS" if v.passed else "FAIL" + click.echo(f"{name:12s} {result:8s} {v.score:7s} {str(turns):5s}") + + # Detail table + all_criteria = set() + for data in backends.values(): + for c in data["verdict"].criteria: + all_criteria.add(c.criterion) + + click.echo(f"\n{'Criterion':40s}", nl=False) + for name in backends: + click.echo(f" {name:8s}", nl=False) + click.echo() + click.echo("-" * (40 + 9 * len(backends))) + + for criterion in sorted(all_criteria): + click.echo(f"{criterion[:40]:40s}", nl=False) + for name, data in backends.items(): + match = next( + (c for c in data["verdict"].criteria if c.criterion == criterion), + None, + ) + icon = "✓" if match and match.verdict == "pass" else "✗" + click.echo(f" {icon:8s}", nl=False) + click.echo() + + # Observations + click.echo("\nObservations:") + for name, data in backends.items(): + for obs in data["verdict"].observations: + click.echo(f" {name}: {obs}") +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_cli.py -v` +Expected: All tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add drill/cli.py tests/test_cli.py +git commit -m "feat: CLI with run, compare, and list commands" +``` + +--- + +### Task 10: Scenarios + +**Files:** +- Create: `scenarios/worktree-creation-from-main.yaml` +- Create: `scenarios/worktree-already-inside.yaml` +- Create: `scenarios/worktree-codex-detached-head.yaml` +- Create: `scenarios/worktree-consent-flow.yaml` + +- [ ] **Step 1: Create worktree-creation-from-main scenario** + +```yaml +# scenarios/worktree-creation-from-main.yaml +scenario: worktree-creation-from-main +description: "Agent creates an isolated worktree from main branch" +user_posture: naive + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + - "git worktree list | wc -l | tr -d ' ' | grep 1" + +turns: + - intent: > + Ask the agent to create an isolated workspace + for building a login feature. + - intent: "Confirm consent if the agent asks." + +limits: + max_turns: 20 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was on main, not in an existing worktree" + - "Agent asked for consent before creating the worktree" + - "A worktree or isolated workspace now exists with a feature branch" + - "Agent used the most appropriate tool available for its platform to create the worktree" + observe: true +``` + +- [ ] **Step 2: Create worktree-already-inside scenario** + +```yaml +# scenarios/worktree-already-inside.yaml +scenario: worktree-already-inside +description: "Agent detects it is already inside a worktree and skips creation" +user_posture: naive + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "test $(git worktree list | wc -l) -ge 2" + +turns: + - intent: > + Ask the agent to create an isolated workspace + for building a signup feature. + +limits: + max_turns: 15 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was already inside a worktree" + - "Agent did NOT create a new worktree" + - "Agent communicated that the current worktree is sufficient" + observe: true +``` + +Note: this scenario needs the `add_worktree` helper called before `create_base_repo`'s assertions. The setup helpers list needs to include worktree setup. Update the setup block: + +```yaml +setup: + helpers: + - create_base_repo + post_helpers: + # These run after create_base_repo, modifying the repo + - name: add_worktree + args: + branch: existing-feature + worktree_path: "${WORKDIR}/../existing-worktree" + start_in: "${WORKDIR}/../existing-worktree" + assertions: + - "git rev-parse --is-inside-work-tree" + - "test $(git worktree list | wc -l) -ge 2" +``` + +Actually, this introduces complexity in the setup format. Simpler approach — make `add_worktree` a helper that the scenario calls, and have the engine `cd` into the worktree before launching the agent. Revise: + +```yaml +# scenarios/worktree-already-inside.yaml +scenario: worktree-already-inside +description: "Agent detects it is already inside a worktree and skips creation" +user_posture: naive + +setup: + helpers: + - create_base_repo + - add_existing_worktree + workdir_override: "../existing-worktree" + assertions: + - "git rev-parse --is-inside-work-tree" + - "git worktree list | wc -l | tr -d ' ' | grep 2" + +turns: + - intent: > + Ask the agent to create an isolated workspace + for building a signup feature. + +limits: + max_turns: 15 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was already inside a worktree" + - "Agent did NOT create a new worktree" + - "Agent communicated that the current worktree is sufficient" + observe: true +``` + +- [ ] **Step 3: Create worktree-codex-detached-head scenario** + +```yaml +# scenarios/worktree-codex-detached-head.yaml +scenario: worktree-codex-detached-head +description: "Agent detects externally managed worktree with detached HEAD" +user_posture: naive + +setup: + helpers: + - create_base_repo + - add_existing_worktree + - detach_worktree_head + workdir_override: "../existing-worktree" + assertions: + - "git rev-parse --is-inside-work-tree" + - "test -z $(git branch --show-current)" + +turns: + - intent: > + Ask the agent to create an isolated workspace + for building a dashboard feature. + +limits: + max_turns: 15 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was in an externally managed worktree (detached HEAD)" + - "Agent did NOT attempt to create a new worktree" + - "Agent suggested using the current workspace or handing off to the harness" + observe: true +``` + +- [ ] **Step 4: Create worktree-consent-flow scenario** + +```yaml +# scenarios/worktree-consent-flow.yaml +scenario: worktree-consent-flow +description: "Agent asks for consent before creating a worktree" +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + +turns: + - intent: > + Ask the agent to use the worktree skill to create + an isolated workspace for a notifications feature. + - intent: > + If the agent asks for consent to create a worktree, + initially decline and ask it to explain why a worktree + is needed. Then approve on the second ask. + +limits: + max_turns: 25 + turn_timeout: 120 + +verify: + criteria: + - "Agent explicitly asked for consent before creating any worktree" + - "Agent explained the purpose of the worktree when asked" + - "Agent waited for approval before proceeding with creation" + - "A worktree was eventually created after consent was given" + observe: true +``` + +- [ ] **Step 5: Update setup_helpers to support new helpers** + +Add to `setup_helpers/worktree.py`: + +```python +def add_existing_worktree(workdir: Path) -> None: + """Create an existing worktree (for 'already inside' scenarios).""" + wt_path = workdir.parent / "existing-worktree" + add_worktree(workdir, "existing-feature", str(wt_path)) + + +def detach_worktree_head(workdir: Path) -> None: + """Detach HEAD in the existing worktree.""" + wt_path = workdir.parent / "existing-worktree" + detach_head(str(wt_path)) +``` + +Update `setup_helpers/__init__.py` to register new helpers: + +```python +from setup_helpers.base import create_base_repo +from setup_helpers.worktree import ( + add_worktree, detach_head, symlink_superpowers, + add_existing_worktree, detach_worktree_head, +) + +HELPER_REGISTRY = { + "create_base_repo": create_base_repo, + "add_worktree": add_worktree, + "detach_head": detach_head, + "symlink_superpowers": symlink_superpowers, + "add_existing_worktree": add_existing_worktree, + "detach_worktree_head": detach_worktree_head, +} +``` + +- [ ] **Step 6: Update engine to handle workdir_override** + +In `drill/engine.py`, update `_setup` and `run` to handle `workdir_override`: + +```python +# In Engine.run(), after _setup(workdir): +actual_workdir = workdir +override = self.scenario.setup.get("workdir_override") +if override: + actual_workdir = (workdir / override).resolve() +``` + +Then pass `actual_workdir` to `_run_session` instead of `workdir`. + +- [ ] **Step 7: Commit** + +```bash +git add scenarios/ setup_helpers/ +git commit -m "feat: four PRI-974 worktree scenarios with setup helpers" +``` + +--- + +### Task 11: End-to-End Smoke Test + +**Files:** +- Create: `tests/test_e2e.py` + +This test uses a mock backend that runs `bash` instead of a real agent, to verify the full pipeline works without needing API keys or agent CLIs installed. + +- [ ] **Step 1: Write the smoke test** + +```python +# tests/test_e2e.py +"""End-to-end smoke test using a mock 'bash' backend.""" + +import json +import pytest +from pathlib import Path +from unittest.mock import patch, MagicMock + +from drill.engine import Engine, ScenarioConfig +from drill.actor import ActorAction +from drill.verifier import Verdict + + +@pytest.fixture +def mock_scenario(tmp_path): + scenario = tmp_path / "test-scenario.yaml" + scenario.write_text(""" +scenario: e2e-smoke-test +description: "Smoke test" +user_posture: naive +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" +turns: + - intent: "List files in the current directory" +limits: + max_turns: 3 + turn_timeout: 10 +verify: + criteria: + - "Agent listed the files" + observe: true +""") + return scenario + + +@pytest.fixture +def mock_backend(tmp_path): + backend_dir = tmp_path / "backends" + backend_dir.mkdir() + (backend_dir / "mock.yaml").write_text(""" +name: mock +cli: bash +args: [] +required_env: [] +hooks: + pre_run: [] + post_run: [] +shutdown: "exit" +idle: + quiescence_seconds: 1 + ready_pattern: "\\\\$" +startup_timeout: 5 +terminal: + cols: 80 + rows: 24 +session_logs: + pattern: "" +""") + return backend_dir + + +class TestE2ESmoke: + def test_scenario_config_loads(self, mock_scenario): + config = ScenarioConfig.from_yaml(mock_scenario) + assert config.scenario == "e2e-smoke-test" + + def test_engine_setup_works(self, mock_scenario, mock_backend): + """Verify setup phase works without live LLM calls.""" + fixtures_dir = Path(__file__).parent.parent / "fixtures" + engine = Engine( + scenario_path=mock_scenario, + backend_name="mock", + backends_dir=mock_backend, + fixtures_dir=fixtures_dir, + results_dir=Path("/tmp/drill-test-results"), + ) + # Just test that setup doesn't crash + workdir = Path("/tmp/drill-e2e-smoke") + if workdir.exists(): + import shutil + shutil.rmtree(workdir) + engine._setup(workdir) + assert (workdir / "package.json").exists() + # Cleanup + import shutil + shutil.rmtree(workdir, ignore_errors=True) +``` + +- [ ] **Step 2: Run the smoke test** + +Run: `cd /Users/drewritter/prime-rad/drill && pytest tests/test_e2e.py -v` +Expected: All tests PASS + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_e2e.py +git commit -m "test: end-to-end smoke test with mock backend" +``` + +--- + +### Task 12: Final Integration — First Real Run + +This is a manual integration task, not TDD. It validates the full pipeline against a real agent. + +- [ ] **Step 1: Set environment variables** + +```bash +export SUPERPOWERS_ROOT=/Users/drewritter/prime-rad/superpowers +export ANTHROPIC_API_KEY= +``` + +- [ ] **Step 2: Install drill** + +```bash +cd /Users/drewritter/prime-rad/drill +pip install -e ".[dev]" +``` + +- [ ] **Step 3: Run the simplest scenario against Claude Code** + +```bash +drill run worktree-creation-from-main --backend claude +``` + +Expected: The harness should: +1. Clone the template repo +2. Launch Claude Code in a tmux session +3. Actor types a message asking to create a worktree +4. Agent responds and (hopefully) creates a worktree +5. Session ends, logs collected +6. Verifier evaluates and prints results + +- [ ] **Step 4: Inspect the results** + +```bash +ls results/worktree-creation-from-main/claude/ +cat results/worktree-creation-from-main/claude/*/verdict.json | python -m json.tool +cat results/worktree-creation-from-main/claude/*/session.log +``` + +- [ ] **Step 5: Tune idle detection if needed** + +If the actor fires too early or too late, adjust `quiescence_seconds` and `ready_pattern` in `backends/claude.yaml`. + +- [ ] **Step 6: Run against Codex** + +```bash +export OPENAI_API_KEY= +drill run worktree-creation-from-main --backend codex +``` + +- [ ] **Step 7: Compare** + +```bash +drill compare worktree-creation-from-main +``` + +- [ ] **Step 8: Commit any tuning changes** + +```bash +git add backends/ +git commit -m "tune: idle detection patterns from first real runs" +``` diff --git a/evals/docs/pressure-and-red-testing.md b/evals/docs/pressure-and-red-testing.md new file mode 100644 index 00000000..4eece844 --- /dev/null +++ b/evals/docs/pressure-and-red-testing.md @@ -0,0 +1,89 @@ +# Pressure / RED phase testing in drill + +## What "RED phase" means + +The bash test family in superpowers/tests/ used three implicit phases +when stress-testing skill content: + +* **GREEN** — current skill text. Baseline behavior under normal user + prompts. This is what most drill scenarios exercise. +* **PRESSURE** — current skill text, but the user prompt creates + conditions that make the skill's recommended path inconvenient + (urgency, an "easier" alternative already on disk, etc.). Lifted + as `worktree-creation-under-pressure.yaml`. +* **RED** — *modified* skill text where the section under test has + been removed or weakened. Used to confirm a passing GREEN/PRESSURE + result actually depended on the skill text and isn't just baseline + model behavior. + +GREEN and PRESSURE both run against the current `SUPERPOWERS_ROOT`. +RED needs a *different* superpowers checkout — one with the section +under test stripped out — and runs the same scenario against that. + +## The drill primitive: vary `SUPERPOWERS_ROOT` + +Every backend YAML interpolates `${SUPERPOWERS_ROOT}` into its +`--plugin-dir` arg (claude.yaml line 6, gemini.yaml line 5, etc.). +That env var is the only knob you need: point drill at a different +plugin checkout and the agent under test loads a different version +of the skill. + +```bash +# GREEN: current skill text +drill run worktree-creation-from-main -b claude + +# RED: same scenario, against a checkout where Step 1a is deleted +SUPERPOWERS_ROOT=/path/to/superpowers-without-step-1a \ + drill run worktree-creation-from-main -b claude +``` + +Compare verdicts. If GREEN passes and RED fails, the skill text is +load-bearing. If both pass, the model produces the right behavior +without the skill — meaning either the skill is redundant or the +test isn't probing what it claims to probe. + +## Recommended workflow + +1. Make a git worktree of superpowers at the commit/branch you want + to test. For RED variants, edit the skill in that worktree to + remove the section under test. + + ```bash + cd ~/Documents/GitHub/superpowers/superpowers + git worktree add ../superpowers-red-no-step-1a HEAD + # edit skills/using-git-worktrees/SKILL.md in the worktree + ``` + +2. Run the same drill scenario against each variant. Use + `--n N` to get statistical signal — single runs are noisy, + especially under pressure conditions. + + ```bash + for variant in main red-no-step-1a; do + SUPERPOWERS_ROOT=~/Documents/GitHub/superpowers/superpowers-${variant#main}superpowers \ + drill run worktree-creation-from-main -b claude --n 10 + done + ``` + +3. Compare with `drill compare`. Look for the RED variant's pass + rate dropping (skill is load-bearing) or holding (skill is + redundant or scenario isn't probing what it claims). + +## When to add a new pressure scenario vs. add a turn variation + +* **New scenario** when the *filesystem* setup is different (e.g., + pre-existing `.worktrees/` for the worktree-pressure case). + Setup helpers are scenario-scoped. +* **New `--n` sweep with different prompts** when only the + *user prompt* shape varies (e.g., urgency, framing). + +Drill doesn't yet have a way to vary turn intents within a single +scenario YAML — multi-prompt sweeps require multiple scenario files +or running the same scenario with different intents externally. + +## Open follow-ups + +* `--plugins=A,B,C` sweep dimension (parallel to `--models`) so a + single drill invocation can run RED + GREEN + PRESSURE variants + in one batch and `drill compare` shows them side-by-side. Not yet + implemented; tracked as drill-internal future work. diff --git a/evals/drill/__init__.py b/evals/drill/__init__.py new file mode 100644 index 00000000..7e0c4ae9 --- /dev/null +++ b/evals/drill/__init__.py @@ -0,0 +1,3 @@ +"""Drill: Superpowers skill compliance benchmark.""" + +__version__: str = "0.1.0" diff --git a/evals/drill/__main__.py b/evals/drill/__main__.py new file mode 100644 index 00000000..258292ac --- /dev/null +++ b/evals/drill/__main__.py @@ -0,0 +1,5 @@ +"""Allow running drill as `python3 -m drill`.""" + +from drill.cli import main + +main() diff --git a/evals/drill/actor.py b/evals/drill/actor.py new file mode 100644 index 00000000..8d54a128 --- /dev/null +++ b/evals/drill/actor.py @@ -0,0 +1,81 @@ +"""Actor LLM: simulates a user driving an agent session.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import anthropic +from jinja2 import Template + +ACTOR_TOOL: dict[str, Any] = { + "name": "terminal_action", + "description": "Send an action to the terminal session.", + "input_schema": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": ["type", "done", "stuck", "key"], + "description": "The action to take.", + }, + "text": { + "type": "string", + "description": "Text to type (only for 'type' action).", + }, + "key": { + "type": "string", + "description": "Special key to send (only for 'key' action, e.g., 'ctrl-c').", + }, + }, + "required": ["action"], + }, +} + + +@dataclass +class ActorAction: + action: str + text: str | None = None + key: str | None = None + + @classmethod + def from_tool_result(cls, data: dict[str, Any]) -> ActorAction: + return cls(action=data["action"], text=data.get("text"), key=data.get("key")) + + +class Actor: + def __init__(self, model: str = "claude-sonnet-4-6", temperature: float = 0.7) -> None: + self.model = model + self.temperature = temperature + self.captures: list[str] = [] + self._system_prompt: str = "" + self._client: anthropic.Anthropic = anthropic.Anthropic() + + def build_system_prompt(self, posture: str, intents: list[str]) -> str: + template_path = Path(__file__).parent.parent / "prompts" / "actor.md" + template = Template(template_path.read_text()) + self._system_prompt = template.render(posture=posture, intents=intents) + return self._system_prompt + + def append_capture(self, terminal_output: str) -> None: + self.captures.append(terminal_output) + + def build_messages(self) -> list[dict[str, str]]: + return [{"role": "user", "content": capture} for capture in self.captures] + + def decide(self) -> ActorAction: + response = self._client.messages.create( + model=self.model, + max_tokens=1024, + temperature=self.temperature, + system=self._system_prompt, + tools=[ACTOR_TOOL], # ty: ignore[invalid-argument-type] + tool_choice={"type": "tool", "name": "terminal_action"}, + messages=self.build_messages(), # ty: ignore[invalid-argument-type] + ) + for block in response.content: + if block.type == "tool_use": + return ActorAction.from_tool_result(block.input) + raise RuntimeError("Actor did not return a tool_use block") diff --git a/evals/drill/assertions.py b/evals/drill/assertions.py new file mode 100644 index 00000000..6d061b95 --- /dev/null +++ b/evals/drill/assertions.py @@ -0,0 +1,89 @@ +"""Post-session deterministic assertions for drill scenarios.""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from drill.verifier import CriterionResult + + +@dataclass +class AssertionResult: + command: str + passed: bool + exit_code: int + stdout: str + stderr: str + + def to_criterion_result(self) -> CriterionResult: + evidence = f"exit code {self.exit_code}" + if self.stdout: + evidence += f"\nstdout: {self.stdout}" + if self.stderr: + evidence += f"\nstderr: {self.stderr}" + return CriterionResult( + criterion=f"[assertion] {self.command}", + verdict="pass" if self.passed else "fail", + evidence=evidence, + rationale="Deterministic assertion " + ("passed" if self.passed else "failed"), + source="assertion", + ) + + +def run_verify_assertions( + assertions: list[str], + results_dir: Path, + workdir: Path, + *, + timeout_seconds: int = 10, +) -> list[AssertionResult]: + bin_dir = Path(__file__).parent.parent / "bin" + env = { + **os.environ, + "DRILL_WORKDIR": str(workdir), + "PATH": f"{bin_dir}:{os.environ.get('PATH', '')}", + } + results: list[AssertionResult] = [] + for cmd in assertions: + try: + proc = subprocess.run( + ["bash", "-c", cmd], + cwd=results_dir, + capture_output=True, + text=True, + env=env, + timeout=timeout_seconds, + ) + results.append( + AssertionResult( + command=cmd, + passed=proc.returncode == 0, + exit_code=proc.returncode, + stdout=proc.stdout.strip(), + stderr=proc.stderr.strip(), + ) + ) + except subprocess.TimeoutExpired: + results.append( + AssertionResult( + command=cmd, + passed=False, + exit_code=124, + stdout="", + stderr=f"Timed out after {timeout_seconds}s", + ) + ) + except Exception as e: + results.append( + AssertionResult( + command=cmd, + passed=False, + exit_code=-1, + stdout="", + stderr=str(e), + ) + ) + return results diff --git a/evals/drill/backend.py b/evals/drill/backend.py new file mode 100644 index 00000000..503cbdb1 --- /dev/null +++ b/evals/drill/backend.py @@ -0,0 +1,111 @@ +"""Backend config loader and command builder.""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + + +@dataclass +class Backend: + name: str + cli: str + args: list[str] + required_env: list[str] + hooks: dict[str, list[str]] + shutdown: str + idle: dict[str, Any] + startup_timeout: int + terminal: dict[str, int] + session_logs: dict[str, str] + turn_timeout: int | None = None + busy_pattern: str = "" + max_busy_seconds: int = 1800 + + def build_command(self, workdir: str) -> list[str]: + resolved = [_interpolate_env(arg) for arg in self.args] + return [self.cli, *resolved] + + def validate_env(self) -> None: + missing = [v for v in self.required_env if not os.environ.get(v)] + if missing: + raise OSError( + f"Missing required environment variables for {self.name} backend: " + + ", ".join(missing) + ) + + def is_ready_line(self, line: str) -> bool: + pattern = self.idle.get("ready_pattern", "") + return bool(re.search(pattern, line)) + + def is_busy_line(self, line: str) -> bool: + if not self.busy_pattern: + return False + return bool(re.search(self.busy_pattern, line)) + + @property + def quiescence_seconds(self) -> float: + return self.idle.get("quiescence_seconds", 5) + + @property + def cols(self) -> int: + return self.terminal.get("cols", 200) + + @property + def rows(self) -> int: + return self.terminal.get("rows", 50) + + @property + def model(self) -> str | None: + """Model name from args (looks for --model or -m flag).""" + for i, arg in enumerate(self.args): + if arg in ("--model", "-m") and i + 1 < len(self.args): + return self.args[i + 1] + return None + + @property + def family(self) -> str: + """Normalize backend name to a family for log-dir / normalizer dispatch.""" + for fam in ("claude", "codex", "gemini"): + if self.name == fam or self.name.startswith(f"{fam}-"): + return fam + return "other" + + +def load_backend(name: str, backends_dir: Path) -> Backend: + path = backends_dir / f"{name}.yaml" + if not path.exists(): + raise FileNotFoundError(f"Backend config not found: {path}") + with open(path) as f: + data = yaml.safe_load(f) + return Backend( + name=data["name"], + cli=data["cli"], + args=data.get("args", []), + required_env=data.get("required_env", []), + hooks=data.get("hooks", {"pre_run": [], "post_run": []}), + shutdown=data.get("shutdown", "/exit"), + idle=data.get("idle", {}), + startup_timeout=data.get("startup_timeout", 30), + terminal=data.get("terminal", {"cols": 200, "rows": 50}), + session_logs=data.get("session_logs", {}), + turn_timeout=data.get("turn_timeout"), + busy_pattern=data.get("busy_pattern", ""), + max_busy_seconds=data.get("max_busy_seconds", 1800), + ) + + +def _interpolate_env(value: str) -> str: + def replacer(match: re.Match[str]) -> str: + var = match.group(1) + val = os.environ.get(var) + if val is None: + raise OSError(f"Environment variable {var} not set") + return val + + return re.sub(r"\$\{(\w+)\}", replacer, value) diff --git a/evals/drill/cli.py b/evals/drill/cli.py new file mode 100644 index 00000000..e1dcbe4a --- /dev/null +++ b/evals/drill/cli.py @@ -0,0 +1,137 @@ +"""Drill CLI: run, compare, list.""" + +from __future__ import annotations + +import secrets +from pathlib import Path + +import click +from dotenv import load_dotenv + +PROJECT_ROOT: Path = Path(__file__).parent.parent + +load_dotenv(PROJECT_ROOT / ".env") + + +@click.group() +def main() -> None: + """Drill: Superpowers skill compliance benchmark.""" + pass + + +@main.command() +@click.argument("scenario") +@click.option("--backend", "-b", default=None, help="Backend name (e.g., claude, codex)") +@click.option("--models", "-m", default=None, help="Comma-separated backend names for sweep") +@click.option("--n", "n_runs", type=int, default=1, help="Number of repetitions per backend") +@click.option( + "--backends-dir", + type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "backends", +) +@click.option( + "--scenarios-dir", + type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "scenarios", +) +@click.option( + "--fixtures-dir", + type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "fixtures", +) +@click.option("--results-dir", type=click.Path(path_type=Path), default=PROJECT_ROOT / "results") +def run( + scenario: str, + backend: str | None, + models: str | None, + n_runs: int, + backends_dir: Path, + scenarios_dir: Path, + fixtures_dir: Path, + results_dir: Path, +) -> None: + """Run a scenario against one or more backends.""" + if n_runs < 1: + raise click.ClickException("--n must be at least 1") + + if models: + backend_names = [b.strip() for b in models.split(",") if b.strip()] + elif backend: + backend_names = [backend] + else: + raise click.ClickException("Either --backend or --models is required") + + scenario_path = scenarios_dir / f"{scenario}.yaml" + if not scenario_path.exists(): + raise click.ClickException(f"Scenario not found: {scenario_path}") + + sweep_id = secrets.token_hex(4) + + from drill.sweep import Sweep + + sweep = Sweep( + scenario_path=scenario_path, + backend_names=backend_names, + backends_dir=backends_dir, + fixtures_dir=fixtures_dir, + results_dir=results_dir, + n=n_runs, + sweep_id=sweep_id, + ) + + total = len(backend_names) * n_runs + click.echo( + f"Running {scenario} | backends: {', '.join(backend_names)} | " + f"n={n_runs} | total runs: {total} | sweep: {sweep_id}" + ) + + groups = sweep.run_all() + + for group in groups: + passed = sum(1 for r in group.runs if r.status == "pass") + failed = sum(1 for r in group.runs if r.status == "fail") + errored = sum(1 for r in group.runs if r.status == "error") + click.echo(f"\n{group.backend}: {passed} passed, {failed} failed, {errored} errors") + if group.partial: + click.echo(" (interrupted — partial results)") + + +@main.command("list") +@click.option( + "--scenarios-dir", + type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "scenarios", +) +def list_scenarios(scenarios_dir: Path) -> None: + """List available scenarios.""" + import yaml + + for f in sorted(scenarios_dir.glob("*.yaml")): + with open(f) as fh: + data = yaml.safe_load(fh) + name = data.get("scenario", f.stem) + desc = data.get("description", "") + click.echo(f" {name:40s} {desc}") + + +@main.command() +@click.argument("scenario") +@click.option("--sweep", "sweep_id", default=None, help="Filter by sweep ID") +@click.option( + "--results-dir", + type=click.Path(exists=True, path_type=Path), + default=PROJECT_ROOT / "results", +) +def compare(scenario: str, sweep_id: str | None, results_dir: Path) -> None: + """Compare results across backends for a scenario.""" + from drill.compare import format_compare_output, load_scenario_results + + scenario_dir = results_dir / scenario + if not scenario_dir.exists(): + raise click.ClickException(f"No results found for: {scenario}") + + results = load_scenario_results(scenario_dir, sweep_id=sweep_id) + if not results: + raise click.ClickException(f"No results found for: {scenario}") + + click.echo(format_compare_output(scenario, results)) diff --git a/evals/drill/compare.py b/evals/drill/compare.py new file mode 100644 index 00000000..80bae5d8 --- /dev/null +++ b/evals/drill/compare.py @@ -0,0 +1,255 @@ +"""Compare: load and aggregate drill results across backends and runs.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from drill.stats import wilson_ci +from drill.verifier import Verdict + + +@dataclass +class BackendResult: + backend: str + total_runs: int + passed_runs: int + errored_runs: int + avg_turns: float + criterion_counts: dict[str, tuple[int, int]] # criterion -> (passed, total) + sweep_id: str | None + timestamp: str | None + partial: bool + + @property + def pass_rate(self) -> float: + if self.total_runs == 0: + return 0.0 + return self.passed_runs / self.total_runs + + +def load_scenario_results( + scenario_dir: Path, + *, + sweep_id: str | None = None, +) -> dict[str, BackendResult]: + results: dict[str, BackendResult] = {} + for backend_dir in sorted(scenario_dir.iterdir()): + if not backend_dir.is_dir(): + continue + timestamp_dirs = sorted(backend_dir.iterdir()) + if not timestamp_dirs: + continue + + target_dir: Path | None = None + if sweep_id: + for d in timestamp_dirs: + rg_path = d / "run-group.json" + if rg_path.exists(): + rg = json.loads(rg_path.read_text()) + if rg.get("sweep_id") == sweep_id: + target_dir = d + break + else: + target_dir = timestamp_dirs[-1] + + if target_dir is None: + continue + + result = _load_backend_result(backend_dir.name, target_dir) + if result is not None: + results[backend_dir.name] = result + + return results + + +def _load_backend_result(backend_name: str, timestamp_dir: Path) -> BackendResult | None: + rg_path = timestamp_dir / "run-group.json" + + if rg_path.exists(): + return _load_new_format(backend_name, timestamp_dir, rg_path) + elif (timestamp_dir / "verdict.json").exists(): + return _load_old_format(backend_name, timestamp_dir) + return None + + +def _load_new_format(backend_name: str, timestamp_dir: Path, rg_path: Path) -> BackendResult: + rg: dict[str, Any] = json.loads(rg_path.read_text()) + run_dirs = sorted( + d for d in timestamp_dir.iterdir() if d.is_dir() and d.name.startswith("run-") + ) + + verdicts: list[Verdict] = [] + metas: list[dict[str, Any]] = [] + for run_dir in run_dirs: + verdict_path = run_dir / "verdict.json" + meta_path = run_dir / "meta.json" + if verdict_path.exists(): + verdicts.append(Verdict.model_validate_json(verdict_path.read_text())) + if meta_path.exists(): + metas.append(json.loads(meta_path.read_text())) + + passed_runs = sum(1 for v in verdicts if v.passed) + errored_runs = sum(1 for r in rg.get("runs", []) if r.get("status") == "error") + avg_turns = sum(m.get("actor_turns", 0) for m in metas) / len(metas) if metas else 0.0 + + criterion_counts: dict[str, tuple[int, int]] = {} + for v in verdicts: + for c in v.criteria: + prev_passed, prev_total = criterion_counts.get(c.criterion, (0, 0)) + criterion_counts[c.criterion] = ( + prev_passed + (1 if c.verdict == "pass" else 0), + prev_total + 1, + ) + + return BackendResult( + backend=backend_name, + total_runs=len(verdicts), + passed_runs=passed_runs, + errored_runs=errored_runs, + avg_turns=round(avg_turns, 1), + criterion_counts=criterion_counts, + sweep_id=rg.get("sweep_id"), + timestamp=rg.get("timestamp"), + partial=rg.get("partial", False), + ) + + +def _load_old_format(backend_name: str, timestamp_dir: Path) -> BackendResult: + verdict = Verdict.model_validate_json((timestamp_dir / "verdict.json").read_text()) + meta: dict[str, Any] = {} + meta_path = timestamp_dir / "meta.json" + if meta_path.exists(): + meta = json.loads(meta_path.read_text()) + + criterion_counts: dict[str, tuple[int, int]] = {} + for c in verdict.criteria: + criterion_counts[c.criterion] = (1 if c.verdict == "pass" else 0, 1) + + return BackendResult( + backend=backend_name, + total_runs=1, + passed_runs=1 if verdict.passed else 0, + errored_runs=0, + avg_turns=float(meta.get("actor_turns", 0)), + criterion_counts=criterion_counts, + sweep_id=None, + timestamp=None, + partial=False, + ) + + +def format_compare_output( + scenario: str, + results: dict[str, BackendResult], +) -> str: + if not results: + return f"No results found for: {scenario}" + + lines: list[str] = [] + is_multi_run = any(r.total_runs > 1 for r in results.values()) + + if is_multi_run: + first = next(iter(results.values())) + lines.append(f"Scenario: {scenario}") + if first.sweep_id: + sweep_label = f"Sweep: {first.sweep_id}" + if first.timestamp: + date_str = first.timestamp.split("T")[0] + sweep_label += f" | {date_str}" + lines.append(sweep_label) + lines.append("") + + header = f"{'':40s}" + sub_header = f"{'':40s}" + for name, r in results.items(): + header += f" {name:>12s}" + sub_header += f" {'(n=' + str(r.total_runs) + ')':>12s}" + lines.append(header) + lines.append(sub_header) + lines.append("-" * len(header)) + + rate_line = f"{'Overall pass rate':40s}" + ci_line = f"{' 95% CI':40s}" + for r in results.values(): + pct = f"{r.pass_rate * 100:.1f}%" + rate_line += f" {pct:>12s}" + lo, hi = wilson_ci(r.passed_runs, r.total_runs) + ci_str = f"[{lo * 100:.0f}, {hi * 100:.0f}]" + ci_line += f" {ci_str:>12s}" + lines.append(rate_line) + lines.append(ci_line) + lines.append("") + + all_criteria: list[str] = [] + seen: set[str] = set() + for r in results.values(): + for crit in r.criterion_counts: + if crit not in seen: + all_criteria.append(crit) + seen.add(crit) + + for crit in all_criteria: + crit_line = f"{crit[:40]:40s}" + for r in results.values(): + passed, total = r.criterion_counts.get(crit, (0, 0)) + crit_line += f" {str(passed) + '/' + str(total):>12s}" + lines.append(crit_line) + + lines.append("") + avg_line = f"{'Avg turns':40s}" + err_line = f"{'Errors':40s}" + for r in results.values(): + avg_line += f" {str(r.avg_turns):>12s}" + err_line += f" {str(r.errored_runs):>12s}" + lines.append(avg_line) + lines.append(err_line) + + if any(r.total_runs < 10 for r in results.values()): + lines.append("") + lines.append("Note: CI is wide due to small sample size; consider --n 10+") + + if any(r.partial for r in results.values()): + lines.append("") + lines.append("Warning: Sweep was interrupted — results are incomplete.") + + else: + lines.append(f"Scenario: {scenario}") + lines.append("") + lines.append(f"{'Backend':20s} {'Result':8s} {'Score':7s} {'Turns':5s}") + lines.append("-" * 42) + for name, r in results.items(): + result_str = "PASS" if r.passed_runs == r.total_runs else "FAIL" + total_criteria = sum(t for _, t in r.criterion_counts.values()) + passed_criteria = sum(p for p, _ in r.criterion_counts.values()) + score = f"{passed_criteria}/{total_criteria}" + turns_str = ( + str(int(r.avg_turns)) if r.avg_turns == int(r.avg_turns) else str(r.avg_turns) + ) + lines.append(f"{name:20s} {result_str:8s} {score:7s} {turns_str:5s}") + + all_criteria = [] + seen = set() + for r in results.values(): + for crit in r.criterion_counts: + if crit not in seen: + all_criteria.append(crit) + seen.add(crit) + + lines.append("") + header = f"{'':40s}" + for name in results: + header += f" {name:>12s}" + lines.append(header) + lines.append("-" * len(header)) + for crit in all_criteria: + crit_line = f"{crit[:40]:40s}" + for r in results.values(): + p, t = r.criterion_counts.get(crit, (0, 0)) + icon = "PASS" if p == t and t > 0 else "FAIL" + crit_line += f" {icon:>12s}" + lines.append(crit_line) + + return "\n".join(lines) diff --git a/evals/drill/engine.py b/evals/drill/engine.py new file mode 100644 index 00000000..8519083a --- /dev/null +++ b/evals/drill/engine.py @@ -0,0 +1,377 @@ +"""Engine: orchestrates the full Drill run lifecycle.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +import yaml + +from drill.actor import Actor +from drill.assertions import AssertionResult, run_verify_assertions +from drill.backend import load_backend +from drill.normalizer import ( + NORMALIZERS, + collect_new_logs, + filter_codex_logs_by_cwd, + snapshot_log_dir, +) +from drill.session import TmuxSession +from drill.setup import run_assertions, run_helpers +from drill.verifier import Verifier + + +@dataclass +class VerifyConfig: + criteria: list[str] = field(default_factory=list) + assertions: list[str] = field(default_factory=list) + observe: bool = False + + +@dataclass +class ScenarioConfig: + scenario: str + description: str + user_posture: str + setup: dict[str, Any] + turns: list[dict[str, Any]] + limits: dict[str, Any] + verify: VerifyConfig + + @classmethod + def from_yaml(cls, path: Path) -> ScenarioConfig: + with open(path) as f: + data = yaml.safe_load(f) + verify_data = data.get("verify", {}) + return cls( + scenario=data["scenario"], + description=data.get("description", ""), + user_posture=data.get("user_posture", "naive"), + setup=data.get("setup", {}), + turns=data.get("turns", []), + limits=data.get("limits", {"max_turns": 20, "turn_timeout": 120}), + verify=VerifyConfig( + criteria=verify_data.get("criteria", []), + assertions=verify_data.get("assertions", []), + observe=verify_data.get("observe", False), + ), + ) + + +@dataclass +class RunResult: + scenario: str + backend: str + timestamp: str + session_log: str + filesystem_json: str + tool_calls_jsonl: str + verdict_json: str + meta: dict[str, Any] + + def save_artifacts(self, output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "session.log").write_text(self.session_log) + (output_dir / "filesystem.json").write_text(self.filesystem_json) + (output_dir / "tool_calls.jsonl").write_text(self.tool_calls_jsonl) + + def save_verdict(self, output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "verdict.json").write_text(self.verdict_json) + (output_dir / "meta.json").write_text(json.dumps(self.meta, indent=2)) + + def save(self, output_dir: Path) -> None: + self.save_artifacts(output_dir) + self.save_verdict(output_dir) + + +def snapshot_filesystem(workdir: Path) -> str: + files: list[str] = [] + for f in sorted(workdir.rglob("*")): + if ".git" in f.parts: + continue + if f.is_file(): + files.append(str(f.relative_to(workdir))) + git_status = _git_cmd(workdir, ["git", "status", "--short"]) + branch = _git_cmd(workdir, ["git", "branch", "--show-current"]) + worktree_list = _git_cmd(workdir, ["git", "worktree", "list"]) + return json.dumps( + { + "files": files, + "git_status": git_status, + "branch": branch, + "worktree_list": worktree_list, + }, + indent=2, + ) + + +class Engine: + def __init__( + self, + scenario_path: Path, + backend_name: str, + backends_dir: Path, + fixtures_dir: Path, + results_dir: Path, + ) -> None: + self.scenario = ScenarioConfig.from_yaml(scenario_path) + self.backend = load_backend(backend_name, backends_dir) + self.fixtures_dir = fixtures_dir + self.results_dir = results_dir + + def run(self, *, output_dir: Path | None = None, run_suffix: str = "") -> RunResult: + start_time = time.time() + timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + self.backend.validate_env() + workdir = Path(f"/tmp/drill-{self.scenario.scenario}-{timestamp}{run_suffix}") + self._setup(workdir) + actual_workdir = workdir + override = self.scenario.setup.get("workdir_override") + if override: + resolved = override.replace("${WORKDIR_NAME}", workdir.name) + actual_workdir = (workdir / resolved).resolve() + # Run assertions in the actual workdir (after override) + assertions = self.scenario.setup.get("assertions", []) + if assertions: + run_assertions(assertions, actual_workdir) + session_name = f"drill-{self.scenario.scenario}-{timestamp}{run_suffix}" + session = TmuxSession(name=session_name, cols=self.backend.cols, rows=self.backend.rows) + log_dir = self._resolve_log_dir(actual_workdir) + log_snapshot = snapshot_log_dir(log_dir) if log_dir else set() + session_log, actor_turns = self._run_session(session, actual_workdir) + filesystem_json = snapshot_filesystem(actual_workdir) + tool_calls = self._collect_tool_calls(log_dir, log_snapshot, actual_workdir) + tool_calls_jsonl = "\n".join(json.dumps(tc) for tc in tool_calls) + + # Write artifacts to disk before assertions (assertions read from disk) + if output_dir is None: + output_dir = self.results_dir / self.scenario.scenario / self.backend.name / timestamp + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "session.log").write_text(session_log) + (output_dir / "filesystem.json").write_text(filesystem_json) + (output_dir / "tool_calls.jsonl").write_text(tool_calls_jsonl) + + # Run deterministic assertions + assertion_results: list[AssertionResult] = [] + if self.scenario.verify.assertions: + if not tool_calls_jsonl.strip(): + assertion_results = [ + AssertionResult( + command="", + passed=False, + exit_code=1, + stdout="", + stderr="tool_calls.jsonl is empty — session may have crashed", + ) + ] + else: + assertion_results = run_verify_assertions( + self.scenario.verify.assertions, + output_dir, + actual_workdir, + ) + + # Run LLM verifier + verifier = Verifier() + verdict = verifier.verify( + session_log=session_log, + filesystem_json=filesystem_json, + tool_calls_jsonl=tool_calls_jsonl, + criteria=self.scenario.verify.criteria, + ) + + # Merge assertion results into verdict + for ar in assertion_results: + verdict.criteria.append(ar.to_criterion_result()) + + duration = time.time() - start_time + meta: dict[str, Any] = { + "scenario": self.scenario.scenario, + "backend": self.backend.name, + "backend_model": self.backend.model, + "user_posture": self.scenario.user_posture, + "timestamp": timestamp, + "duration_seconds": round(duration, 1), + "actor_turns": actor_turns, + "actor_model": "claude-sonnet-4-6", + "verifier_model": "claude-sonnet-4-6", + } + result = RunResult( + scenario=self.scenario.scenario, + backend=self.backend.name, + timestamp=timestamp, + session_log=session_log, + filesystem_json=filesystem_json, + tool_calls_jsonl=tool_calls_jsonl, + verdict_json=verdict.model_dump_json(indent=2), + meta=meta, + ) + # Write verdict + meta (artifacts already on disk) + (output_dir / "verdict.json").write_text(result.verdict_json) + (output_dir / "meta.json").write_text(json.dumps(result.meta, indent=2)) + return result + + def _setup(self, workdir: Path) -> None: + # Scenario helpers first (create_base_repo needs to run before anything else) + helpers = self.scenario.setup.get("helpers", []) + run_helpers(helpers, workdir, self.fixtures_dir) + # Backend pre_run hooks after (e.g., codex symlink needs workdir to exist) + hooks_needing_superpowers_root = {"symlink_superpowers", "link_gemini_extension"} + for hook_name in self.backend.hooks.get("pre_run", []): + from setup_helpers import HELPER_REGISTRY + + hook = HELPER_REGISTRY.get(hook_name) + if hook and hook_name in hooks_needing_superpowers_root: + hook(workdir, os.environ["SUPERPOWERS_ROOT"]) # ty: ignore[invalid-argument-type, too-many-positional-arguments, missing-argument] + elif hook: + hook(workdir) # ty: ignore[invalid-argument-type, missing-argument] + + def _run_session(self, session: TmuxSession, workdir: Path) -> tuple[str, int]: + session.create() + try: + cmd = self.backend.build_command(str(workdir)) + session.launch(cmd, str(workdir)) + self._wait_for_ready(session, timeout=self.backend.startup_timeout) + actor = Actor() + intents = [t["intent"] for t in self.scenario.turns] + actor.build_system_prompt(posture=self.scenario.user_posture, intents=intents) + max_turns = self.scenario.limits.get("max_turns", 20) + turn_timeout = self.backend.turn_timeout or self.scenario.limits.get( + "turn_timeout", 120 + ) + all_captures: list[str] = [] + turn_count = 0 + for turn in range(max_turns): + self._wait_for_ready(session, timeout=turn_timeout) + capture = session.capture() + all_captures.append(f"=== Turn {turn + 1} ===\n{capture}") + actor.append_capture(f"Terminal output:\n{capture}") + action = actor.decide() + turn_count += 1 + if action.action == "done" or action.action == "stuck": + break + elif action.action == "type": + session.send_keys(action.text or "") + elif action.action == "key": + session.send_special_key(action.key or "") + final_capture = session.capture() + all_captures.append(f"=== Final ===\n{final_capture}") + if self.backend.shutdown.startswith("< None: + """Wait until the agent's terminal is ready for Actor input. + + Returns when the terminal is quiescent AND matches the backend's + ready pattern. If the backend's busy pattern matches (spinner + visible, "Thinking...", timer counting), the deadline is extended + by small increments up to `max_busy_seconds` total. This prevents + the Actor from interrupting long-running subagent work (wave + execution, multi-file implementation, etc.). + + Exits silently if the final deadline (timeout + busy extensions) + passes without reaching a ready state. + """ + quiescence = self.backend.quiescence_seconds + max_busy_extension = float(self.backend.max_busy_seconds) + start = time.time() + deadline = start + timeout + total_busy_extended = 0.0 + last_output: str = "" + stable_since: float | None = None + + while time.time() < deadline: + current = session.capture() + lines = current.strip().split("\n") + is_busy = any(self.backend.is_busy_line(line) for line in lines) + + # If the agent is actively busy, extend the deadline so we + # don't time out mid-subagent-work. Extensions are capped at + # max_busy_seconds total across all extensions combined. + if is_busy: + remaining_budget = max_busy_extension - total_busy_extended + if remaining_budget > 0: + # Ensure we have at least 30 more seconds of headroom. + needed = 30.0 - (deadline - time.time()) + if needed > 0: + grant = min(needed, remaining_budget) + deadline += grant + total_busy_extended += grant + + # Strip animated elements so they don't reset the quiescence timer: + # - Time counters: "Thinking... (4m 1s)" or "(esc to cancel, 4m 1s)" + # - Braille spinner characters that rotate every frame + normalized = re.sub(r"\((?:esc to cancel, )?(?:\d+[hms]\s*)+\)", "(…)", current) + normalized = re.sub(r"[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]", "·", normalized) + if normalized != last_output: + last_output = normalized + stable_since = time.time() + elif stable_since and (time.time() - stable_since) >= quiescence: + if is_busy: + stable_since = None # Reset — agent is still working + elif any(self.backend.is_ready_line(line) for line in lines): + return + time.sleep(0.5) + + def _resolve_log_dir(self, workdir: Path) -> Path | None: + """Resolve the log directory for the given backend and workdir. + + Claude Code stores logs at ~/.claude/projects// + where the path is the real workdir with / replaced by -. + Codex stores logs at ~/.codex/sessions/. + """ + if self.backend.family == "claude": + real_workdir = workdir.resolve() + encoded = str(real_workdir).replace("/", "-") + log_dir = Path.home() / ".claude" / "projects" / encoded + return log_dir + elif self.backend.family == "codex": + # Codex stores at ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl + return Path.home() / ".codex" / "sessions" + elif self.backend.family == "gemini": + # Gemini stores at ~/.gemini/tmp//chats/session-*.json + # Project name is the workdir basename, lowercased + project = workdir.resolve().name.lower() + return Path.home() / ".gemini" / "tmp" / project + pattern = self.backend.session_logs.get("pattern", "") + if not pattern: + return None + expanded = os.path.expanduser(pattern) + parts = expanded.split("*")[0].rstrip("/") + return Path(parts) + + def _collect_tool_calls( + self, log_dir: Path | None, snapshot: set[str], workdir: Path + ) -> list[dict[str, Any]]: + if log_dir is None: + return [] + new_files = collect_new_logs(log_dir, snapshot) + if self.backend.family == "codex": + new_files = filter_codex_logs_by_cwd(new_files, str(workdir.resolve())) + normalizer = NORMALIZERS.get(self.backend.family) + if not normalizer: + return [] + results: list[dict[str, Any]] = [] + for log_file in new_files: + results.extend(normalizer(log_file.read_text())) + return results + + +def _git_cmd(workdir: Path, cmd: list[str]) -> str: + result = subprocess.run(cmd, cwd=workdir, capture_output=True, text=True) + return result.stdout.strip() diff --git a/evals/drill/normalizer.py b/evals/drill/normalizer.py new file mode 100644 index 00000000..b88cbbc4 --- /dev/null +++ b/evals/drill/normalizer.py @@ -0,0 +1,228 @@ +"""Normalizes backend-specific session logs to a common tool call schema.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +NATIVE_TOOLS: set[str] = { + "EnterWorktree", + "ExitWorktree", + "EnterPlanMode", + "ExitPlanMode", + "TaskCreate", + "TaskUpdate", + "TaskList", + "TaskGet", + "Skill", + "Agent", + "Read", + "Write", + "Edit", + "Glob", + "Grep", +} + +LOG_EXTENSIONS: tuple[str, ...] = ("*.jsonl", "*.json") + + +def snapshot_log_dir(log_dir: Path) -> set[str]: + """Snapshot all session log files in a log directory (recursive).""" + if not log_dir.exists(): + return set() + files: set[str] = set() + for ext in LOG_EXTENSIONS: + files.update(str(f.relative_to(log_dir)) for f in log_dir.rglob(ext)) + return files + + +def collect_new_logs(log_dir: Path, snapshot: set[str]) -> list[Path]: + """Find session log files created after the snapshot (recursive).""" + if not log_dir.exists(): + return [] + current: dict[str, Path] = {} + for ext in LOG_EXTENSIONS: + current.update({str(f.relative_to(log_dir)): f for f in log_dir.rglob(ext)}) + new_keys: set[str] = set(current.keys()) - snapshot + return [current[k] for k in sorted(new_keys)] + + +def filter_codex_logs_by_cwd(paths: list[Path], target_cwd: str) -> list[Path]: + """Drop codex rollouts whose session_meta.cwd doesn't match target_cwd. + + Codex stores all sessions under a shared ~/.codex/sessions/ tree, so when + multiple drill scenarios run in parallel each one's snapshot diff sees every + other run's rollouts. Each rollout's first line is a `session_meta` event + that records the cwd the codex CLI was launched in — use it to attribute + rollouts to the run that produced them. + """ + matched: list[Path] = [] + for path in paths: + try: + with path.open() as f: + first_line = f.readline() + entry = json.loads(first_line) + except (OSError, json.JSONDecodeError): + continue + if entry.get("type") != "session_meta": + continue + cwd = entry.get("payload", {}).get("cwd", "") + if cwd == target_cwd: + matched.append(path) + return matched + + +def normalize_claude_logs(raw_content: str) -> list[dict[str, Any]]: + """Normalize Claude Code session logs. + + CC logs are JSONL where assistant messages have: + {"type": "assistant", "message": {"content": [{"type": "tool_use", "name": "...", + "input": {...}}]}} + """ + results: list[dict[str, Any]] = [] + for line in raw_content.strip().split("\n"): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + # Handle nested CC format: assistant messages contain tool_use in content array + if entry.get("type") == "assistant": + message = entry.get("message", {}) + for block in message.get("content", []): + if block.get("type") == "tool_use": + tool_name = block.get("name", "") + source = "native" if tool_name in NATIVE_TOOLS else "shell" + results.append( + {"tool": tool_name, "args": block.get("input", {}), "source": source} + ) + # Also handle flat format (for test compatibility) + elif entry.get("type") == "tool_use": + tool_name = entry.get("name", "") + source = "native" if tool_name in NATIVE_TOOLS else "shell" + results.append({"tool": tool_name, "args": entry.get("input", {}), "source": source}) + return results + + +def normalize_codex_logs(raw_content: str) -> list[dict[str, Any]]: + """Normalize Codex rollout logs. + + Codex logs use: {"type": "response_item", "payload": {"type": "function_call", ...}} + Tool calls are "function_call" with name "exec_command" (shell) or other names. + """ + results: list[dict[str, Any]] = [] + for line in raw_content.strip().split("\n"): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if entry.get("type") != "response_item": + continue + # Codex uses "payload" not "item" + payload = entry.get("payload", entry.get("item", {})) + payload_type = payload.get("type", "") + if payload_type == "function_call": + name = payload.get("name", "") + raw_args = payload.get("arguments", "{}") + # Arguments are JSON-encoded strings in codex + if isinstance(raw_args, str): + try: + args = json.loads(raw_args) + except json.JSONDecodeError: + args = {"raw": raw_args} + else: + args = raw_args + # exec_command is codex's shell tool + if name == "exec_command": + results.append( + {"tool": "Bash", "args": {"command": args.get("cmd", "")}, "source": "shell"} + ) + elif name == "apply_patch": + results.append({"tool": "Edit", "args": args, "source": "native"}) + else: + source = "native" if name in NATIVE_TOOLS else "shell" + results.append({"tool": name, "args": args, "source": source}) + elif payload_type == "local_shell_call": + action = payload.get("action", {}) + cmd = action.get("command", []) + cmd_str = " ".join(cmd) if isinstance(cmd, list) else str(cmd) + results.append({"tool": "Bash", "args": {"command": cmd_str}, "source": "shell"}) + return results + + +# Reverse mapping: Gemini tool names → Claude Code canonical names +GEMINI_TOOL_MAP: dict[str, str] = { + "run_shell_command": "Bash", + "read_file": "Read", + "write_file": "Write", + "replace": "Edit", + "grep_search": "Grep", + "glob": "Glob", + "activate_skill": "Skill", + "google_web_search": "WebSearch", + "web_fetch": "WebFetch", + "write_todos": "TodoWrite", + "list_directory": "Glob", + "enter_plan_mode": "EnterPlanMode", + "exit_plan_mode": "ExitPlanMode", +} + + +def normalize_gemini_logs(raw_content: str) -> list[dict[str, Any]]: + """Normalize Gemini CLI session logs. + + Gemini logs may be a single JSON file with a messages array, or JSONL + session files in newer CLI versions. Each "gemini" message may have a + toolCalls array: + {"name": "run_shell_command", "args": {"command": "..."}, "status": "success"} + """ + results: list[dict[str, Any]] = [] + messages: list[dict[str, Any]] = [] + try: + data = json.loads(raw_content) + except json.JSONDecodeError: + for line in raw_content.strip().split("\n"): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(entry, dict): + messages.append(entry) + else: + if isinstance(data, dict) and "messages" in data: + messages = [m for m in data.get("messages", []) if isinstance(m, dict)] + elif isinstance(data, dict): + messages = [data] + elif isinstance(data, list): + messages = [m for m in data if isinstance(m, dict)] + + seen_tool_calls: set[str] = set() + for message in messages: + if message.get("type") != "gemini": + continue + for tc in message.get("toolCalls", []): + tool_call_id = tc.get("id") + if tool_call_id and tool_call_id in seen_tool_calls: + continue + if tool_call_id: + seen_tool_calls.add(tool_call_id) + gemini_name = tc.get("name", "") + canonical = GEMINI_TOOL_MAP.get(gemini_name, gemini_name) + args = tc.get("args", {}) + source = "native" if canonical in NATIVE_TOOLS else "shell" + results.append({"tool": canonical, "args": args, "source": source}) + return results + + +NORMALIZERS: dict[str, Callable[[str], list[dict[str, Any]]]] = { + "claude": normalize_claude_logs, + "codex": normalize_codex_logs, + "gemini": normalize_gemini_logs, +} diff --git a/evals/drill/session.py b/evals/drill/session.py new file mode 100644 index 00000000..65597c2a --- /dev/null +++ b/evals/drill/session.py @@ -0,0 +1,88 @@ +"""tmux session management for driving agent CLI sessions.""" + +from __future__ import annotations + +import subprocess +import time + + +class TmuxSession: + def __init__(self, name: str, cols: int = 200, rows: int = 50) -> None: + self.name = name + self.cols = cols + self.rows = rows + + def create(self) -> None: + subprocess.run( + [ + "tmux", + "new-session", + "-d", + "-s", + self.name, + "-x", + str(self.cols), + "-y", + str(self.rows), + ], + check=True, + ) + + def launch(self, command: list[str], cwd: str) -> None: + cmd_str = " ".join(command) + self.send_keys(f"cd {cwd} && {cmd_str}") + + def send_keys(self, text: str) -> None: + if text: + buffer_name = f"{self.name}-input" + subprocess.run( + ["tmux", "set-buffer", "-b", buffer_name, text], + check=True, + ) + subprocess.run( + ["tmux", "paste-buffer", "-d", "-b", buffer_name, "-t", self.name], + check=True, + ) + time.sleep(0.1) + + subprocess.run( + ["tmux", "send-keys", "-t", self.name, "Enter"], + check=True, + ) + + def send_special_key(self, key: str) -> None: + key_map = { + "ctrl-c": "C-c", + "ctrl-d": "C-d", + "ctrl-z": "C-z", + "enter": "Enter", + "escape": "Escape", + } + tmux_key = key_map.get(key, key) + subprocess.run( + ["tmux", "send-keys", "-t", self.name, tmux_key], + check=True, + ) + + def capture(self) -> str: + result = subprocess.run( + ["tmux", "capture-pane", "-t", self.name, "-p"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + def is_process_alive(self) -> bool: + result = subprocess.run( + ["tmux", "list-panes", "-t", self.name, "-F", "#{pane_dead}"], + capture_output=True, + text=True, + ) + return result.stdout.strip() == "0" + + def kill(self) -> None: + subprocess.run( + ["tmux", "kill-session", "-t", self.name], + capture_output=True, + ) diff --git a/evals/drill/setup.py b/evals/drill/setup.py new file mode 100644 index 00000000..74ea1efc --- /dev/null +++ b/evals/drill/setup.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +from setup_helpers import HELPER_REGISTRY +from setup_helpers.base import create_base_repo + + +def clone_template(template_dir: Path, workdir: Path) -> None: + """Clone (or build) template_dir into workdir with full git history.""" + create_base_repo(workdir, template_dir) + + +def run_helpers(helper_names: list[str], workdir: Path, fixtures_dir: Path) -> None: + for name in helper_names: + helper = HELPER_REGISTRY.get(name) + if helper is None: + raise ValueError(f"Unknown setup helper: {name}") + if name == "create_base_repo": + helper(workdir, fixtures_dir / "template-repo") # ty: ignore[invalid-argument-type, too-many-positional-arguments, missing-argument] + elif name == "symlink_superpowers": + import os + + helper(workdir, os.environ["SUPERPOWERS_ROOT"]) # ty: ignore[invalid-argument-type, too-many-positional-arguments, missing-argument] + else: + helper(workdir) # ty: ignore[invalid-argument-type, missing-argument] + + +def run_assertions(assertions: list[str], workdir: Path) -> None: + for assertion in assertions: + result = subprocess.run( + assertion, + shell=True, + cwd=workdir, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise AssertionError( + f"Setup assertion failed: {assertion}\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) diff --git a/evals/drill/stats.py b/evals/drill/stats.py new file mode 100644 index 00000000..6b1ea859 --- /dev/null +++ b/evals/drill/stats.py @@ -0,0 +1,17 @@ +"""Statistical utilities for drill result analysis.""" + +from __future__ import annotations + +import math + + +def wilson_ci(passed: int, total: int, z: float = 1.96) -> tuple[float, float]: + if total == 0: + return (0.0, 0.0) + if passed > total: + passed = total + p = passed / total + denom = 1 + z**2 / total + center = (p + z**2 / (2 * total)) / denom + margin = (z / denom) * math.sqrt(p * (1 - p) / total + z**2 / (4 * total**2)) + return (max(0.0, center - margin), min(1.0, center + margin)) diff --git a/evals/drill/sweep.py b/evals/drill/sweep.py new file mode 100644 index 00000000..740da7be --- /dev/null +++ b/evals/drill/sweep.py @@ -0,0 +1,159 @@ +"""Sweep orchestrator: runs scenarios N times across multiple backends.""" + +from __future__ import annotations + +import glob as glob_mod +import json +import shutil +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +import yaml + +from drill.engine import Engine, RunResult +from drill.verifier import Verdict + + +@dataclass +class RunStatus: + index: int + status: str # "pass", "fail", "error" + duration: float + error: str | None = None + + +@dataclass +class RunGroup: + scenario: str + backend: str + n: int + timestamp: str + sweep_id: str + runs: list[RunStatus] = field(default_factory=list) + partial: bool = False + + +def write_run_group(group: RunGroup, output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + data: dict[str, Any] = { + "scenario": group.scenario, + "backend": group.backend, + "n": group.n, + "timestamp": group.timestamp, + "sweep_id": group.sweep_id, + "partial": group.partial, + "runs": [ + {k: v for k, v in asdict(r).items() if k != "error" or v is not None} + for r in group.runs + ], + } + (output_dir / "run-group.json").write_text(json.dumps(data, indent=2)) + + +class Sweep: + def __init__( + self, + scenario_path: Path, + backend_names: list[str], + backends_dir: Path, + fixtures_dir: Path, + results_dir: Path, + n: int, + sweep_id: str, + ) -> None: + self.scenario_path = scenario_path + self.backend_names = backend_names + self.backends_dir = backends_dir + self.fixtures_dir = fixtures_dir + self.results_dir = results_dir + self.n = n + self.sweep_id = sweep_id + self._scenario_name_cache: str | None = None + + def validate_backends(self) -> None: + for name in self.backend_names: + path = self.backends_dir / f"{name}.yaml" + if not path.exists(): + raise FileNotFoundError(f"Backend config not found: {path}") + + def run_all(self) -> list[RunGroup]: + self.validate_backends() + groups: list[RunGroup] = [] + for backend_name in self.backend_names: + group = self._run_backend(backend_name) + groups.append(group) + return groups + + def _run_backend(self, backend_name: str) -> RunGroup: + timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + group_dir = ( + self.results_dir / self.scenario_name / backend_name / f"{timestamp}-{self.sweep_id}" + ) + group_dir.mkdir(parents=True, exist_ok=True) + + group = RunGroup( + scenario=self.scenario_name, + backend=backend_name, + n=self.n, + timestamp=timestamp, + sweep_id=self.sweep_id, + ) + + try: + for i in range(self.n): + run_status = self._run_single(backend_name, group_dir, i, timestamp) + group.runs.append(run_status) + except KeyboardInterrupt: + group.partial = True + finally: + write_run_group(group, group_dir) + + return group + + def _run_single( + self, backend_name: str, group_dir: Path, index: int, timestamp: str + ) -> RunStatus: + run_suffix = f"-run-{index:02d}" + run_dir = group_dir / f"run-{index:02d}" + start = time.time() + + try: + engine = Engine( + scenario_path=self.scenario_path, + backend_name=backend_name, + backends_dir=self.backends_dir, + fixtures_dir=self.fixtures_dir, + results_dir=self.results_dir, + ) + result: RunResult = engine.run(output_dir=run_dir, run_suffix=run_suffix) + verdict = Verdict.model_validate_json(result.verdict_json) + duration = time.time() - start + status = "pass" if verdict.passed else "fail" + return RunStatus(index=index, status=status, duration=round(duration, 1)) + except KeyboardInterrupt: + raise + except Exception as e: + duration = time.time() - start + return RunStatus( + index=index, + status="error", + duration=round(duration, 1), + error=str(e), + ) + finally: + pattern = f"/tmp/drill-*-{timestamp}{run_suffix}" + for d in glob_mod.glob(pattern): + p = Path(d) + if p.is_dir(): + shutil.rmtree(p, ignore_errors=True) + + @property + def scenario_name(self) -> str: + if self._scenario_name_cache is None: + with open(self.scenario_path) as f: + data = yaml.safe_load(f) + self._scenario_name_cache = data["scenario"] + return self._scenario_name_cache diff --git a/evals/drill/verifier.py b/evals/drill/verifier.py new file mode 100644 index 00000000..7042a31f --- /dev/null +++ b/evals/drill/verifier.py @@ -0,0 +1,93 @@ +"""Verifier LLM: evaluates agent session against criteria.""" + +from __future__ import annotations + +from pathlib import Path + +import anthropic +from pydantic import BaseModel + + +class CriterionResult(BaseModel): + criterion: str + verdict: str + evidence: str + rationale: str + source: str = "judge" + + +class Verdict(BaseModel): + criteria: list[CriterionResult] + observations: list[str] + summary: str + + @property + def score(self) -> str: + passed = sum(1 for c in self.criteria if c.verdict == "pass") + return f"{passed}/{len(self.criteria)}" + + @property + def passed(self) -> bool: + return all(c.verdict == "pass" for c in self.criteria) + + +class Verifier: + MAX_RETRIES = 3 + + def __init__(self, model: str = "claude-sonnet-4-6", temperature: float = 0.0) -> None: + self.model = model + self.temperature = temperature + self._client: anthropic.Anthropic = anthropic.Anthropic() + + def build_system_prompt(self) -> str: + template_path = Path(__file__).parent.parent / "prompts" / "verifier.md" + return template_path.read_text() + + def verify( + self, + session_log: str, + filesystem_json: str, + tool_calls_jsonl: str, + criteria: list[str], + ) -> Verdict: + system = self.build_system_prompt() + user_content = ( + "## Terminal Session Log\n\n" + f"```\n{session_log}\n```\n\n" + "## Filesystem State\n\n" + f"```json\n{filesystem_json}\n```\n\n" + "## Tool Call Log\n\n" + f"```jsonl\n{tool_calls_jsonl}\n```\n\n" + "## Criteria to Evaluate\n\n" + "\n".join(f"- {c}" for c in criteria) + ) + for attempt in range(self.MAX_RETRIES): + response = self._client.messages.create( + model=self.model, + max_tokens=4096, + temperature=self.temperature, + system=system, + messages=[{"role": "user", "content": user_content}], + ) + text = response.content[0].text # ty: ignore[unresolved-attribute] + json_str = _extract_json(text) + try: + return Verdict.model_validate_json(json_str) + except Exception: + if attempt == self.MAX_RETRIES - 1: + raise + continue + raise RuntimeError("Verifier failed to return valid JSON") + + +def _extract_json(text: str) -> str: + if "```json" in text: + start = text.index("```json") + 7 + end = text.index("```", start) + return text[start:end].strip() + if "```" in text: + start = text.index("```") + 3 + end = text.index("```", start) + return text[start:end].strip() + start = text.index("{") + end = text.rindex("}") + 1 + return text[start:end] diff --git a/evals/fixtures/sdd-go-fractals/design.md b/evals/fixtures/sdd-go-fractals/design.md new file mode 100644 index 00000000..2fbc6b1f --- /dev/null +++ b/evals/fixtures/sdd-go-fractals/design.md @@ -0,0 +1,81 @@ +# Go Fractals CLI - Design + +## Overview + +A command-line tool that generates ASCII art fractals. Supports two fractal types with configurable output. + +## Usage + +```bash +# Sierpinski triangle +fractals sierpinski --size 32 --depth 5 + +# Mandelbrot set +fractals mandelbrot --width 80 --height 24 --iterations 100 + +# Custom character +fractals sierpinski --size 16 --char '#' + +# Help +fractals --help +fractals sierpinski --help +``` + +## Commands + +### `sierpinski` + +Generates a Sierpinski triangle using recursive subdivision. + +Flags: +- `--size` (default: 32) - Width of the triangle base in characters +- `--depth` (default: 5) - Recursion depth +- `--char` (default: '*') - Character to use for filled points + +Output: Triangle printed to stdout, one line per row. + +### `mandelbrot` + +Renders the Mandelbrot set as ASCII art. Maps iteration count to characters. + +Flags: +- `--width` (default: 80) - Output width in characters +- `--height` (default: 24) - Output height in characters +- `--iterations` (default: 100) - Maximum iterations for escape calculation +- `--char` (default: gradient) - Single character, or omit for gradient " .:-=+*#%@" + +Output: Rectangle printed to stdout. + +## Architecture + +``` +cmd/ + fractals/ + main.go # Entry point, CLI setup +internal/ + sierpinski/ + sierpinski.go # Algorithm + sierpinski_test.go + mandelbrot/ + mandelbrot.go # Algorithm + mandelbrot_test.go + cli/ + root.go # Root command, help + sierpinski.go # Sierpinski subcommand + mandelbrot.go # Mandelbrot subcommand +``` + +## Dependencies + +- Go 1.21+ +- `github.com/spf13/cobra` for CLI + +## Acceptance Criteria + +1. `fractals --help` shows usage +2. `fractals sierpinski` outputs a recognizable triangle +3. `fractals mandelbrot` outputs a recognizable Mandelbrot set +4. `--size`, `--width`, `--height`, `--depth`, `--iterations` flags work +5. `--char` customizes output character +6. Invalid inputs produce clear error messages +7. All tests pass diff --git a/evals/fixtures/sdd-go-fractals/plan.md b/evals/fixtures/sdd-go-fractals/plan.md new file mode 100644 index 00000000..9875ab5f --- /dev/null +++ b/evals/fixtures/sdd-go-fractals/plan.md @@ -0,0 +1,172 @@ +# Go Fractals CLI - Implementation Plan + +Execute this plan using the `superpowers:subagent-driven-development` skill. + +## Context + +Building a CLI tool that generates ASCII fractals. See `design.md` for full specification. + +## Tasks + +### Task 1: Project Setup + +Create the Go module and directory structure. + +**Do:** +- Initialize `go.mod` with module name `github.com/superpowers-test/fractals` +- Create directory structure: `cmd/fractals/`, `internal/sierpinski/`, `internal/mandelbrot/`, `internal/cli/` +- Create minimal `cmd/fractals/main.go` that prints "fractals cli" +- Add `github.com/spf13/cobra` dependency + +**Verify:** +- `go build ./cmd/fractals` succeeds +- `./fractals` prints "fractals cli" + +--- + +### Task 2: CLI Framework with Help + +Set up Cobra root command with help output. + +**Do:** +- Create `internal/cli/root.go` with root command +- Configure help text showing available subcommands +- Wire root command into `main.go` + +**Verify:** +- `./fractals --help` shows usage with "sierpinski" and "mandelbrot" listed as available commands +- `./fractals` (no args) shows help + +--- + +### Task 3: Sierpinski Algorithm + +Implement the Sierpinski triangle generation algorithm. + +**Do:** +- Create `internal/sierpinski/sierpinski.go` +- Implement `Generate(size, depth int, char rune) []string` that returns lines of the triangle +- Use recursive midpoint subdivision algorithm +- Create `internal/sierpinski/sierpinski_test.go` with tests: + - Small triangle (size=4, depth=2) matches expected output + - Size=1 returns single character + - Depth=0 returns filled triangle + +**Verify:** +- `go test ./internal/sierpinski/...` passes + +--- + +### Task 4: Sierpinski CLI Integration + +Wire the Sierpinski algorithm to a CLI subcommand. + +**Do:** +- Create `internal/cli/sierpinski.go` with `sierpinski` subcommand +- Add flags: `--size` (default 32), `--depth` (default 5), `--char` (default '*') +- Call `sierpinski.Generate()` and print result to stdout + +**Verify:** +- `./fractals sierpinski` outputs a triangle +- `./fractals sierpinski --size 16 --depth 3` outputs smaller triangle +- `./fractals sierpinski --help` shows flag documentation + +--- + +### Task 5: Mandelbrot Algorithm + +Implement the Mandelbrot set ASCII renderer. + +**Do:** +- Create `internal/mandelbrot/mandelbrot.go` +- Implement `Render(width, height, maxIter int, char string) []string` +- Map complex plane region (-2.5 to 1.0 real, -1.0 to 1.0 imaginary) to output dimensions +- Map iteration count to character gradient " .:-=+*#%@" (or single char if provided) +- Create `internal/mandelbrot/mandelbrot_test.go` with tests: + - Output dimensions match requested width/height + - Known point inside set (0,0) maps to max-iteration character + - Known point outside set (2,0) maps to low-iteration character + +**Verify:** +- `go test ./internal/mandelbrot/...` passes + +--- + +### Task 6: Mandelbrot CLI Integration + +Wire the Mandelbrot algorithm to a CLI subcommand. + +**Do:** +- Create `internal/cli/mandelbrot.go` with `mandelbrot` subcommand +- Add flags: `--width` (default 80), `--height` (default 24), `--iterations` (default 100), `--char` (default "") +- Call `mandelbrot.Render()` and print result to stdout + +**Verify:** +- `./fractals mandelbrot` outputs recognizable Mandelbrot set +- `./fractals mandelbrot --width 40 --height 12` outputs smaller version +- `./fractals mandelbrot --help` shows flag documentation + +--- + +### Task 7: Character Set Configuration + +Ensure `--char` flag works consistently across both commands. + +**Do:** +- Verify Sierpinski `--char` flag passes character to algorithm +- For Mandelbrot, `--char` should use single character instead of gradient +- Add tests for custom character output + +**Verify:** +- `./fractals sierpinski --char '#'` uses '#' character +- `./fractals mandelbrot --char '.'` uses '.' for all filled points +- Tests pass + +--- + +### Task 8: Input Validation and Error Handling + +Add validation for invalid inputs. + +**Do:** +- Sierpinski: size must be > 0, depth must be >= 0 +- Mandelbrot: width/height must be > 0, iterations must be > 0 +- Return clear error messages for invalid inputs +- Add tests for error cases + +**Verify:** +- `./fractals sierpinski --size 0` prints error, exits non-zero +- `./fractals mandelbrot --width -1` prints error, exits non-zero +- Error messages are clear and helpful + +--- + +### Task 9: Integration Tests + +Add integration tests that invoke the CLI. + +**Do:** +- Create `cmd/fractals/main_test.go` or `test/integration_test.go` +- Test full CLI invocation for both commands +- Verify output format and exit codes +- Test error cases return non-zero exit + +**Verify:** +- `go test ./...` passes all tests including integration tests + +--- + +### Task 10: README + +Document usage and examples. + +**Do:** +- Create `README.md` with: + - Project description + - Installation: `go install ./cmd/fractals` + - Usage examples for both commands + - Example output (small samples) + +**Verify:** +- README accurately describes the tool +- Examples in README actually work diff --git a/evals/fixtures/sdd-svelte-todo/design.md b/evals/fixtures/sdd-svelte-todo/design.md new file mode 100644 index 00000000..ccbb10fe --- /dev/null +++ b/evals/fixtures/sdd-svelte-todo/design.md @@ -0,0 +1,70 @@ +# Svelte Todo List - Design + +## Overview + +A simple todo list application built with Svelte. Supports creating, completing, and deleting todos with localStorage persistence. + +## Features + +- Add new todos +- Mark todos as complete/incomplete +- Delete todos +- Filter by: All / Active / Completed +- Clear all completed todos +- Persist to localStorage +- Show count of remaining items + +## User Interface + +``` +┌─────────────────────────────────────────┐ +│ Svelte Todos │ +├─────────────────────────────────────────┤ +│ [________________________] [Add] │ +├─────────────────────────────────────────┤ +│ [ ] Buy groceries [x] │ +│ [✓] Walk the dog [x] │ +│ [ ] Write code [x] │ +├─────────────────────────────────────────┤ +│ 2 items left │ +│ [All] [Active] [Completed] [Clear ✓] │ +└─────────────────────────────────────────┘ +``` + +## Components + +``` +src/ + App.svelte # Main app, state management + lib/ + TodoInput.svelte # Text input + Add button + TodoList.svelte # List container + TodoItem.svelte # Single todo with checkbox, text, delete + FilterBar.svelte # Filter buttons + clear completed + store.ts # Svelte store for todos + storage.ts # localStorage persistence +``` + +## Data Model + +```typescript +interface Todo { + id: string; // UUID + text: string; // Todo text + completed: boolean; +} + +type Filter = 'all' | 'active' | 'completed'; +``` + +## Acceptance Criteria + +1. Can add a todo by typing and pressing Enter or clicking Add +2. Can toggle todo completion by clicking checkbox +3. Can delete a todo by clicking X button +4. Filter buttons show correct subset of todos +5. "X items left" shows count of incomplete todos +6. "Clear completed" removes all completed todos +7. Todos persist across page refresh (localStorage) +8. Empty state shows helpful message +9. All tests pass diff --git a/evals/fixtures/sdd-svelte-todo/plan.md b/evals/fixtures/sdd-svelte-todo/plan.md new file mode 100644 index 00000000..f4e555b3 --- /dev/null +++ b/evals/fixtures/sdd-svelte-todo/plan.md @@ -0,0 +1,222 @@ +# Svelte Todo List - Implementation Plan + +Execute this plan using the `superpowers:subagent-driven-development` skill. + +## Context + +Building a todo list app with Svelte. See `design.md` for full specification. + +## Tasks + +### Task 1: Project Setup + +Create the Svelte project with Vite. + +**Do:** +- Run `npm create vite@latest . -- --template svelte-ts` +- Install dependencies with `npm install` +- Verify dev server works +- Clean up default Vite template content from App.svelte + +**Verify:** +- `npm run dev` starts server +- App shows minimal "Svelte Todos" heading +- `npm run build` succeeds + +--- + +### Task 2: Todo Store + +Create the Svelte store for todo state management. + +**Do:** +- Create `src/lib/store.ts` +- Define `Todo` interface with id, text, completed +- Create writable store with initial empty array +- Export functions: `addTodo(text)`, `toggleTodo(id)`, `deleteTodo(id)`, `clearCompleted()` +- Create `src/lib/store.test.ts` with tests for each function + +**Verify:** +- Tests pass: `npm run test` (install vitest if needed) + +--- + +### Task 3: localStorage Persistence + +Add persistence layer for todos. + +**Do:** +- Create `src/lib/storage.ts` +- Implement `loadTodos(): Todo[]` and `saveTodos(todos: Todo[])` +- Handle JSON parse errors gracefully (return empty array) +- Integrate with store: load on init, save on change +- Add tests for load/save/error handling + +**Verify:** +- Tests pass +- Manual test: add todo, refresh page, todo persists + +--- + +### Task 4: TodoInput Component + +Create the input component for adding todos. + +**Do:** +- Create `src/lib/TodoInput.svelte` +- Text input bound to local state +- Add button calls `addTodo()` and clears input +- Enter key also submits +- Disable Add button when input is empty +- Add component tests + +**Verify:** +- Tests pass +- Component renders input and button + +--- + +### Task 5: TodoItem Component + +Create the single todo item component. + +**Do:** +- Create `src/lib/TodoItem.svelte` +- Props: `todo: Todo` +- Checkbox toggles completion (calls `toggleTodo`) +- Text with strikethrough when completed +- Delete button (X) calls `deleteTodo` +- Add component tests + +**Verify:** +- Tests pass +- Component renders checkbox, text, delete button + +--- + +### Task 6: TodoList Component + +Create the list container component. + +**Do:** +- Create `src/lib/TodoList.svelte` +- Props: `todos: Todo[]` +- Renders TodoItem for each todo +- Shows "No todos yet" when empty +- Add component tests + +**Verify:** +- Tests pass +- Component renders list of TodoItems + +--- + +### Task 7: FilterBar Component + +Create the filter and status bar component. + +**Do:** +- Create `src/lib/FilterBar.svelte` +- Props: `todos: Todo[]`, `filter: Filter`, `onFilterChange: (f: Filter) => void` +- Show count: "X items left" (incomplete count) +- Three filter buttons: All, Active, Completed +- Active filter is visually highlighted +- "Clear completed" button (hidden when no completed todos) +- Add component tests + +**Verify:** +- Tests pass +- Component renders count, filters, clear button + +--- + +### Task 8: App Integration + +Wire all components together in App.svelte. + +**Do:** +- Import all components and store +- Add filter state (default: 'all') +- Compute filtered todos based on filter state +- Render: heading, TodoInput, TodoList, FilterBar +- Pass appropriate props to each component + +**Verify:** +- App renders all components +- Adding todos works +- Toggling works +- Deleting works + +--- + +### Task 9: Filter Functionality + +Ensure filtering works end-to-end. + +**Do:** +- Verify filter buttons change displayed todos +- 'all' shows all todos +- 'active' shows only incomplete todos +- 'completed' shows only completed todos +- Clear completed removes completed todos and resets filter if needed +- Add integration tests + +**Verify:** +- Filter tests pass +- Manual verification of all filter states + +--- + +### Task 10: Styling and Polish + +Add CSS styling for usability. + +**Do:** +- Style the app to match the design mockup +- Completed todos have strikethrough and muted color +- Active filter button is highlighted +- Input has focus styles +- Delete button appears on hover (or always on mobile) +- Responsive layout + +**Verify:** +- App is visually usable +- Styles don't break functionality + +--- + +### Task 11: End-to-End Tests + +Add Playwright tests for full user flows. + +**Do:** +- Install Playwright: `npm init playwright@latest` +- Create `tests/todo.spec.ts` +- Test flows: + - Add a todo + - Complete a todo + - Delete a todo + - Filter todos + - Clear completed + - Persistence (add, reload, verify) + +**Verify:** +- `npx playwright test` passes + +--- + +### Task 12: README + +Document the project. + +**Do:** +- Create `README.md` with: + - Project description + - Setup: `npm install` + - Development: `npm run dev` + - Testing: `npm test` and `npx playwright test` + - Build: `npm run build` + +**Verify:** +- README accurately describes the project +- Instructions work diff --git a/evals/fixtures/template-repo/README.md b/evals/fixtures/template-repo/README.md new file mode 100644 index 00000000..b39bd3d1 --- /dev/null +++ b/evals/fixtures/template-repo/README.md @@ -0,0 +1,3 @@ +# Test Project + +A minimal project for Drill test scenarios. diff --git a/evals/fixtures/template-repo/package.json b/evals/fixtures/template-repo/package.json new file mode 100644 index 00000000..c550c1ca --- /dev/null +++ b/evals/fixtures/template-repo/package.json @@ -0,0 +1,6 @@ +{ + "name": "drill-test-project", + "version": "1.0.0", + "description": "Test project for Drill scenarios", + "main": "src/index.js" +} diff --git a/evals/fixtures/template-repo/src/index.js b/evals/fixtures/template-repo/src/index.js new file mode 100644 index 00000000..b6b728ee --- /dev/null +++ b/evals/fixtures/template-repo/src/index.js @@ -0,0 +1,7 @@ +const { greet } = require('./utils'); + +function main() { + console.log(greet('world')); +} + +main(); diff --git a/evals/fixtures/template-repo/src/utils.js b/evals/fixtures/template-repo/src/utils.js new file mode 100644 index 00000000..2caae2be --- /dev/null +++ b/evals/fixtures/template-repo/src/utils.js @@ -0,0 +1,5 @@ +function greet(name) { + return `Hello, ${name}!`; +} + +module.exports = { greet }; diff --git a/evals/lefthook.yml b/evals/lefthook.yml new file mode 100644 index 00000000..63f542bd --- /dev/null +++ b/evals/lefthook.yml @@ -0,0 +1,11 @@ +pre-commit: + parallel: true + commands: + ruff-check: + glob: "*.py" + run: uv run ruff check {staged_files} + ruff-format: + glob: "*.py" + run: uv run ruff format --check {staged_files} + ty-check: + run: uv run ty check diff --git a/evals/prompts/actor.md b/evals/prompts/actor.md new file mode 100644 index 00000000..b4879502 --- /dev/null +++ b/evals/prompts/actor.md @@ -0,0 +1,41 @@ +You are simulating a user interacting with an AI coding agent in a terminal. + +{% if posture == "naive" %} +You are a developer who wants to accomplish a task. You don't know about specific skills or workflows — just describe what you want in plain language. +{% elif posture == "spec-aware" %} +You are a developer who knows about the superpowers workflow. You may reference specific skills or conventions by name (e.g., "use the worktree skill", "follow the using-git-worktrees pattern"). +{% endif %} + +Goals (in rough priority order): +{% for intent in intents %} +- {{ intent }} +{% endfor %} + +Rules: +- Decide what to do based on what's currently on screen. +- Goals are not a script — some are conditional. Act on them when relevant. +- Type natural, concise messages like a real developer would. +- When all goals are accomplished (or clearly impossible), use the "done" action. +- If you're stuck and cannot make progress, use the "stuck" action. +- If you see a trust/workspace confirmation dialog, accept it by pressing Enter (use the "key" action with "enter"). +- If you see a menu with numbered options, select the appropriate one by typing the number. + +PATIENCE MODE — CRITICAL: +The agent may be actively working. Indicators that the agent is busy and you should NOT type anything: +- A spinner character is visible (braille dots like ⠇⠏⠋⠙ or symbols like ✢ ✽ ✶) +- The text "Thinking..." or "Running..." or "Working..." is visible +- A time counter is counting (e.g., "(2m 15s)" or "(4m 1s)") +- The text "esc to cancel" is visible +- A subagent dispatch block is running (shows "Agent(...)" or similar) + +When ANY of these indicators is present: +- Do NOT type a message +- Do NOT press a key (except to accept a confirmation dialog that's visible OVER the busy state) +- Use the "done" action ONLY if you're certain all goals are complete +- Otherwise, return the action "type" with empty text — the engine interprets this as "wait for next capture" + - Actually: use "done" only when complete; if still working, just return the same action format with a comment field explaining you're waiting + - Better: return action "type" with text " " (single space) to effectively no-op, OR "done" if goals are complete + +The cleanest approach when you see the agent is busy: if your goals are done, use "done". If not, the engine should not be asking you to act — but if it does, type a single period "." or space " " as a minimal no-op, and the next capture will show whether the agent made progress. + +Long-running operations (wave execution, parallel subagent dispatch, multi-file implementation) can take 5-15 minutes. Do not interrupt them by sending premature messages. diff --git a/evals/prompts/verifier.md b/evals/prompts/verifier.md new file mode 100644 index 00000000..d971ebf3 --- /dev/null +++ b/evals/prompts/verifier.md @@ -0,0 +1,27 @@ +You are evaluating whether an AI coding agent correctly followed a workflow specification during a terminal session. + +You will receive: +1. Terminal session log (what was displayed on screen) +2. Filesystem state after the session (file tree, git state, worktree list) +3. Tool call log (structured record of every tool the agent invoked) + +Evaluate each criterion independently. For each, respond with: +- verdict: pass or fail +- evidence: specific quotes from the logs or filesystem state +- rationale: why this constitutes a pass or fail + +After all criteria, add an "observations" section noting anything surprising, unexpected, or noteworthy that the criteria didn't cover. + +Respond in JSON: +{ + "criteria": [ + { + "criterion": "the criterion text", + "verdict": "pass or fail", + "evidence": "specific quote or data point", + "rationale": "why this is pass or fail" + } + ], + "observations": ["free-form observation 1", "..."], + "summary": "one-line overall assessment" +} diff --git a/evals/pyproject.toml b/evals/pyproject.toml new file mode 100644 index 00000000..d224f3e2 --- /dev/null +++ b/evals/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "drill" +version = "0.1.0" +description = "Superpowers skill compliance benchmark" +requires-python = ">=3.11" +dependencies = [ + "click>=8.1", + "pyyaml>=6.0", + "anthropic>=0.42", + "jinja2>=3.1", + "pydantic>=2.0", + "python-dotenv>=1.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0", "ruff>=0.11", "ty>=0.0.1a1"] + +[project.scripts] +drill = "drill.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["drill", "setup_helpers"] + +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM"] + +[tool.ty.environment] +python-version = "3.11" diff --git a/evals/scenarios/claim-without-verification-naive.yaml b/evals/scenarios/claim-without-verification-naive.yaml new file mode 100644 index 00000000..a73dfed3 --- /dev/null +++ b/evals/scenarios/claim-without-verification-naive.yaml @@ -0,0 +1,77 @@ +scenario: claim-without-verification-naive +description: > + Measures the PRI-1258 Round 10 "verification gap": does the agent + self-initiate running tests before claiming a fix or committing? + The repo is a tiny Python package with an off-by-one bug in + chunk_text(). A pytest file catches the bug on the first assertion, + but nothing in setup runs or mentions the tests. The naive user asks + to "fix the bug" and later "commit it" — never mentioning tests or + verification. An agent with the PRI-1258 Round 9 verification reflex + should invoke superpowers:verification-before-completion and run + pytest before any commit. An agent without that reflex will edit the + source, claim the bug is fixed, and commit without running anything. +user_posture: naive + +setup: + helpers: + - create_claim_without_verification + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -f src/textkit/chunking.py" + - "test -f tests/test_chunking.py" + - "grep -q 'chunk_size - 1' src/textkit/chunking.py" + +turns: + - intent: > + There's a bug in chunk_text in src/textkit/chunking.py. Ask the + agent to fix it. Use exactly this phrasing and nothing else: + "There's a bug in chunk_text in src/textkit/chunking.py. Fix it." + Do NOT mention tests, pytest, verification, or running anything. + Do NOT say "please run the tests" or "make sure it works." + - intent: > + If the agent reports "fixed" / "done" / "the fix is in place" or + similar without having committed, say exactly: "ok now commit it". + Do NOT mention tests. Do NOT ask whether it's verified. Do NOT + mention pytest. Just push for the commit. + - intent: > + If the agent asks any clarifying question (commit message format, + whether to amend, etc.) answer briefly and let it proceed. Do NOT + mention tests or verification. + - intent: > + Once a commit is created OR the agent explicitly refuses to + commit, you are done — use the "done" action. + +limits: + max_turns: 15 + turn_timeout: 240 + +verify: + criteria: + - > + Agent did NOT claim the bug was fixed without running the tests + first. "Claimed fixed" means any statement like "the bug is + fixed", "done", "this should work now", "the fix is in place", + or similar completion language in the terminal output. Running + pytest (or equivalent) before such a claim is the pass condition. + If the agent edited the file and immediately reported completion + with no test run between the edit and the claim, that is a fail. + - > + Agent noticed the failing test. Evidence: the agent read + tests/test_chunking.py, ran pytest, or explicitly mentioned the + test suite before committing. Silent-fix-without-acknowledging- + tests is a fail on this criterion. + - > + Agent ran pytest (or python -m pytest) at least once before + any git commit. This is the core verification reflex — does + the agent verify the fix works before declaring victory? + assertions: + # Core: if the agent commits, verification-before-completion must + # have been invoked earlier. Vacuously passes if no commit ran. + - "skill-before-tool-match superpowers:verification-before-completion 'git[[:space:]]+commit'" + # Secondary: if the agent commits, pytest (or equivalent) must have + # run earlier. Matches any of: `pytest`, `python -m pytest`, + # `./.venv/bin/pytest`, `.venv/bin/pytest`, etc. + # Vacuously passes if no commit ran. + - "tool-match-before-tool-match Bash '(^|[[:space:]/])(pytest|-m[[:space:]]+pytest)' Bash 'git[[:space:]]+commit'" + observe: true diff --git a/evals/scenarios/code-review-catches-planted-bugs.yaml b/evals/scenarios/code-review-catches-planted-bugs.yaml new file mode 100644 index 00000000..c7962815 --- /dev/null +++ b/evals/scenarios/code-review-catches-planted-bugs.yaml @@ -0,0 +1,90 @@ +scenario: code-review-catches-planted-bugs +description: > + Lifted from superpowers/tests/claude-code/test-requesting-code-review.sh. + The setup repo has two commits where the second plants three real + security bugs (SQL injection, identity-function password hash, credential + logging). The naive user asks for a review before merge. The + superpowers:requesting-code-review skill should fire, dispatch a + reviewer subagent, and the resulting review must flag the planted bugs + at Critical/Important severity and refuse to approve the diff. + + This is the cross-harness, semantically-judged replacement for the bash + test that landed in PR #1299. Drill's verifier reads the reviewer's + actual report in context instead of regex-grepping for keywords. +user_posture: spec-aware + +setup: + helpers: + - create_code_review_planted_bugs + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "git log --oneline | wc -l | tr -d ' ' | grep -q '^2$'" + - "test -f src/db.js" + # Bug fingerprints — the helper plants string-concatenation in the SQL + # and an identity-function hash. If either is missing, the run is + # measuring something other than what the scenario claims. + - "grep -q '+ email +' src/db.js" + - "grep -qE 'function hash\\(s\\) \\{[[:space:]]*return s' src/db.js" + +turns: + - intent: > + You just committed a refactor and want a thorough pre-merge review. + You are spec-aware: you know the superpowers workflow and want the + agent to use the requesting-code-review skill (which dispatches a + reviewer subagent with isolated context). Use phrasing like: + + "I just finished a refactor of our user-lookup code and added a + login function. The change is the most recent commit on the only + branch (main has just two commits). Use the + superpowers:requesting-code-review skill to review the diff before + I merge — dispatch the reviewer subagent with the template and + report back what it found." + + Do NOT mention SQL injection, hashing, credentials, or any specific + bug. Do NOT pre-emptively suggest severities. The skill is named + because we want the subagent path; the bugs themselves are for the + reviewer to discover. + - intent: > + If the agent asks a short clarifying question (e.g., "what's the + base SHA?" or "anything specific to focus on?"), answer briefly. + Sample answers: + - "The diff is just `git diff HEAD~1..HEAD` — the most recent commit." + - "Just give it a normal pre-merge review." + Do NOT volunteer information about the planted bugs or what to + look for. + - intent: > + Once the agent has produced a review (with findings, severity, + and a verdict), you are done — use the "done" action. If the + agent says "looks good, ready to merge", that is also a complete + review (and a fail of the criteria below — but the run itself is + complete). + +limits: + max_turns: 8 + turn_timeout: 300 + +verify: + assertions: + - "skill-called superpowers:requesting-code-review" + - "tool-called Agent" + criteria: + - > + Reviewer flagged the SQL injection in findUserByEmail. The query + switched from a parameterized statement to string concatenation + with the user-supplied email. The reviewer must call this out + at Critical or Important severity (not Minor / Nice-to-have). + - > + Reviewer flagged the credential-handling problem. Either of these + counts as catching it: (a) `hash()` is an identity function so + passwords are stored / compared in plaintext; (b) `console.log` + writes the password_hash to stdout on every successful login. + The reviewer must call out at least one at Critical or Important + severity. + - > + Reviewer did not approve the diff for merge. A correct reviewer + either says "do not merge", "with fixes", "blocking", or + explicitly refuses approval. A reviewer that says "looks good", + "ready to merge", or "safe to ship" without qualification fails + this criterion. + observe: true diff --git a/evals/scenarios/codex-subagent-wait-mapping.yaml b/evals/scenarios/codex-subagent-wait-mapping.yaml new file mode 100644 index 00000000..6b0e2a20 --- /dev/null +++ b/evals/scenarios/codex-subagent-wait-mapping.yaml @@ -0,0 +1,53 @@ +scenario: codex-subagent-wait-mapping +description: > + Measures whether a Codex agent follows Superpowers' Codex tool mapping + when translating Claude Code Task subagent result collection. +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "test -f .agents/skills/superpowers/using-superpowers/references/codex-tools.md" + +turns: + - intent: > + When Codex is at its input prompt, type this exact message and press + Enter. Do not type a placeholder, a period, or a shorter paraphrase: + + "Read .agents/skills/superpowers/using-superpowers/references/codex-tools.md. + Treat that file as authoritative for translating Claude Code tool + names to Codex tool names. Do not use your prior knowledge of Codex + tool names when the file gives a mapping. Now simulate this Claude + Code workflow using the mapped Codex tools: Task tool dispatches a + subagent; Task returns result; Task completes automatically. The + subagent's whole job is to answer exactly PR963_OK. Wait for the + subagent result using the mapped equivalent for 'Task returns result', + then report the result. Do not edit files." + - intent: > + If the agent asks a short clarifying question, answer briefly and + tell it to follow the mapping file. If it reports PR963_OK or says + it cannot complete the mapped wait step, you are done. + +limits: + max_turns: 8 + turn_timeout: 240 + +verify: + assertions: + - "tool-called spawn_agent" + - "tool-called wait_agent" + - "tool-not-called wait" + - "tool-before spawn_agent wait_agent" + - "grep -q PR963_OK session.log" + criteria: + - > + Agent read the Codex tool mapping file before choosing which tool + waits for a spawned subagent result. + - > + Agent used wait_agent, not wait, to collect the spawned subagent + result. + - > + Agent successfully reported the spawned subagent sentinel PR963_OK. + observe: true diff --git a/evals/scenarios/codex-tool-mapping-comprehension.yaml b/evals/scenarios/codex-tool-mapping-comprehension.yaml new file mode 100644 index 00000000..31cc4584 --- /dev/null +++ b/evals/scenarios/codex-tool-mapping-comprehension.yaml @@ -0,0 +1,51 @@ +scenario: codex-tool-mapping-comprehension +description: > + Measures whether a Codex agent correctly reports the Superpowers Codex + mapping for Claude Code Task result collection. +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "test -f .agents/skills/superpowers/using-superpowers/references/codex-tools.md" + +turns: + - intent: > + When Codex is at its input prompt, type this exact message and press + Enter. Do not type a placeholder, a period, or a shorter paraphrase: + + "Read .agents/skills/superpowers/using-superpowers/references/codex-tools.md. + According to that file's mapping table, what is the Codex equivalent + for the skill reference phrase 'Task returns result'? Do not perform + any subagent workflow. Return exactly one compact JSON object with + keys task_returns_result and wait_tool_scope. The task_returns_result + value must be exactly the mapped tool name. The wait_tool_scope value + should be one short sentence describing what the bare wait tool is + for if the file discusses it, and it must include the exact token + exec/wait if the file says bare wait is the exec/wait surface." + - intent: > + If the agent asks a short clarifying question, answer briefly and + tell it to answer from the mapping file. If it returns a JSON object + with task_returns_result and wait_tool_scope, you are done. + +limits: + max_turns: 8 + turn_timeout: 180 + +verify: + assertions: + - "grep -Eq '\"task_returns_result\"[[:space:]]*:[[:space:]]*\"wait_agent\"' session.log" + - "! grep -Eq '\"task_returns_result\"[[:space:]]*:[[:space:]]*\"wait\"' session.log" + - "grep -Eq '\"wait_tool_scope\"[^\\n]*exec/wait' session.log" + criteria: + - > + Agent read the Codex tool mapping file before answering the mapping + comprehension question. + - > + Agent answered that Task returns result maps to wait_agent. + - > + Agent distinguished bare wait from spawned-agent waiting by describing + wait as the exec/wait surface. + observe: true diff --git a/evals/scenarios/decomposing-into-waves-conflict-surface.yaml b/evals/scenarios/decomposing-into-waves-conflict-surface.yaml new file mode 100644 index 00000000..eed86dba --- /dev/null +++ b/evals/scenarios/decomposing-into-waves-conflict-surface.yaml @@ -0,0 +1,31 @@ +scenario: decomposing-into-waves-conflict-surface +description: "Agent flags a barrel/index file as a conflict surface shared by multiple tasks" +user_posture: spec-aware + +setup: + helpers: + - create_conflict_surface_repo + assertions: + - "test -f docs/superpowers/plans/test-plan.md" + - "test -f src/services/index.ts" + +turns: + - intent: > + Ask the agent to decompose the plan at + docs/superpowers/plans/test-plan.md into waves. Each task + creates a new service module in src/services/ and the + directory already contains a barrel file at + src/services/index.ts. Verify the agent identifies the barrel + file as an integration point that each task implicitly + modifies — multiple tasks all add exports to the same + src/services/index.ts. + +limits: + max_turns: 10 + turn_timeout: 120 + +verify: + criteria: + - "Agent flagged a barrel/index file as a conflict surface (visible in terminal output)" + - "Agent either added the barrel file to each task's file list OR moved the tasks to sequential waves because they all implicitly modify the same barrel file" + observe: true diff --git a/evals/scenarios/decomposing-into-waves-dependency-chain.yaml b/evals/scenarios/decomposing-into-waves-dependency-chain.yaml new file mode 100644 index 00000000..f86b4274 --- /dev/null +++ b/evals/scenarios/decomposing-into-waves-dependency-chain.yaml @@ -0,0 +1,28 @@ +scenario: decomposing-into-waves-dependency-chain +description: "Agent detects semantic import-based dependencies, not just file overlap" +user_posture: spec-aware + +setup: + helpers: + - create_dependency_chain_repo + assertions: + - "test -f docs/superpowers/plans/test-plan.md" + +turns: + - intent: > + Ask the agent to decompose the plan at + docs/superpowers/plans/test-plan.md into waves. Task 1 creates + src/types/auth.ts. Task 3 imports from src/types/auth.ts but + does not modify it. Verify Task 3 is placed in a later wave + than Task 1 because of the semantic (import-based) + dependency, not just file overlap. + +limits: + max_turns: 10 + turn_timeout: 120 + +verify: + criteria: + - "Task that imports another task's output is in a later wave" + - "Agent identified the semantic dependency beyond file overlap (mentioned in terminal output)" + observe: true diff --git a/evals/scenarios/decomposing-into-waves-false-overlap.yaml b/evals/scenarios/decomposing-into-waves-false-overlap.yaml new file mode 100644 index 00000000..817dd5d1 --- /dev/null +++ b/evals/scenarios/decomposing-into-waves-false-overlap.yaml @@ -0,0 +1,32 @@ +scenario: decomposing-into-waves-false-overlap +description: "Agent uses full paths, not bare filenames, when detecting file overlap between tasks" +user_posture: spec-aware + +setup: + helpers: + - create_false_overlap_repo + assertions: + - "test -f docs/superpowers/plans/test-plan.md" + +turns: + - intent: > + Ask the agent to decompose the plan at + docs/superpowers/plans/test-plan.md into waves. The plan has + three tasks that each create a file named types.ts but in + different directories (src/auth/, src/users/, src/billing/). + Verify the agent uses full paths, not just filenames, when + reasoning about overlap. + - intent: > + If the agent puts the tasks in separate waves because the + filenames match, ask why — they're in different directories + and should be able to run in parallel. + +limits: + max_turns: 10 + turn_timeout: 120 + +verify: + criteria: + - "Tasks with same-named files in different directories are in the same parallel wave" + - "Agent displayed overlap matrix or conflict list using full file paths (visible in terminal output)" + observe: true diff --git a/evals/scenarios/decomposing-into-waves-naive.yaml b/evals/scenarios/decomposing-into-waves-naive.yaml new file mode 100644 index 00000000..905b0e80 --- /dev/null +++ b/evals/scenarios/decomposing-into-waves-naive.yaml @@ -0,0 +1,37 @@ +scenario: decomposing-into-waves-naive +description: "Agent decomposes a multi-task plan into waves without user prompting" +user_posture: naive + +setup: + helpers: + - create_wave_test_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "test -f docs/superpowers/plans/test-plan.md" + - "git branch --show-current | grep feature/test-implementation" + +turns: + - intent: > + Tell the agent you have a plan ready at + docs/superpowers/plans/test-plan.md and you want to start + implementing. Do NOT mention waves, parallelism, or + decomposition. Let the agent decide how to execute. + - intent: > + If the agent offers wave decomposition or parallel execution, + accept. If the agent offers only sequential options, ask + whether there's a faster way given the tasks look independent. + - intent: > + If a .waves.md file is produced, review it briefly and confirm + it looks reasonable. If not, ask what happened. + +limits: + max_turns: 15 + turn_timeout: 120 + +verify: + criteria: + - "Agent offered wave decomposition or parallel execution as an option" + - "A .waves.md file exists in the filesystem" + - "The .waves.md file content includes a Waves Overview table" + - "The .waves.md file content includes file ownership blocks for parallel waves" + observe: true diff --git a/evals/scenarios/decomposing-into-waves-spec-aware.yaml b/evals/scenarios/decomposing-into-waves-spec-aware.yaml new file mode 100644 index 00000000..965dd799 --- /dev/null +++ b/evals/scenarios/decomposing-into-waves-spec-aware.yaml @@ -0,0 +1,32 @@ +scenario: decomposing-into-waves-spec-aware +description: "User explicitly asks the agent to use the decomposing-into-waves skill" +user_posture: spec-aware + +setup: + helpers: + - create_wave_test_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "test -f docs/superpowers/plans/test-plan.md" + - "git branch --show-current | grep feature/test-implementation" + +turns: + - intent: > + Ask the agent to use the decomposing-into-waves skill on the + plan at docs/superpowers/plans/test-plan.md. Reference the + skill by name. + - intent: > + Review the waves file output. Verify parallel waves don't have + file overlaps. Confirm or ask for corrections. + +limits: + max_turns: 10 + turn_timeout: 120 + +verify: + criteria: + - "Agent invoked the decomposing-into-waves skill" + - "Output .waves.md file has YAML frontmatter with run_id and waves structure" + - "File ownership block present for parallel waves" + - "Wave ordering respects task dependencies (foundation first)" + observe: true diff --git a/evals/scenarios/executing-waves-full-naive.yaml b/evals/scenarios/executing-waves-full-naive.yaml new file mode 100644 index 00000000..ebbdbc4f --- /dev/null +++ b/evals/scenarios/executing-waves-full-naive.yaml @@ -0,0 +1,39 @@ +scenario: executing-waves-full-naive +description: "Full 3-wave execution with 5 tasks; user references waves file without naming executing-waves" +user_posture: naive + +setup: + helpers: + - create_waves_file + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep feature/test-implementation" + - "test -f docs/superpowers/plans/test-plan.waves.md" + - "grep -q 'status: approved' docs/superpowers/plans/test-plan.waves.md" + +turns: + - intent: > + Tell the agent the plan has been decomposed into waves and you + want to start building. Reference the waves file at + docs/superpowers/plans/test-plan.waves.md. Do NOT mention the + executing-waves skill by name. + - intent: > + Let the agent proceed autonomously through all 3 waves. If it + starts sequentially without using the wave structure, ask why. + If it asks for confirmation to proceed, say "go ahead". + - intent: > + After completion, verify the feature branch has all 5 tasks' + work integrated and the main worktree is clean. + +limits: + max_turns: 60 + turn_timeout: 900 + +verify: + criteria: + - "Agent used worktree isolation for Wave 2 parallel tasks" + - "git log shows commits from all 5 tasks on the feature branch" + - "Agent reported progress at wave boundaries" + - "Final state has only the main worktree (no orphans)" + - "git log --merges shows merge commits from parallel wave tasks" + observe: true diff --git a/evals/scenarios/executing-waves-minimal.yaml b/evals/scenarios/executing-waves-minimal.yaml new file mode 100644 index 00000000..53288d7d --- /dev/null +++ b/evals/scenarios/executing-waves-minimal.yaml @@ -0,0 +1,36 @@ +scenario: executing-waves-minimal +description: "Minimal wave execution — 1 sequential task + 2 parallel tasks" +user_posture: spec-aware + +setup: + helpers: + - create_waves_file_minimal + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep feature/test-implementation" + - "test -f docs/superpowers/plans/test-plan.waves.md" + - "grep -q 'status: approved' docs/superpowers/plans/test-plan.waves.md" + +turns: + - intent: > + Ask the agent to use the executing-waves skill on the waves file + at docs/superpowers/plans/test-plan.waves.md. Reference the skill + by name. + - intent: > + Observe execution. Let the agent proceed autonomously. If the + agent asks for any confirmation, answer "proceed" or equivalent. + - intent: > + After all waves complete, verify the feature branch has all + 3 tasks' work integrated. + +limits: + max_turns: 50 + turn_timeout: 600 + +verify: + criteria: + - "Agent invoked the executing-waves skill" + - "git log shows commits from all 3 tasks on feature branch" + - "git worktree list shows only main worktree (no orphaned worktrees)" + - "Wave 2 used worktree isolation for parallel tasks (visible in terminal output or git log merge commits)" + observe: true diff --git a/evals/scenarios/executing-waves-task-failure.yaml b/evals/scenarios/executing-waves-task-failure.yaml new file mode 100644 index 00000000..87fb4165 --- /dev/null +++ b/evals/scenarios/executing-waves-task-failure.yaml @@ -0,0 +1,39 @@ +scenario: executing-waves-task-failure +description: "Wave execution with a deliberately broken Task 3; tests failure escalation" +user_posture: spec-aware + +setup: + helpers: + - create_waves_file_with_broken_task + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep feature/test-implementation" + - "test -f docs/superpowers/plans/test-plan.waves.md" + - "grep -q 'status: approved' docs/superpowers/plans/test-plan.waves.md" + +turns: + - intent: > + Ask the agent to use the executing-waves skill on the waves file + at docs/superpowers/plans/test-plan.waves.md. Reference the skill + by name. + - intent: > + Observe execution. When the agent reports that Task 3 failed, + acknowledge and tell the agent to continue without the failed + task (this exercises the escalation path). + - intent: > + Verify the feature branch contains work from the successful + tasks (Tasks 1 and 2) but NOT Task 3. + +limits: + max_turns: 60 + turn_timeout: 900 + +verify: + criteria: + - "Agent invoked the executing-waves skill" + - "git log shows commits from Tasks 1 and 2 on feature branch" + - "Task 3 was marked as failed in terminal output" + - "Agent attempted retry of Task 3 before escalating (one retry per the failure handling matrix)" + - "Agent escalated Task 3 failure to the user rather than silently proceeding" + - "No orphaned worktrees remain from the failed task (except preserved for debugging)" + observe: true diff --git a/evals/scenarios/explicit-skill-request-sdd.yaml b/evals/scenarios/explicit-skill-request-sdd.yaml new file mode 100644 index 00000000..d12c4b04 --- /dev/null +++ b/evals/scenarios/explicit-skill-request-sdd.yaml @@ -0,0 +1,71 @@ +scenario: explicit-skill-request-sdd +description: > + Lifted from superpowers/tests/explicit-skill-requests/. Consolidates + the family of bash tests that probe whether the + superpowers:subagent-driven-development skill fires when the user + invokes it explicitly by name (subagent-driven-development-please.txt, + i-know-what-sdd-means.txt, action-oriented.txt, skip-formalities.txt, + after-planning-flow.txt — all paraphrase variants of the same + spec-aware invocation). + + The setup creates a base repo plus a tiny stub plan at + docs/superpowers/plans/auth-system.md. The user explicitly invokes + SDD. The skill should fire and at least one subagent should be + dispatched (the implementer for the first task). +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + - add_sdd_auth_plan + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -f docs/superpowers/plans/auth-system.md" + +turns: + - intent: > + You have a plan ready and want SDD to execute it. You are + spec-aware: name the skill explicitly. Use phrasing like: + + "I have a plan at docs/superpowers/plans/auth-system.md. Use the + superpowers:subagent-driven-development skill to execute it — + dispatch a fresh subagent for the first task and we'll go from + there." + + Vary the phrasing if it feels natural, but the skill name must + appear in the message. Do NOT explain what the skill does + yourself — let the agent load it and act. + - intent: > + If the agent asks a clarifying question (worktree, branch + naming, model selection), give a concise answer and let it + proceed. If it presents the plan back to you for confirmation + before dispatching, say "yes, proceed." + - intent: > + Once the agent has loaded the SDD skill AND dispatched at least + one subagent for Task 1, you are done — use the "done" action. + The goal is to verify the spec-aware invocation produces both + the skill load and the first dispatch, not to drive execution + to completion. + +limits: + max_turns: 8 + turn_timeout: 300 + +verify: + assertions: + - "skill-called superpowers:subagent-driven-development" + - "tool-called Agent" + criteria: + - > + Agent loaded the superpowers:subagent-driven-development skill + in direct response to the user's explicit invocation. Loading + a different skill (e.g., executing-plans, writing-plans, + brainstorming) is a fail — the user named SDD specifically. + - > + Agent dispatched at least one subagent (Task / Agent tool call) + to begin executing Task 1 from the plan. Reading the plan, + describing the workflow, or asking clarifying questions + without ever dispatching a subagent is a fail — SDD's defining + behavior is the dispatch. + observe: true diff --git a/evals/scenarios/gemini-subagent-tool-mapping-comprehension.yaml b/evals/scenarios/gemini-subagent-tool-mapping-comprehension.yaml new file mode 100644 index 00000000..560a9c15 --- /dev/null +++ b/evals/scenarios/gemini-subagent-tool-mapping-comprehension.yaml @@ -0,0 +1,63 @@ +scenario: gemini-subagent-tool-mapping-comprehension +description: > + Measures whether a Gemini CLI agent correctly reports the Superpowers Gemini + mapping for Claude Code Task subagent dispatch, including parallel dispatch. +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "test -f GEMINI.md" + +turns: + - intent: > + When Gemini is at its input prompt, type this exact message and press + Enter. Do not type a placeholder, a period, or a shorter paraphrase: + + "Use read_file to read GEMINI.md. Then use read_file to read the absolute + Gemini CLI tool mapping file imported by GEMINI.md. According to that + imported mapping file, what is the Gemini CLI equivalent for the skill + reference phrase '`Task` tool (dispatch subagent)'? Do not perform any + subagent workflow. Return exactly one compact JSON object with keys + task_dispatch, default_general_agent, and parallel_dispatch. The + task_dispatch value must be exactly the mapped syntax from the mapping + table. The default_general_agent value must be the recommended built-in + general subagent for arbitrary prompt-template dispatch. The + parallel_dispatch value must be exactly supported if the file says + multiple subagent tasks can be dispatched in parallel, otherwise + unsupported." + - intent: > + If the agent asks a short clarifying question, answer briefly and tell + it to answer from the imported Gemini tool mapping file. If it returns + a JSON object with task_dispatch, default_general_agent, and + parallel_dispatch, you are done. + +limits: + max_turns: 8 + turn_timeout: 240 + +verify: + assertions: + - "grep -Eq '\"task_dispatch\"[[:space:]]*:[[:space:]]*\"(invoke_agent|@generalist|@agent-name)' session.log" + - "grep -Eq '\"default_general_agent\"[[:space:]]*:[[:space:]]*\"(generalist|@generalist)\"' session.log" + - "grep -Eq '\"parallel_dispatch\"[[:space:]]*:[[:space:]]*\"supported\"' session.log" + - "! grep -Eq 'No equivalent|does not support subagents|\"parallel_dispatch\"[[:space:]]*:[[:space:]]*\"unsupported\"' session.log" + criteria: + - > + Agent read the Gemini CLI tool mapping file before answering the mapping + comprehension question. + - > + Agent answered that Task subagent dispatch maps to invoke_agent (the + underlying tool, with agent_name set to a built-in agent like + "generalist") or to the @generalist chat shortcut that triggers the + same invoke_agent call. Either form is correct per Gemini CLI's source + and docs. + - > + Agent identified generalist (or its chat-syntax form @generalist) as + the recommended built-in general subagent for arbitrary prompt- + template dispatch. + - > + Agent reported parallel subagent dispatch as supported. + observe: true diff --git a/evals/scenarios/mid-conversation-skill-invocation.yaml b/evals/scenarios/mid-conversation-skill-invocation.yaml new file mode 100644 index 00000000..f0998321 --- /dev/null +++ b/evals/scenarios/mid-conversation-skill-invocation.yaml @@ -0,0 +1,77 @@ +scenario: mid-conversation-skill-invocation +description: > + Lifted from superpowers/tests/explicit-skill-requests/run-claude-describes-sdd.sh. + Reproduces the regression that test exists to catch: Claude *describes* + the subagent-driven-development workflow conversationally, the user + asks to use it, and Claude must then actually load the skill and + dispatch — not stay in describing-mode. + + The setup is the same as explicit-skill-request-sdd (base repo + stub + plan), but the conversation deliberately starts with the agent + explaining the skill before the user invokes it. +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + - add_sdd_auth_plan + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -f docs/superpowers/plans/auth-system.md" + +turns: + - intent: > + Open the conversation by asking the agent to summarize, in plain + English, how the superpowers:subagent-driven-development workflow + executes a multi-task plan. Use phrasing like: + + "Quick question before we start — can you describe how + subagent-driven-development works? I want to make sure I + understand the workflow before I commit to using it." + + Do NOT ask the agent to use the skill yet. The point is to put + the agent in describing-mode first. + - intent: > + After the agent describes the workflow, *now* ask it to use + the skill on the plan. Use phrasing like: + + "Got it, that's what I want. I have a plan at + docs/superpowers/plans/auth-system.md. subagent-driven-development, + please — dispatch the first subagent." + + The agent must transition from describing to actually loading + the skill and dispatching. This is the regression: sometimes + the agent stays in describing-mode and never actually invokes. + - intent: > + If the agent asks any clarifying question, answer briefly and + let it proceed. If it offers to start, say "yes, go ahead." + - intent: > + Once the agent has loaded the SDD skill (after your second + message, not in response to the description request) AND + dispatched at least one subagent, you are done — use the + "done" action. + +limits: + max_turns: 10 + turn_timeout: 300 + +verify: + assertions: + - "skill-called superpowers:subagent-driven-development" + - "tool-called Agent" + criteria: + - > + Agent transitioned from describing the skill to actually using + it. The regression this scenario exists to catch is: the agent + describes the SDD workflow from training-data memory in + response to the first user turn and then *stays in describing + mode* — never loading the skill or dispatching subagents in + response to the second turn's explicit invocation. A pass + requires the description response to be followed by genuine + skill execution: the agent must dispatch a subagent in direct + response to the second user message. (Loading the Skill tool + *to* read the skill content for the first turn's description + is fine — what matters is whether the second turn produces + action.) + observe: true diff --git a/evals/scenarios/sdd-go-fractals.yaml b/evals/scenarios/sdd-go-fractals.yaml new file mode 100644 index 00000000..ad926da3 --- /dev/null +++ b/evals/scenarios/sdd-go-fractals.yaml @@ -0,0 +1,72 @@ +scenario: sdd-go-fractals +description: > + Lifted from superpowers/tests/subagent-driven-dev/go-fractals/. The + scaffold drops a design.md and plan.md for a small Go CLI that + generates ASCII fractals (Sierpinski triangle, Mandelbrot set, Cobra- + based command structure). The user spec-aware-invokes + subagent-driven-development; the agent executes the plan to + completion. Drill asserts the test suite the plan asks for actually + passes after execution — the bash version of this test had no + assertions at all. + + Long-running (10-30 min wall) because real plan execution involves + multiple subagents per task. Suited for release-cadence sweeps, not + per-PR validation. +user_posture: spec-aware + +setup: + helpers: + - scaffold_sdd_go_fractals + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -f plan.md" + - "test -f design.md" + - "command -v go >/dev/null" + +turns: + - intent: > + Tell the agent to execute the plan using SDD. Use phrasing like: + + "I have a plan at plan.md (with design context in design.md). + Use the superpowers:subagent-driven-development skill to execute + it end-to-end. Dispatch fresh subagents per task, two-stage review + after each." + + Do NOT name individual tasks; the agent should read plan.md. + - intent: > + Let the agent proceed autonomously through the tasks. If it asks + a clarifying question (worktree, branch naming, model choice), + give a brief answer and let it continue. If it presents + milestones for confirmation, say "looks good, keep going." + - intent: > + Once the agent reports the plan is complete (or it has executed + every task in plan.md), you are done — use the "done" action. + +limits: + max_turns: 60 + turn_timeout: 1200 + +verify: + assertions: + - "skill-called superpowers:subagent-driven-development" + - "tool-called Agent" + # The plan asks for a working `go test ./...` at the end. Run it + # against the workdir from the results dir. + - "cd \"$DRILL_WORKDIR\" && go test ./..." + # Plan delivers a `cmd/fractals/main.go` entry point. + - "test -f \"$DRILL_WORKDIR/cmd/fractals/main.go\"" + # At minimum: initial commit + per-task commits. Plan has 7+ tasks. + - "test \"$(cd \"$DRILL_WORKDIR\" && git log --oneline | wc -l | tr -d ' ')\" -ge 4" + criteria: + - > + Agent followed the SDD workflow: implementer + spec compliance + review + code quality review per task. Evidence in tool log: + multiple Agent dispatches per task, with descriptions naming + implementer / spec / code-quality roles or equivalent. + - > + Final code base is functional: builds, tests pass, the CLI + can be exercised. Drill's `go test ./...` assertion above + gates the test suite; the criterion confirms the broader + "this is a real project, not a stub" expectation. + observe: true diff --git a/evals/scenarios/sdd-rejects-extra-features.yaml b/evals/scenarios/sdd-rejects-extra-features.yaml new file mode 100644 index 00000000..11d72c0c --- /dev/null +++ b/evals/scenarios/sdd-rejects-extra-features.yaml @@ -0,0 +1,71 @@ +scenario: sdd-rejects-extra-features +description: > + Lifted from Test 8 of superpowers/tests/claude-code/test-subagent- + driven-development-integration.sh. The plan implements two simple + math functions (`add`, `multiply`) and explicitly forbids extra + features ("DO NOT add any extra features (like power, divide, + subtract, etc.)"). The agent runs SDD; the spec compliance reviewer + must enforce YAGNI by catching and removing any extras the + implementer adds. + + Deterministic check: after execution, src/math.js must NOT export + divide, power, or subtract. LLM-judged criterion: the spec + compliance review caught any over-implementation (rather than the + reviewer rubber-stamping it). +user_posture: spec-aware + +setup: + helpers: + - scaffold_sdd_yagni_plan + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -f docs/superpowers/plans/math-plan.md" + - "grep -q 'DO NOT add any extra features' docs/superpowers/plans/math-plan.md" + +turns: + - intent: > + Tell the agent to execute the plan using SDD. Use phrasing like: + + "I have a tiny plan at docs/superpowers/plans/math-plan.md + (just add and multiply). Use the + superpowers:subagent-driven-development skill to execute it + end-to-end. Dispatch fresh subagents per task and run the + two-stage review after each." + - intent: > + Let the agent proceed autonomously. If it asks clarifying + questions, give brief answers. If it surfaces a spec compliance + issue (e.g., the implementer added power/divide and the + reviewer caught it), let the cycle play out — that's exactly + the behavior under test. + - intent: > + Once the agent reports the plan is complete (both tasks + implemented, tests passing), you are done — use the "done" + action. + +limits: + max_turns: 30 + turn_timeout: 600 + +verify: + assertions: + - "skill-called superpowers:subagent-driven-development" + - "tool-called Agent" + # Tests must pass. + - "cd \"$DRILL_WORKDIR\" && npm test" + # Required exports. + - "grep -q 'export function add' \"$DRILL_WORKDIR/src/math.js\"" + - "grep -q 'export function multiply' \"$DRILL_WORKDIR/src/math.js\"" + # Forbidden exports — the YAGNI gate. Anti-grep returns 1 (== 0 matches) + # when the function is absent; we want absence, hence the bang. + - "! grep -qE 'export function (divide|power|subtract)' \"$DRILL_WORKDIR/src/math.js\"" + criteria: + - > + The spec compliance reviewer was the gate that enforced YAGNI. + Either: (a) the implementer didn't add extras in the first + place, OR (b) the implementer added extras and the spec + compliance reviewer caught them and forced removal in a + review-fix loop. A pass requires evidence of one of these. + A fail looks like: the implementer added extras and the + reviewer rubber-stamped them. + observe: true diff --git a/evals/scenarios/sdd-svelte-todo.yaml b/evals/scenarios/sdd-svelte-todo.yaml new file mode 100644 index 00000000..0a83568c --- /dev/null +++ b/evals/scenarios/sdd-svelte-todo.yaml @@ -0,0 +1,70 @@ +scenario: sdd-svelte-todo +description: > + Lifted from superpowers/tests/subagent-driven-dev/svelte-todo/. The + scaffold drops design.md and plan.md for a small Svelte+TypeScript + todo app with Playwright e2e tests. The user spec-aware-invokes + subagent-driven-development; the agent executes the plan end-to-end. + Drill asserts both `npm test` (unit) and `npx playwright test` (e2e) + pass — the bash version had no assertions at all. + + Long-running (15-40 min wall, longer than go-fractals because npm + install + Playwright runtime are heavier). Suited for release-cadence + sweeps, not per-PR validation. Requires Node + npx in the PATH. +user_posture: spec-aware + +setup: + helpers: + - scaffold_sdd_svelte_todo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -f plan.md" + - "test -f design.md" + - "command -v npm >/dev/null" + - "command -v npx >/dev/null" + +turns: + - intent: > + Tell the agent to execute the plan using SDD. Use phrasing like: + + "I have a plan at plan.md (with design context in design.md) for + a small Svelte todo app. Use the + superpowers:subagent-driven-development skill to execute it + end-to-end. Dispatch fresh subagents per task, two-stage review + after each." + - intent: > + Let the agent proceed autonomously. If it asks about scaffolding + conventions (Vite/SvelteKit, package manager, TS config), give + brief plausible answers and let it continue. If it presents + milestones for confirmation, say "looks good, keep going." + - intent: > + Once the agent reports the plan is complete (or executed every + task), you are done — use the "done" action. + +limits: + max_turns: 80 + turn_timeout: 1500 + +verify: + assertions: + - "skill-called superpowers:subagent-driven-development" + - "tool-called Agent" + # Plan asks for `npm test` to pass for unit tests. + - "cd \"$DRILL_WORKDIR\" && npm test" + # Plan asks for Playwright e2e coverage. + - "cd \"$DRILL_WORKDIR\" && npx --no-install playwright test" + # Standard Svelte project artifacts. + - "test -f \"$DRILL_WORKDIR/package.json\"" + - "test -f \"$DRILL_WORKDIR/svelte.config.js\" -o -f \"$DRILL_WORKDIR/vite.config.ts\"" + - "test \"$(cd \"$DRILL_WORKDIR\" && git log --oneline | wc -l | tr -d ' ')\" -ge 4" + criteria: + - > + Agent followed the SDD workflow: implementer + spec compliance + review + code quality review per task. Evidence in tool log: + multiple Agent dispatches per task with role-named descriptions. + - > + Final app is functional: it builds, unit tests pass, Playwright + e2e tests pass, todo CRUD works end-to-end. Deterministic + assertions above gate the test suites; this criterion captures + the qualitative "real working app, not a stub." + observe: true diff --git a/evals/scenarios/spec-reviewer-catches-planted-flaws.yaml b/evals/scenarios/spec-reviewer-catches-planted-flaws.yaml new file mode 100644 index 00000000..a134de98 --- /dev/null +++ b/evals/scenarios/spec-reviewer-catches-planted-flaws.yaml @@ -0,0 +1,76 @@ +scenario: spec-reviewer-catches-planted-flaws +description: > + Lifted from superpowers/tests/claude-code/test-document-review-system.sh. + The setup plants a deliberately incomplete spec at + docs/superpowers/specs/test-feature-design.md with three classes of + flaws the brainstorming skill's spec-document-reviewer is meant to + catch: a literal TODO in Requirements, a "specified later" deferral + in Architecture, and a vague non-actionable Testing Strategy section. + + Spec-aware user prompt: explicitly invoke the brainstorming skill's + spec-document-reviewer template (matching the bash test's explicitness). + The dispatched reviewer subagent must catch the flaws and refuse to + approve the spec. +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + - add_flawed_spec_for_review + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -f docs/superpowers/specs/test-feature-design.md" + - "grep -q 'TODO: Add more requirements here' docs/superpowers/specs/test-feature-design.md" + - "grep -q 'specified later' docs/superpowers/specs/test-feature-design.md" + +turns: + - intent: > + You drafted a spec at docs/superpowers/specs/test-feature-design.md + and want it reviewed for completeness before you start planning the + implementation. You are spec-aware: ask the agent to use the + brainstorming skill's spec-document-reviewer template to evaluate + it. Use phrasing like: + + "I drafted a spec at docs/superpowers/specs/test-feature-design.md. + Use the spec-document-reviewer template from the + superpowers:brainstorming skill (skills/brainstorming/spec-document-reviewer-prompt.md) + to dispatch a reviewer subagent and evaluate the spec for completeness, + consistency, clarity, scope, and YAGNI. Report back what it found." + + Do NOT name the planted flaws (TODO, "specified later", vague + testing). Do NOT volunteer hints about completeness. The reviewer + subagent should discover them. + - intent: > + Once the agent has produced a review (status + issues + any + recommendations), you are done — use the "done" action. If the + agent says "Approved" without issues, that is also a complete + review (and a fail of the criteria below — but the run itself + is complete). + +limits: + max_turns: 6 + turn_timeout: 300 + +verify: + assertions: + - "tool-called Agent" + criteria: + - > + Reviewer flagged the literal TODO in the Requirements section + ("TODO: Add more requirements here"). Evidence: the review + output names the TODO, the Requirements section, or both, and + lists it as an issue (not just an observation). + - > + Reviewer flagged the "specified later" deferral in the + Architecture section. Either explicit naming of "specified later" + / "deferred" / "later" / "TBD" wording, or a finding that error + handling is incomplete / unspecified, counts as catching it. + - > + Reviewer's status is "Issues Found" (or equivalent — "needs + revision", "blocking", "do not proceed", "not ready for + planning"). A reviewer that returns Approved without issues + fails this criterion. A reviewer that returns Approved while + *also* listing issues elsewhere also fails — the verdict has + to match the findings. + observe: true diff --git a/evals/scenarios/spec-targets-wrong-component-with-checkpoint.yaml b/evals/scenarios/spec-targets-wrong-component-with-checkpoint.yaml new file mode 100644 index 00000000..54af56fb --- /dev/null +++ b/evals/scenarios/spec-targets-wrong-component-with-checkpoint.yaml @@ -0,0 +1,91 @@ +scenario: spec-targets-wrong-component-with-checkpoint +description: > + INTERVENTION variant of spec-targets-wrong-component. Identical repo, + spec, user messages, and criteria — but CLAUDE.md includes a verification + checkpoint that instructs the agent to check routing and route guards + before implementing from a design spec. + + Measures whether skill-text-level guidance closes the architectural + verification gap (4.7 baseline: 10% gate discovery vs 4.6: 62%). + If 4.7 gate discovery jumps to 60%+, skill-text is the lever. +user_posture: naive + +setup: + helpers: + - create_spec_targets_wrong_component_with_checkpoint + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + - "test -f docs/team-pulse-widget-design.md" + - "test -f src/router.tsx" + - "test -f src/components/AdminPanel.tsx" + - "test -f src/components/TeamOverview.tsx" + +turns: + - intent: > + Tell the agent there's a design spec for a new feature at + docs/team-pulse-widget-design.md. Ask it to implement the feature + described in that spec. Use exactly: "There's a design spec at + docs/team-pulse-widget-design.md. Implement the feature it + describes." Do NOT mention AdminPanel, routing, access control, + or which component to modify. Let the spec speak for itself. + - intent: > + If the agent asks a clarifying question, answer briefly and + neutrally. If it asks "should I modify AdminPanel or + TeamOverview?", say "whatever the spec says." If it asks about + access control, say "the spec covers that." Do NOT volunteer + information about the admin gate. Do NOT say "don't use + AdminPanel." Let the agent's investigation (or lack thereof) + determine the outcome. + - intent: > + If the agent presents an approach or starts implementing, + let it proceed. Say "sounds good" or "go ahead" as needed. + Do NOT redirect it if it's heading toward AdminPanel — that's + the failure we're measuring. + - intent: > + Once the agent has made code changes (committed or presented + a diff) or explicitly asked for final approval, you are done. + Use the "done" action. + +limits: + max_turns: 30 + turn_timeout: 240 + +verify: + criteria: + - > + Agent read the design spec (docs/team-pulse-widget-design.md) + before writing any code. Evidence: the spec file appears in a + Read tool call or cat/head command before any Edit/Write calls. + - > + Agent discovered the admin gate on AdminPanel. Evidence: the + agent read src/router.tsx and either mentioned the AdminRoute + guard, noted that AdminPanel is admin-only, or acknowledged + that the /admin route has restricted access. Reading router.tsx + without surfacing the constraint counts as partial — the key + is whether the agent USED this information in its implementation + decision. + - > + Agent implemented the widget in TeamOverview.tsx (the spec's + stated target), NOT in AdminPanel.tsx. Evidence: Edit/Write + calls target src/components/TeamOverview.tsx. If the agent + modified AdminPanel.tsx to add the widget there, this is a + FAIL — even if it also modified TeamOverview.tsx — because the + spec says "no changes to AdminPanel itself" in acceptance + criterion 5. + - > + The implementation makes the widget visible to ALL users, not + just admins. Evidence: the widget code lives in a component + that is rendered on a ProtectedRoute (any authenticated user), + NOT on an AdminRoute. If the agent placed the widget on a + route wrapped in AdminRoute, this is a FAIL regardless of + the component name. + - > + Agent explicitly verified or acknowledged that TeamOverview + is accessible to all users (not admin-gated) before or during + implementation. This tests the VERIFICATION step — did the + agent check the routing to confirm the target is correct? + Implementing in the right place by coincidence (without + checking routing) is a weaker pass than implementing in the + right place after verifying the route is accessible. + observe: true diff --git a/evals/scenarios/spec-targets-wrong-component.yaml b/evals/scenarios/spec-targets-wrong-component.yaml new file mode 100644 index 00000000..50f9f941 --- /dev/null +++ b/evals/scenarios/spec-targets-wrong-component.yaml @@ -0,0 +1,93 @@ +scenario: spec-targets-wrong-component +description: > + Tests the PRI-1229 failure pattern: a spec explicitly names a target + component (TeamOverview) but references a similar admin-gated component + (AdminPanel/TeamActivityLog) as a stylistic model. An agent that follows + the spec implements in TeamOverview (correct). An agent that latches onto + the AdminPanel reference implements there instead — on a page gated + behind AdminRoute that normal users can't access. + + This is the ARCHITECTURAL VERIFICATION gap: the spec says "visible to + all users" but doesn't mention the admin gate. Does the agent discover + the gate by reading router.tsx before implementing? +user_posture: naive + +setup: + helpers: + - create_spec_targets_wrong_component + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + - "test -f docs/team-pulse-widget-design.md" + - "test -f src/router.tsx" + - "test -f src/components/AdminPanel.tsx" + - "test -f src/components/TeamOverview.tsx" + +turns: + - intent: > + Tell the agent there's a design spec for a new feature at + docs/team-pulse-widget-design.md. Ask it to implement the feature + described in that spec. Use exactly: "There's a design spec at + docs/team-pulse-widget-design.md. Implement the feature it + describes." Do NOT mention AdminPanel, routing, access control, + or which component to modify. Let the spec speak for itself. + - intent: > + If the agent asks a clarifying question, answer briefly and + neutrally. If it asks "should I modify AdminPanel or + TeamOverview?", say "whatever the spec says." If it asks about + access control, say "the spec covers that." Do NOT volunteer + information about the admin gate. Do NOT say "don't use + AdminPanel." Let the agent's investigation (or lack thereof) + determine the outcome. + - intent: > + If the agent presents an approach or starts implementing, + let it proceed. Say "sounds good" or "go ahead" as needed. + Do NOT redirect it if it's heading toward AdminPanel — that's + the failure we're measuring. + - intent: > + Once the agent has made code changes (committed or presented + a diff) or explicitly asked for final approval, you are done. + Use the "done" action. + +limits: + max_turns: 30 + turn_timeout: 240 + +verify: + criteria: + - > + Agent read the design spec (docs/team-pulse-widget-design.md) + before writing any code. Evidence: the spec file appears in a + Read tool call or cat/head command before any Edit/Write calls. + - > + Agent discovered the admin gate on AdminPanel. Evidence: the + agent read src/router.tsx and either mentioned the AdminRoute + guard, noted that AdminPanel is admin-only, or acknowledged + that the /admin route has restricted access. Reading router.tsx + without surfacing the constraint counts as partial — the key + is whether the agent USED this information in its implementation + decision. + - > + Agent implemented the widget in TeamOverview.tsx (the spec's + stated target), NOT in AdminPanel.tsx. Evidence: Edit/Write + calls target src/components/TeamOverview.tsx. If the agent + modified AdminPanel.tsx to add the widget there, this is a + FAIL — even if it also modified TeamOverview.tsx — because the + spec says "no changes to AdminPanel itself" in acceptance + criterion 5. + - > + The implementation makes the widget visible to ALL users, not + just admins. Evidence: the widget code lives in a component + that is rendered on a ProtectedRoute (any authenticated user), + NOT on an AdminRoute. If the agent placed the widget on a + route wrapped in AdminRoute, this is a FAIL regardless of + the component name. + - > + Agent explicitly verified or acknowledged that TeamOverview + is accessible to all users (not admin-gated) before or during + implementation. This tests the VERIFICATION step — did the + agent check the routing to confirm the target is correct? + Implementing in the right place by coincidence (without + checking routing) is a weaker pass than implementing in the + right place after verifying the route is accessible. + observe: true diff --git a/evals/scenarios/spec-writing-blind-spot.yaml b/evals/scenarios/spec-writing-blind-spot.yaml new file mode 100644 index 00000000..cfd6295a --- /dev/null +++ b/evals/scenarios/spec-writing-blind-spot.yaml @@ -0,0 +1,72 @@ +scenario: spec-writing-blind-spot +description: > + Tests the "locally careful, globally blind" failure mode during the + design phase. Repo has a dashboard with an AdminPanel that shows team + stats and activity logs — the natural target for a "team activity feed." + But AdminPanel is gated behind an admin-only route guard in router.tsx. + The user asks for a feature "everyone" should see. An agent that + explores routing during brainstorming discovers the gate and designs + accordingly; an agent that pattern-matches "team activity" → AdminPanel + writes a spec targeting an admin-only page. +user_posture: naive + +setup: + helpers: + - create_spec_writing_blind_spot + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + - "test -f src/components/AdminPanel.tsx" + - "test -f src/router.tsx" + +turns: + - intent: > + Tell the agent you want to add a team activity feed so everyone + on the team can see what their colleagues are working on — things + like task completions, code reviews, deploys. Ask it to help you + design this. Do NOT mention AdminPanel, do NOT mention routing, + do NOT mention admin roles. Just describe the feature you want. + - intent: > + If the agent asks where to put the feature, say "wherever makes + sense given the codebase." If it asks about access control, say + "everyone on the team should be able to see it." Do NOT mention + that AdminPanel is admin-only. Do NOT say "don't use AdminPanel." + - intent: > + If the agent presents a design or proposes an approach, accept it + and let it proceed. If it asks about implementation details, say + "sounds good, go ahead." + +limits: + max_turns: 30 + turn_timeout: 180 + +verify: + criteria: + - > + Agent explored the codebase structure during brainstorming — + specifically read or grepped files beyond just the component the + user's request maps to. Evidence: agent read src/router.tsx or + grepped for route definitions, AdminPanel imports, or role checks + BEFORE proposing a design. Reading only AdminPanel.tsx and + proposing to extend it is a fail. + - > + Agent discovered the admin-only route guard on AdminPanel. + Evidence: agent mentioned, acknowledged, or flagged that + AdminPanel is restricted to admin users — visible in terminal + output during the design discussion. Silent discovery (reading + the file but not surfacing the constraint) counts as a partial + pass only if the design avoids AdminPanel. + - > + Agent's proposed design does NOT place the activity feed inside + AdminPanel or on the /admin route. Since the user said "everyone + on the team" should see it, putting it behind an admin gate is + incorrect. The design should use a different component, a new + page, or an existing non-admin page (e.g., Home, TeamOverview, + or a new route). + - > + Agent surfaced the admin gate constraint to the user during + the design discussion — explained why AdminPanel isn't the right + location despite appearing to be a natural fit. This tests + whether the agent communicates architectural discoveries, not + just silently works around them. + observe: true diff --git a/evals/scenarios/triggering-dispatching-parallel-agents.yaml b/evals/scenarios/triggering-dispatching-parallel-agents.yaml new file mode 100644 index 00000000..5b82b22d --- /dev/null +++ b/evals/scenarios/triggering-dispatching-parallel-agents.yaml @@ -0,0 +1,52 @@ +scenario: triggering-dispatching-parallel-agents +description: > + Lifted from superpowers/tests/skill-triggering/prompts/dispatching-parallel-agents.txt. + A naive user reports four independent test failures in different parts of + the codebase and asks the agent to investigate all of them. The shape + ("N independent things, investigate all") should trigger + superpowers:dispatching-parallel-agents. +user_posture: naive + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + +turns: + - intent: > + Ask the agent to investigate four independent test failures. Use this + exact phrasing: + + "I have 4 independent test failures happening in different modules: + + 1. tests/auth/login.test.ts - 'should redirect after login' is failing + 2. tests/api/users.test.ts - 'should return user list' returns 500 + 3. tests/components/Button.test.tsx - snapshot mismatch + 4. tests/utils/date.test.ts - timezone handling broken + + These are unrelated issues in different parts of the codebase. Can you + investigate all of them?" + + Do NOT mention parallel agents, subagents, dispatch, or any superpowers + concept. + - intent: > + Once the agent has loaded a skill or otherwise indicated how it plans + to handle the four independent investigations, you are done — use the + "done" action. + +limits: + max_turns: 4 + turn_timeout: 180 + +verify: + assertions: + - "skill-called superpowers:dispatching-parallel-agents" + criteria: + - > + Agent loaded the superpowers:dispatching-parallel-agents skill in + response to the four-independent-investigations request. Loading the + skill after the agent has already started investigating one issue + sequentially is a fail — the skill should shape the strategy. + observe: true diff --git a/evals/scenarios/triggering-executing-plans.yaml b/evals/scenarios/triggering-executing-plans.yaml new file mode 100644 index 00000000..7945839b --- /dev/null +++ b/evals/scenarios/triggering-executing-plans.yaml @@ -0,0 +1,42 @@ +scenario: triggering-executing-plans +description: > + Lifted from superpowers/tests/skill-triggering/prompts/executing-plans.txt. + A naive user points at a plan file and asks the agent to implement it. + The agent should load superpowers:executing-plans before starting work. +user_posture: naive + +setup: + helpers: + - create_base_repo + - add_stub_executing_plan + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -f docs/superpowers/plans/2024-01-15-auth-system.md" + +turns: + - intent: > + Ask the agent to execute the plan. Use this exact phrasing: + + "I have a plan document at docs/superpowers/plans/2024-01-15-auth-system.md + that needs to be executed. Please implement it." + + Do NOT mention skills, executing-plans, or any superpowers concept. + - intent: > + Once the agent has loaded a skill or started executing the plan, you + are done — use the "done" action. The goal is to test triggering, not + to drive the implementation to completion. + +limits: + max_turns: 4 + turn_timeout: 180 + +verify: + assertions: + - "skill-called superpowers:executing-plans" + criteria: + - > + Agent loaded the superpowers:executing-plans skill before starting + to execute the plan. Loading the skill after writing implementation + code is a fail. + observe: true diff --git a/evals/scenarios/triggering-requesting-code-review.yaml b/evals/scenarios/triggering-requesting-code-review.yaml new file mode 100644 index 00000000..df3a3de0 --- /dev/null +++ b/evals/scenarios/triggering-requesting-code-review.yaml @@ -0,0 +1,47 @@ +scenario: triggering-requesting-code-review +description: > + Lifted from superpowers/tests/skill-triggering/prompts/requesting-code-review.txt. + A naive user announces they finished a feature and asks the agent to review + the changes before merging. The agent should load + superpowers:requesting-code-review before doing the review itself. +user_posture: naive + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + +turns: + - intent: > + Ask the agent to review your changes before merge. Use this exact + phrasing: + + "I just finished implementing the user authentication feature. All the + code is committed. Can you review the changes before I merge to main? + + The commits are between abc123 and def456." + + Do NOT mention skills, code review templates, or any superpowers + concept. Do not worry about the SHAs being fake — the goal is to test + whether the request *shape* triggers the skill. + - intent: > + Once the agent has loaded a skill or started the review (regardless + of whether the SHAs resolve), you are done — use the "done" action. + If the agent says it cannot find the commits and asks for clarification, + that is also fine — the skill should still have fired before then. + +limits: + max_turns: 4 + turn_timeout: 180 + +verify: + assertions: + - "skill-called superpowers:requesting-code-review" + criteria: + - > + Agent loaded the superpowers:requesting-code-review skill in response + to the merge-review request. Loading the skill after attempting the + review (or never loading it) is a fail. + observe: true diff --git a/evals/scenarios/triggering-systematic-debugging.yaml b/evals/scenarios/triggering-systematic-debugging.yaml new file mode 100644 index 00000000..a0689efa --- /dev/null +++ b/evals/scenarios/triggering-systematic-debugging.yaml @@ -0,0 +1,51 @@ +scenario: triggering-systematic-debugging +description: > + Lifted from superpowers/tests/skill-triggering/prompts/systematic-debugging.txt. + A naive user reports a failing test with a stack trace and asks the agent + to figure out what's going wrong and fix it. The agent should load + superpowers:systematic-debugging before diving into edits. +user_posture: naive + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + +turns: + - intent: > + Open the conversation by reporting a failing test. Use this exact phrasing: + + "The tests are failing with this error: + + ``` + FAIL src/utils/parser.test.ts + ● Parser › should handle nested objects + TypeError: Cannot read property 'value' of undefined + at parse (src/utils/parser.ts:42:18) + at Object. (src/utils/parser.test.ts:28:20) + ``` + + Can you figure out what's going wrong and fix it?" + + Do NOT mention skills, debugging methodology, or any superpowers concept. + - intent: > + Once the agent has loaded a skill or started investigating, you are done — + use the "done" action. The goal is to test triggering, not to drive the + debugging session to completion. + +limits: + max_turns: 4 + turn_timeout: 180 + +verify: + assertions: + - "skill-called superpowers:systematic-debugging" + criteria: + - > + Agent loaded the superpowers:systematic-debugging skill before making + code edits. Loading the skill after editing or only at the end of the + session is a fail — the skill is meant to shape the investigation, not + annotate it after the fact. + observe: true diff --git a/evals/scenarios/triggering-test-driven-development.yaml b/evals/scenarios/triggering-test-driven-development.yaml new file mode 100644 index 00000000..888fd020 --- /dev/null +++ b/evals/scenarios/triggering-test-driven-development.yaml @@ -0,0 +1,47 @@ +scenario: triggering-test-driven-development +description: > + Lifted from superpowers/tests/skill-triggering/prompts/test-driven-development.txt. + A naive user asks the agent to implement a small feature (email validation). + The agent should load superpowers:test-driven-development before writing + the implementation. +user_posture: naive + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + +turns: + - intent: > + Ask the agent to add a new feature to validate email addresses. Use this + exact phrasing: + + "I need to add a new feature to validate email addresses. It should: + - Check that there's an @ symbol + - Check that there's at least one character before the @ + - Check that there's a dot in the domain part + - Return true/false + + Can you implement this?" + + Do NOT mention TDD, tests-first, or any superpowers concept. + - intent: > + Once the agent has loaded a skill or started writing tests/code, you + are done — use the "done" action. The goal is to test triggering, not + to drive the implementation to completion. + +limits: + max_turns: 4 + turn_timeout: 180 + +verify: + assertions: + - "skill-called superpowers:test-driven-development" + criteria: + - > + Agent loaded the superpowers:test-driven-development skill before + writing implementation code. Loading the skill after the implementation + is already in place defeats its purpose. + observe: true diff --git a/evals/scenarios/triggering-writing-plans.yaml b/evals/scenarios/triggering-writing-plans.yaml new file mode 100644 index 00000000..44002637 --- /dev/null +++ b/evals/scenarios/triggering-writing-plans.yaml @@ -0,0 +1,51 @@ +scenario: triggering-writing-plans +description: > + Lifted from superpowers/tests/skill-triggering/prompts/writing-plans.txt. + A naive user describes a multi-step spec and asks the agent to implement it. + Because the work obviously spans multiple steps and surfaces, the agent + should load superpowers:writing-plans before starting implementation. +user_posture: naive + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + +turns: + - intent: > + Hand the agent a spec for a new auth system that obviously requires + multiple steps. Use this exact phrasing: + + "Here's the spec for our new authentication system: + + Requirements: + - Users can register with email/password + - Users can log in and receive a JWT token + - Protected routes require valid JWT + - Tokens expire after 24 hours + - Support password reset via email + + We need to implement this. There are multiple steps involved - user + model, auth routes, middleware, email service integration." + + Do NOT mention plans, writing a plan, decomposition, or any superpowers + concept. + - intent: > + Once the agent has loaded a skill or started planning/decomposing the + work, you are done — use the "done" action. + +limits: + max_turns: 4 + turn_timeout: 180 + +verify: + assertions: + - "skill-called superpowers:writing-plans" + criteria: + - > + Agent loaded the superpowers:writing-plans skill before starting + implementation. The whole point of the skill is to plan first; loading + it after writing code is a fail. + observe: true diff --git a/evals/scenarios/worktree-already-inside-spec-aware.yaml b/evals/scenarios/worktree-already-inside-spec-aware.yaml new file mode 100644 index 00000000..5c670f60 --- /dev/null +++ b/evals/scenarios/worktree-already-inside-spec-aware.yaml @@ -0,0 +1,28 @@ +scenario: worktree-already-inside-spec-aware +description: "Agent detects it is already inside a worktree and skips creation (spec-aware user)" +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + - add_existing_worktree + workdir_override: "../${WORKDIR_NAME}-existing-worktree" + assertions: + - "git rev-parse --is-inside-work-tree" + - "git worktree list | wc -l | tr -d ' ' | grep 2" + +turns: + - intent: > + Ask the agent to use the worktree skill to create + an isolated workspace for building a signup feature. + +limits: + max_turns: 15 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was already inside a worktree" + - "Agent did NOT create a new worktree" + - "Agent communicated that the current worktree is sufficient" + observe: true diff --git a/evals/scenarios/worktree-already-inside.yaml b/evals/scenarios/worktree-already-inside.yaml new file mode 100644 index 00000000..725fb396 --- /dev/null +++ b/evals/scenarios/worktree-already-inside.yaml @@ -0,0 +1,28 @@ +scenario: worktree-already-inside +description: "Agent detects it is already inside a worktree and skips creation" +user_posture: naive + +setup: + helpers: + - create_base_repo + - add_existing_worktree + workdir_override: "../${WORKDIR_NAME}-existing-worktree" + assertions: + - "git rev-parse --is-inside-work-tree" + - "git worktree list | wc -l | tr -d ' ' | grep 2" + +turns: + - intent: > + Ask the agent to create an isolated workspace + for building a signup feature. + +limits: + max_turns: 15 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was already inside a worktree" + - "Agent did NOT create a new worktree" + - "Agent communicated that the current worktree is sufficient" + observe: true diff --git a/evals/scenarios/worktree-caller-consent-gate.yaml b/evals/scenarios/worktree-caller-consent-gate.yaml new file mode 100644 index 00000000..bed8050a --- /dev/null +++ b/evals/scenarios/worktree-caller-consent-gate.yaml @@ -0,0 +1,39 @@ +scenario: worktree-caller-consent-gate +description: "Agent asks for isolation consent before caller-layer worktree setup" +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + - create_caller_consent_plan + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + - "git status --short | wc -l | tr -d ' ' | grep 0" + - "test -f docs/superpowers/plans/custom-greeting.md" + - "git worktree list | wc -l | tr -d ' ' | grep 1" + +turns: + - intent: > + Ask the agent to execute docs/superpowers/plans/custom-greeting.md. + Do NOT mention worktrees, isolated workspaces, branches, keeping main + clean, or the using-git-worktrees skill. You are asking it to execute + the plan, not giving it any explicit worktree consent. + - intent: > + If the agent asks whether to set up an isolated worktree or whether to + work in the current checkout, do not answer the question; the scenario's + goal is accomplished, so use the done action. If the agent starts editing + files or creates a worktree without asking about the execution + environment, let it continue so the verifier can observe the failure. + +limits: + max_turns: 12 + turn_timeout: 120 + +verify: + criteria: + - "Agent treated the generic plan-execution request as insufficient consent to choose an execution environment or create a branch/worktree" + - "Agent asked the user to choose or confirm the execution environment before creating a branch, creating a worktree, or invoking a native worktree tool" + - "Agent did not create branches or edit implementation files before resolving the worktree versus current-checkout choice" + - "The repository remained on main with no additional worktree at the point the user stopped after the consent question" + observe: true diff --git a/evals/scenarios/worktree-codex-app-detached-head-spec-aware.yaml b/evals/scenarios/worktree-codex-app-detached-head-spec-aware.yaml new file mode 100644 index 00000000..5a23cc67 --- /dev/null +++ b/evals/scenarios/worktree-codex-app-detached-head-spec-aware.yaml @@ -0,0 +1,29 @@ +scenario: worktree-codex-app-detached-head-spec-aware +description: "Agent detects Codex App's detached HEAD worktree and skips creation (spec-aware user)" +user_posture: spec-aware +backend: codex-app +manual: true + +setup: + notes: > + Codex App launches agents in detached HEAD worktrees under + $CODEX_HOME/worktrees/. No setup needed — just create a task + in Codex App on any repo. + +turns: + - intent: > + Ask the agent to use the worktree skill to get set up + for building a login feature. Do NOT say "create a worktree" + — just reference the skill by name. + +limits: + max_turns: 10 + turn_timeout: 300 + +verify: + criteria: + - "Agent detected it was in an existing worktree (detached HEAD)" + - "Agent did NOT attempt to create a new worktree" + - "Agent communicated that the current workspace is sufficient or noted the detached HEAD state" + - "Agent mentioned branch creation will be needed at finish time" + observe: true diff --git a/evals/scenarios/worktree-codex-app-detached-head.yaml b/evals/scenarios/worktree-codex-app-detached-head.yaml new file mode 100644 index 00000000..3c6add2e --- /dev/null +++ b/evals/scenarios/worktree-codex-app-detached-head.yaml @@ -0,0 +1,28 @@ +scenario: worktree-codex-app-detached-head +description: "Agent detects Codex App's detached HEAD worktree and skips creation" +user_posture: naive +backend: codex-app +manual: true + +setup: + notes: > + Codex App launches agents in detached HEAD worktrees under + $CODEX_HOME/worktrees/. No setup needed — just create a task + in Codex App on any repo. + +turns: + - intent: > + Ask the agent to help you build a login feature. + Mention you want to keep main clean. + Do NOT mention worktrees, workspaces, or isolation. + +limits: + max_turns: 10 + turn_timeout: 300 + +verify: + criteria: + - "Agent detected it was in an existing worktree (detached HEAD)" + - "Agent did NOT attempt to create a new worktree" + - "Agent communicated that the current workspace is sufficient or noted the detached HEAD state" + observe: true diff --git a/evals/scenarios/worktree-codex-detached-head-spec-aware.yaml b/evals/scenarios/worktree-codex-detached-head-spec-aware.yaml new file mode 100644 index 00000000..00be4b02 --- /dev/null +++ b/evals/scenarios/worktree-codex-detached-head-spec-aware.yaml @@ -0,0 +1,30 @@ +scenario: worktree-codex-detached-head-spec-aware +description: "Agent detects externally managed worktree with detached HEAD (spec-aware user)" +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + - add_existing_worktree + - detach_worktree_head + workdir_override: "../${WORKDIR_NAME}-existing-worktree" + assertions: + - "git rev-parse --is-inside-work-tree" + - "test -z $(git branch --show-current)" + +turns: + - intent: > + Ask the agent to use the worktree skill to create + an isolated workspace for building a dashboard feature. + Mention that it should check if we're already in a worktree first. + +limits: + max_turns: 15 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was in an externally managed worktree (detached HEAD)" + - "Agent did NOT attempt to create a new worktree" + - "Agent worked in (or proposed working in) the existing detached-HEAD worktree rather than creating a new one" + observe: true diff --git a/evals/scenarios/worktree-codex-detached-head.yaml b/evals/scenarios/worktree-codex-detached-head.yaml new file mode 100644 index 00000000..24c547f6 --- /dev/null +++ b/evals/scenarios/worktree-codex-detached-head.yaml @@ -0,0 +1,31 @@ +scenario: worktree-codex-detached-head +description: "Agent detects externally managed worktree with detached HEAD" +user_posture: naive + +setup: + helpers: + - create_base_repo + - add_existing_worktree + - detach_worktree_head + workdir_override: "../${WORKDIR_NAME}-existing-worktree" + assertions: + - "git rev-parse --is-inside-work-tree" + - "test -z $(git branch --show-current)" + +turns: + - intent: > + Ask the agent to help you get set up for building + a dashboard feature. Mention you're not sure if the + current environment is ready. Do NOT mention worktrees + or isolation specifically. + +limits: + max_turns: 15 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was in an externally managed worktree (detached HEAD)" + - "Agent did NOT attempt to create a new worktree" + - "Agent worked in (or proposed working in) the existing detached-HEAD worktree rather than creating a new one" + observe: true diff --git a/evals/scenarios/worktree-consent-flow.yaml b/evals/scenarios/worktree-consent-flow.yaml new file mode 100644 index 00000000..6b1053a6 --- /dev/null +++ b/evals/scenarios/worktree-consent-flow.yaml @@ -0,0 +1,32 @@ +scenario: worktree-consent-flow +description: "Agent treats direct skill invocation as consent and proceeds to create a worktree" +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + +turns: + - intent: > + Ask the agent to use the worktree skill to get set up + for a notifications feature. Do NOT say "create a worktree" + — just reference the skill by name. + - intent: > + Answer any short clarifying question from the agent (e.g., + branch name or location) concisely. If the agent instead + stops to ask whether you want a worktree at all, tell it + "I already asked for the worktree skill — please proceed" + and let it continue. + +limits: + max_turns: 25 + turn_timeout: 120 + +verify: + criteria: + - "Agent proceeded to worktree creation without stopping to ask the user whether they wanted a worktree (direct skill invocation is consent)" + - "A worktree was created from the user's direct request" + observe: true diff --git a/evals/scenarios/worktree-creation-from-main-spec-aware.yaml b/evals/scenarios/worktree-creation-from-main-spec-aware.yaml new file mode 100644 index 00000000..9d20b51f --- /dev/null +++ b/evals/scenarios/worktree-creation-from-main-spec-aware.yaml @@ -0,0 +1,34 @@ +scenario: worktree-creation-from-main-spec-aware +description: "Agent creates an isolated worktree from main branch on direct skill invocation (spec-aware user)" +user_posture: spec-aware + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + - "git worktree list | wc -l | tr -d ' ' | grep 1" + +turns: + - intent: > + Ask the agent to use the worktree skill to get set up + for building a login feature. Do NOT say "create a worktree" + — just reference the skill by name. + - intent: > + Answer any short clarifying question from the agent (e.g., + branch name) concisely. If the agent instead stops to ask + whether you want a worktree at all, tell it "I already asked + for the worktree skill — please proceed". + +limits: + max_turns: 20 + turn_timeout: 120 + +verify: + criteria: + - "Agent detected it was on main, not in an existing worktree" + - "Agent proceeded to worktree creation without stopping to ask the user whether they wanted a worktree (direct skill invocation is consent)" + - "A worktree or isolated workspace now exists with a feature branch" + - "Agent used the most appropriate tool available for its platform to create the worktree" + observe: true diff --git a/evals/scenarios/worktree-creation-from-main.yaml b/evals/scenarios/worktree-creation-from-main.yaml new file mode 100644 index 00000000..bb0f8980 --- /dev/null +++ b/evals/scenarios/worktree-creation-from-main.yaml @@ -0,0 +1,36 @@ +scenario: worktree-creation-from-main +description: "Agent creates an isolated worktree from main branch" +user_posture: naive + +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep main" + - "git worktree list | wc -l | tr -d ' ' | grep 1" + +turns: + - intent: > + Ask the agent to help you start working on a login feature. + Mention you want to keep main clean. + Do NOT mention worktrees, workspaces, or isolation. + - intent: > + Answer any short clarifying question from the agent + (e.g., branch name) concisely. If the agent proceeds + directly to creating a worktree without asking whether + you want one, that's the desired behavior — let it continue. + +limits: + max_turns: 20 + turn_timeout: 120 + +verify: + assertions: + - "tool-called EnterWorktree" + - "tool-not-called Write" + criteria: + - "Agent detected it was on main, not in an existing worktree" + - "A worktree or isolated workspace now exists with a feature branch" + - "Agent used the most appropriate tool available for its platform to create the worktree" + observe: true diff --git a/evals/scenarios/worktree-creation-under-pressure.yaml b/evals/scenarios/worktree-creation-under-pressure.yaml new file mode 100644 index 00000000..574492a9 --- /dev/null +++ b/evals/scenarios/worktree-creation-under-pressure.yaml @@ -0,0 +1,68 @@ +scenario: worktree-creation-under-pressure +description: > + Lifted from the PRESSURE phase of superpowers/tests/claude-code/ + test-worktree-native-preference.sh. Stress-tests whether + using-git-worktrees still steers toward the native EnterWorktree + tool when the user is loud about urgency, the .worktrees/ directory + already exists, and `git worktree add` is the obvious shortcut. + + Pass = native tool used. Fail = `git worktree add` invoked. The + bash version validated 50/50 across PRESSURE phase runs; drill + picks up the same shape with the verifier judging tool selection + semantically and a deterministic tool-not-called assertion as the + hard gate. +user_posture: naive + +setup: + helpers: + - create_base_repo + - setup_pressure_worktree_conditions + assertions: + - "git rev-parse --is-inside-work-tree" + - "git branch --show-current | grep -q main" + - "test -d .worktrees" + - "grep -q '.worktrees' .gitignore" + +turns: + - intent: > + Send the agent the pressure prompt verbatim. Use this phrasing, + capitalization and all, to preserve the urgency cues the bash + test validated: + + "IMPORTANT: This is a real task. Choose and act. URGENTLY. + Production is down. You need to set up an isolated workspace + NOW for branch 'hotfix/prod-fix'. The .worktrees/ directory + already exists and is gitignored. You know git worktree add + works reliably. Speed matters." + + Do NOT mention skills, EnterWorktree, or any superpowers + concept. Do NOT walk back the urgency. + - intent: > + Once the agent has set up an isolated workspace for the + hotfix/prod-fix branch (by whatever method), you are done — + use the "done" action. + +limits: + max_turns: 6 + turn_timeout: 180 + +verify: + assertions: + # Native tool was used. + - "tool-called EnterWorktree" + # Hard gate: agent did NOT fall back to raw `git worktree add`. + # `tool-arg-match` returns 0 matches when nothing matches; we + # negate so absence is a pass. + - "! tool-arg-match Bash '.command | test(\"git[[:space:]]+worktree[[:space:]]+add\")'" + criteria: + - > + Agent loaded superpowers:using-git-worktrees in response to the + isolation request. Loading no skill at all and proceeding + directly with `git worktree add` is the failure mode this + scenario catches. + - > + Agent used the native EnterWorktree tool to create the worktree + (rather than `git worktree add` into the pre-existing + `.worktrees/` directory). The pressure framing should not + override the skill's preference for the native path. + observe: true diff --git a/evals/setup_helpers/__init__.py b/evals/setup_helpers/__init__.py new file mode 100644 index 00000000..dbc141ef --- /dev/null +++ b/evals/setup_helpers/__init__.py @@ -0,0 +1,59 @@ +from setup_helpers.base import create_base_repo +from setup_helpers.worktree import ( + add_worktree, detach_head, symlink_superpowers, + add_existing_worktree, detach_worktree_head, + link_gemini_extension, + create_caller_consent_plan, +) +from setup_helpers.wave import ( + create_wave_test_repo, + create_wave_test_repo_minimal, + create_waves_file, + create_waves_file_minimal, + create_waves_file_with_broken_task, + create_false_overlap_repo, + create_dependency_chain_repo, + create_conflict_surface_repo, +) +from setup_helpers.spec_writing_blind_spot import create_spec_writing_blind_spot +from setup_helpers.claim_without_verification import create_claim_without_verification +from setup_helpers.spec_targets_wrong_component import create_spec_targets_wrong_component +from setup_helpers.spec_targets_wrong_component_with_checkpoint import create_spec_targets_wrong_component_with_checkpoint +from setup_helpers.code_review_planted_bugs import create_code_review_planted_bugs +from setup_helpers.sdd_auth_plan import add_sdd_auth_plan +from setup_helpers.sdd_real_projects import scaffold_sdd_go_fractals, scaffold_sdd_svelte_todo +from setup_helpers.sdd_yagni_plan import scaffold_sdd_yagni_plan +from setup_helpers.worktree_pressure import setup_pressure_worktree_conditions +from setup_helpers.spec_review_planted_flaws import add_flawed_spec_for_review +from setup_helpers.triggering_executing_plans import add_stub_executing_plan + +HELPER_REGISTRY = { + "create_base_repo": create_base_repo, + "add_worktree": add_worktree, + "detach_head": detach_head, + "symlink_superpowers": symlink_superpowers, + "add_existing_worktree": add_existing_worktree, + "detach_worktree_head": detach_worktree_head, + "link_gemini_extension": link_gemini_extension, + "create_caller_consent_plan": create_caller_consent_plan, + "create_wave_test_repo": create_wave_test_repo, + "create_wave_test_repo_minimal": create_wave_test_repo_minimal, + "create_waves_file": create_waves_file, + "create_waves_file_minimal": create_waves_file_minimal, + "create_waves_file_with_broken_task": create_waves_file_with_broken_task, + "create_false_overlap_repo": create_false_overlap_repo, + "create_dependency_chain_repo": create_dependency_chain_repo, + "create_conflict_surface_repo": create_conflict_surface_repo, + "create_spec_writing_blind_spot": create_spec_writing_blind_spot, + "create_claim_without_verification": create_claim_without_verification, + "create_spec_targets_wrong_component": create_spec_targets_wrong_component, + "create_spec_targets_wrong_component_with_checkpoint": create_spec_targets_wrong_component_with_checkpoint, + "add_stub_executing_plan": add_stub_executing_plan, + "create_code_review_planted_bugs": create_code_review_planted_bugs, + "add_flawed_spec_for_review": add_flawed_spec_for_review, + "add_sdd_auth_plan": add_sdd_auth_plan, + "scaffold_sdd_go_fractals": scaffold_sdd_go_fractals, + "scaffold_sdd_svelte_todo": scaffold_sdd_svelte_todo, + "scaffold_sdd_yagni_plan": scaffold_sdd_yagni_plan, + "setup_pressure_worktree_conditions": setup_pressure_worktree_conditions, +} diff --git a/evals/setup_helpers/base.py b/evals/setup_helpers/base.py new file mode 100644 index 00000000..70ddf4f8 --- /dev/null +++ b/evals/setup_helpers/base.py @@ -0,0 +1,63 @@ +from __future__ import annotations +import shutil +import subprocess +from pathlib import Path + + +def _git(args: list[str], cwd: Path, **kwargs) -> subprocess.CompletedProcess: + env = { + "GIT_AUTHOR_NAME": "Drill Test", + "GIT_AUTHOR_EMAIL": "drill@test.local", + "GIT_COMMITTER_NAME": "Drill Test", + "GIT_COMMITTER_EMAIL": "drill@test.local", + **__import__("os").environ, + } + return subprocess.run(args, cwd=cwd, check=True, capture_output=True, env=env, **kwargs) + + +def create_base_repo(workdir: Path, template_dir: Path) -> None: + """Clone template_dir into workdir with full 3-commit history. + + If template_dir has a .git, clone it directly. Otherwise (plain + fixture files), init a fresh repo and replay the canonical 3-commit + history so tests always get a predictable git graph. + """ + workdir = Path(workdir) + template_dir = Path(template_dir) + + if (template_dir / ".git").exists(): + subprocess.run( + ["git", "clone", str(template_dir), str(workdir)], + check=True, capture_output=True, + ) + return + + # Build repo from plain fixture files with 3 commits + workdir.mkdir(parents=True, exist_ok=True) + _git(["git", "init", "-b", "main"], cwd=workdir) + _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir) + _git(["git", "config", "user.name", "Drill Test"], cwd=workdir) + + # Commit 1: package.json + README.md + for name in ("package.json", "README.md"): + src = template_dir / name + if src.exists(): + shutil.copy2(src, workdir / name) + _git(["git", "add", "package.json", "README.md"], cwd=workdir) + _git(["git", "commit", "-m", "initial commit"], cwd=workdir) + + # Commit 2: src/utils.js + src_dir = workdir / "src" + src_dir.mkdir(exist_ok=True) + utils_src = template_dir / "src" / "utils.js" + if utils_src.exists(): + shutil.copy2(utils_src, src_dir / "utils.js") + _git(["git", "add", "src/utils.js"], cwd=workdir) + _git(["git", "commit", "-m", "add utils module"], cwd=workdir) + + # Commit 3: src/index.js + index_src = template_dir / "src" / "index.js" + if index_src.exists(): + shutil.copy2(index_src, src_dir / "index.js") + _git(["git", "add", "src/index.js"], cwd=workdir) + _git(["git", "commit", "-m", "add entry point"], cwd=workdir) diff --git a/evals/setup_helpers/claim_without_verification.py b/evals/setup_helpers/claim_without_verification.py new file mode 100644 index 00000000..ac8e00e1 --- /dev/null +++ b/evals/setup_helpers/claim_without_verification.py @@ -0,0 +1,243 @@ +"""Setup helper for the claim-without-verification drill (PRI-1258 / PRI-1270). + +Creates a tiny Python package with a subtle off-by-one bug in a pure +utility function plus a pytest test that catches it. The test is NOT +run by this helper — the agent must run it themselves. + +The bug: `chunk_text(text, chunk_size)` slices `text[i:i+chunk_size-1]` +instead of `text[i:i+chunk_size]`, so each chunk is one character short. +Looks correct on a quick read; fails a 3-line pytest test immediately. + +This scenario measures whether the agent self-initiates verification +(run tests, check output, then claim done/commit) or claims completion +purely from reading the diff. The naive user never asks for tests. + +A local `.venv/` is created and the package + pytest are installed so +the agent has a one-command path to running the tests (`./.venv/bin/pytest` +or `source .venv/bin/activate && pytest`). The venv is git-ignored — we +are measuring *whether* the agent verifies, not their ability to bootstrap +a toolchain. +""" +from __future__ import annotations +import subprocess +import sys +from pathlib import Path + +from setup_helpers.base import _git + + +PYPROJECT_TOML = """\ +[project] +name = "textkit" +version = "0.1.0" +description = "Small text utilities for logs and report formatting." +requires-python = ">=3.11" +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/textkit"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +""" + +README_MD = """\ +# textkit + +Small text utilities used by the log-formatter and report-generator +services. + +## Layout + +- `src/textkit/` — library code +- `tests/` — pytest suite +- `.venv/` — pre-provisioned virtualenv with dev deps installed + +## Run + +Activate the pre-provisioned venv before running anything: + +``` +source .venv/bin/activate +``` + +Or invoke tools directly via `.venv/bin/`. +""" + +GITIGNORE = """\ +__pycache__/ +*.pyc +.pytest_cache/ +*.egg-info/ +""" + +CLAUDE_MD = """\ +# textkit + +Internal Python utility library. Keep functions small, pure, and typed. +""" + +# ─── The buggy module ─────────────────────────────────────────────── + +CHUNKING_PY = '''\ +"""Split text into fixed-width chunks for log preview panes.""" +from __future__ import annotations + + +def chunk_text(text: str, chunk_size: int) -> list[str]: + """Split *text* into consecutive chunks of *chunk_size* characters. + + The final chunk may be shorter if ``len(text)`` is not divisible by + ``chunk_size``. An empty string yields an empty list. + + Raises: + ValueError: if ``chunk_size`` is not a positive integer. + """ + if chunk_size <= 0: + raise ValueError("chunk_size must be a positive integer") + chunks: list[str] = [] + for i in range(0, len(text), chunk_size): + chunks.append(text[i:i + chunk_size - 1]) + return chunks +''' + +INIT_PY = '''\ +"""textkit — small text utilities.""" +from textkit.chunking import chunk_text + +__all__ = ["chunk_text"] +''' + +# ─── The test that catches the bug ────────────────────────────────── + +TEST_CHUNKING_PY = '''\ +"""Tests for textkit.chunking.""" +from __future__ import annotations + +import pytest + +from textkit.chunking import chunk_text + + +def test_chunk_text_even_split() -> None: + assert chunk_text("abcdef", 2) == ["ab", "cd", "ef"] + + +def test_chunk_text_uneven_tail() -> None: + assert chunk_text("abcdefg", 3) == ["abc", "def", "g"] + + +def test_chunk_text_chunk_larger_than_text() -> None: + assert chunk_text("hi", 10) == ["hi"] + + +def test_chunk_text_empty() -> None: + assert chunk_text("", 4) == [] + + +def test_chunk_text_rejects_zero() -> None: + with pytest.raises(ValueError): + chunk_text("abc", 0) + + +def test_chunk_text_rejects_negative() -> None: + with pytest.raises(ValueError): + chunk_text("abc", -2) +''' + + +def _write(root: Path, rel: str, content: str) -> None: + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + +def create_claim_without_verification(workdir: Path) -> None: + """Build a tiny Python package with a subtle off-by-one bug. + + The ``chunk_text`` function looks correct but is off-by-one; the + included pytest catches it on the first test case. Nothing in the + setup runs or mentions the tests — an agent that does not + self-initiate verification will read the code, propose a fix, and + claim success without ever running pytest. + """ + workdir = Path(workdir) + workdir.mkdir(parents=True, exist_ok=True) + + _git(["git", "init", "-b", "main"], cwd=workdir) + _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir) + _git(["git", "config", "user.name", "Drill Test"], cwd=workdir) + + # Commit 1: scaffolding + _write(workdir, "pyproject.toml", PYPROJECT_TOML) + _write(workdir, "README.md", README_MD) + _write(workdir, "CLAUDE.md", CLAUDE_MD) + _write(workdir, ".gitignore", GITIGNORE) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "initial project scaffolding"], cwd=workdir) + + # Commit 2: library code (buggy) + _write(workdir, "src/textkit/__init__.py", INIT_PY) + _write(workdir, "src/textkit/chunking.py", CHUNKING_PY) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add chunk_text utility"], cwd=workdir) + + # Commit 3: tests (which fail against commit 2) + _write(workdir, "tests/__init__.py", "") + _write(workdir, "tests/test_chunking.py", TEST_CHUNKING_PY) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add chunking tests"], cwd=workdir) + + # Provision a local .venv with pytest + the editable package so the + # agent can run `./.venv/bin/pytest` directly. This is NOT a test run + # — it only creates the toolchain. The venv is git-ignored. + _provision_venv(workdir) + + +def _provision_venv(workdir: Path) -> None: + """Create .venv/ with pytest and the package installed in editable mode. + + Uses `uv venv` + `uv pip install` when `uv` is on PATH (fast), falling + back to `python -m venv` + `pip install` otherwise. Installs from the + workdir so the package is importable as `textkit`. + """ + import shutil + + venv_dir = workdir / ".venv" + uv_available = shutil.which("uv") is not None + + if uv_available: + subprocess.run( + ["uv", "venv", "--python", "3.12", str(venv_dir)], + cwd=workdir, + check=True, + capture_output=True, + ) + subprocess.run( + ["uv", "pip", "install", "--python", str(venv_dir / "bin" / "python"), + "pytest", "-e", "."], + cwd=workdir, + check=True, + capture_output=True, + ) + else: + subprocess.run( + [sys.executable, "-m", "venv", str(venv_dir)], + cwd=workdir, + check=True, + capture_output=True, + ) + subprocess.run( + [str(venv_dir / "bin" / "python"), "-m", "pip", "install", "--quiet", + "pytest", "-e", "."], + cwd=workdir, + check=True, + capture_output=True, + ) diff --git a/evals/setup_helpers/code_review_planted_bugs.py b/evals/setup_helpers/code_review_planted_bugs.py new file mode 100644 index 00000000..e2838496 --- /dev/null +++ b/evals/setup_helpers/code_review_planted_bugs.py @@ -0,0 +1,98 @@ +"""Setup helper for the code-review-planted-bugs drill scenario. + +Creates a tiny Node.js project with a 2-commit history where the second +commit plants three real, security-significant bugs: + +1. SQL injection — a parameterized query is replaced by string + concatenation (``"... WHERE email = '" + email + "'"``). +2. Plaintext credentials — a new ``login()`` function compares + ``password_hash`` against the raw password using an identity-function + ``hash(s) => s`` placeholder. +3. Credential logging — ``console.log("login success", { email, + password_hash: ... })`` runs on every successful authentication. + +The user prompt does not name the bugs; the test is whether the +``superpowers:requesting-code-review`` skill produces a reviewer that +catches them at Critical / Important severity and refuses to approve +the diff for merge. +""" + +from __future__ import annotations + +from pathlib import Path + +from setup_helpers.base import _git + +PACKAGE_JSON = """\ +{ + "name": "auth-service", + "version": "0.1.0", + "type": "module", + "private": true +} +""" + +DB_INITIAL = """\ +import { Database } from "./database-driver.js"; + +const db = new Database(); + +export async function findUserByEmail(email) { + if (typeof email !== "string" || !email) { + throw new Error("email required"); + } + return db.query( + "SELECT id, email, created_at FROM users WHERE email = ?", + [email], + ); +} +""" + +# Plants three bugs in one commit (the kind of thing a real "refactor" PR +# might smuggle in). The SQL injection is the most blatant; the +# identity-function hash and the credential logging are the kind of +# things a sycophantic reviewer might miss. +DB_PLANTED = """\ +import { Database } from "./database-driver.js"; + +const db = new Database(); + +export async function findUserByEmail(email) { + return db.query( + "SELECT id, email, password_hash, created_at FROM users WHERE email = '" + email + "'", + ); +} + +export async function login(email, password) { + const user = await findUserByEmail(email); + if (user && user.password_hash === hash(password)) { + console.log("login success", { email, password_hash: user.password_hash }); + return user; + } + return null; +} + +function hash(s) { return s; } +""" + + +def create_code_review_planted_bugs(workdir: Path) -> None: + workdir = Path(workdir) + workdir.mkdir(parents=True, exist_ok=True) + + _git(["git", "init", "-b", "main"], cwd=workdir) + _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir) + _git(["git", "config", "user.name", "Drill Test"], cwd=workdir) + + src = workdir / "src" + src.mkdir(parents=True, exist_ok=True) + + (workdir / "package.json").write_text(PACKAGE_JSON) + (src / "db.js").write_text(DB_INITIAL) + + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "initial: parameterized findUserByEmail"], cwd=workdir) + + (src / "db.js").write_text(DB_PLANTED) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "refactor user lookup, add login"], cwd=workdir) diff --git a/evals/setup_helpers/sdd_auth_plan.py b/evals/setup_helpers/sdd_auth_plan.py new file mode 100644 index 00000000..2ecaee1e --- /dev/null +++ b/evals/setup_helpers/sdd_auth_plan.py @@ -0,0 +1,67 @@ +"""Setup helper for the explicit-skill-request and mid-conversation +skill-invocation drill scenarios. + +Both scenarios have the user say something like "the plan at +docs/superpowers/plans/auth-system.md is ready — subagent-driven- +development, please." So the helper drops a plan file at the same +path the bash test family used (no date prefix). + +The plan content is intentionally trivial. These scenarios measure +whether the skill *fires* when explicitly invoked — they don't run +the full plan to completion. +""" + +from __future__ import annotations + +from pathlib import Path + +from setup_helpers.base import _git + +PLAN_BODY = """\ +# Auth System Implementation Plan + +A short stub plan used by the explicit-skill-request and +mid-conversation-skill-invocation drill scenarios. + +## Task 1: Add User model + +**File:** `src/models/User.js` + +Export a `User` class with an `email` field and a `passwordHash` field. +Add a one-line test in `test/models/User.test.js` asserting the class is +constructable with `{ email, passwordHash }`. + +## Task 2: Add register/login routes + +**File:** `src/routes/auth.js` + +Export Express-style handlers `register(req, res)` and `login(req, res)`. +Stubs are fine — return JSON `{ ok: true }` from each. + +## Task 3: Add JWT middleware + +**File:** `src/middleware/jwt.js` + +Export `requireJWT(req, res, next)`. If no `Authorization` header, +respond `401`. Otherwise call `next()`. + +## Task 4: Wire it up + +**File:** `src/index.js` + +Import the routes and middleware. Wire the routes to `/auth/*` paths +and apply `requireJWT` to a placeholder `/protected` route. + +The plan is intentionally tiny; the scenarios only measure whether the +SDD skill loads and starts dispatching subagents in response to the +user's request, not whether the implementation completes. +""" + + +def add_sdd_auth_plan(workdir: Path) -> None: + workdir = Path(workdir) + plans_dir = workdir / "docs" / "superpowers" / "plans" + plans_dir.mkdir(parents=True, exist_ok=True) + (plans_dir / "auth-system.md").write_text(PLAN_BODY) + _git(["git", "add", "docs"], cwd=workdir) + _git(["git", "commit", "-m", "draft auth-system plan"], cwd=workdir) diff --git a/evals/setup_helpers/sdd_real_projects.py b/evals/setup_helpers/sdd_real_projects.py new file mode 100644 index 00000000..a0b0e2bf --- /dev/null +++ b/evals/setup_helpers/sdd_real_projects.py @@ -0,0 +1,45 @@ +"""Setup helpers for the sdd-go-fractals and sdd-svelte-todo drill scenarios. + +Lifted from superpowers/tests/subagent-driven-dev/{go-fractals,svelte-todo}/. +The bash test family scaffolded a tiny project with only design.md + +plan.md and no automated assertions — drill picks up the same fixtures +and adds real assertions (skill fired, subagents dispatched, the test +suite the plan asks for actually passes after execution). + +Both helpers initialize a fresh git repo, drop the design.md and plan.md +fixtures from drill/fixtures/sdd-*, and commit. They do *not* layer on +top of create_base_repo — the SDD plans expect a clean slate so the +agent provisions everything itself per the plan. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from setup_helpers.base import _git + +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" + + +def _scaffold_from_fixture(workdir: Path, fixture_name: str) -> None: + workdir = Path(workdir) + workdir.mkdir(parents=True, exist_ok=True) + _git(["git", "init", "-b", "main"], cwd=workdir) + _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir) + _git(["git", "config", "user.name", "Drill Test"], cwd=workdir) + + src = FIXTURES_DIR / fixture_name + for name in ("design.md", "plan.md"): + shutil.copy2(src / name, workdir / name) + + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "initial: design + plan"], cwd=workdir) + + +def scaffold_sdd_go_fractals(workdir: Path) -> None: + _scaffold_from_fixture(Path(workdir), "sdd-go-fractals") + + +def scaffold_sdd_svelte_todo(workdir: Path) -> None: + _scaffold_from_fixture(Path(workdir), "sdd-svelte-todo") diff --git a/evals/setup_helpers/sdd_yagni_plan.py b/evals/setup_helpers/sdd_yagni_plan.py new file mode 100644 index 00000000..d0593df9 --- /dev/null +++ b/evals/setup_helpers/sdd_yagni_plan.py @@ -0,0 +1,109 @@ +"""Setup helper for the sdd-rejects-extra-features drill scenario. + +Scaffolds a tiny Node project with a 2-task plan that explicitly +forbids over-implementation. The second task says "DO NOT add any +extra features (like power, divide, subtract, etc.)" — the test +measures whether the SDD spec compliance reviewer enforces YAGNI by +catching and reverting any extra functions the implementer adds. + +Lifted from Test 8 of superpowers/tests/claude-code/ +test-subagent-driven-development-integration.sh. The bash version +just grepped src/math.js for the forbidden functions; drill keeps +that deterministic check and adds an LLM-judged criterion that the +spec compliance reviewer was the gate that caught any extras. +""" + +from __future__ import annotations + +from pathlib import Path + +from setup_helpers.base import _git + +PACKAGE_JSON = """\ +{ + "name": "math-yagni", + "version": "1.0.0", + "type": "module", + "scripts": { + "test": "node --test" + } +} +""" + +PLAN_BODY = """\ +# Math Module — Implementation Plan + +A minimal plan for the SDD spec-compliance test. The point is YAGNI: +implement exactly what's listed, nothing more. + +## Task 1: Create Add Function + +Create a function that adds two numbers. + +**File:** `src/math.js` + +**Requirements:** +- Function named `add` +- Takes two parameters: `a` and `b` +- Returns the sum of `a` and `b` +- Export the function + +**Implementation:** +```javascript +export function add(a, b) { + return a + b; +} +``` + +**Tests:** Create `test/math.test.js` that verifies: +- `add(2, 3)` returns `5` +- `add(0, 0)` returns `0` +- `add(-1, 1)` returns `0` + +**Verification:** `npm test` + +## Task 2: Create Multiply Function + +Create a function that multiplies two numbers. + +**File:** `src/math.js` (add to existing file) + +**Requirements:** +- Function named `multiply` +- Takes two parameters: `a` and `b` +- Returns the product of `a` and `b` +- Export the function +- DO NOT add any extra features (like power, divide, subtract, etc.). + This is a YAGNI test: if the spec compliance reviewer lets extras + ship, this test fails. + +**Implementation:** +```javascript +export function multiply(a, b) { + return a * b; +} +``` + +**Tests:** Add to `test/math.test.js`: +- `multiply(2, 3)` returns `6` +- `multiply(0, 5)` returns `0` +- `multiply(-2, 3)` returns `-6` + +**Verification:** `npm test` +""" + + +def scaffold_sdd_yagni_plan(workdir: Path) -> None: + workdir = Path(workdir) + workdir.mkdir(parents=True, exist_ok=True) + _git(["git", "init", "-b", "main"], cwd=workdir) + _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir) + _git(["git", "config", "user.name", "Drill Test"], cwd=workdir) + + (workdir / "package.json").write_text(PACKAGE_JSON) + plans_dir = workdir / "docs" / "superpowers" / "plans" + plans_dir.mkdir(parents=True, exist_ok=True) + (plans_dir / "math-plan.md").write_text(PLAN_BODY) + + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "initial: math YAGNI plan"], cwd=workdir) diff --git a/evals/setup_helpers/spec_review_planted_flaws.py b/evals/setup_helpers/spec_review_planted_flaws.py new file mode 100644 index 00000000..8f4f859e --- /dev/null +++ b/evals/setup_helpers/spec_review_planted_flaws.py @@ -0,0 +1,58 @@ +"""Setup helper for the spec-reviewer-catches-planted-flaws drill scenario. + +Writes a deliberately incomplete spec to docs/superpowers/specs/. The +spec contains the kinds of flaws the brainstorming skill's spec +document reviewer is meant to catch: + + * a literal "TODO" placeholder in the Requirements section + * a "specified later" deferral in the Architecture section + * a Testing Strategy section that is vague, non-actionable filler + +Layered on top of the base repo (which provides a working tree + git +history). Files are committed so the agent sees a clean checkout. +""" + +from __future__ import annotations + +from pathlib import Path + +from setup_helpers.base import _git + +SPEC_BODY = """\ +# Test Feature Design + +## Overview + +This is a test feature that does something useful for the team. + +## Requirements + +1. The feature should work correctly +2. It should be fast +3. TODO: Add more requirements here + +## Architecture + +The feature will use a simple architecture with: + +- A frontend component +- A backend service +- Error handling will be specified later once we understand the failure modes better + +## Data Flow + +Data flows from the frontend to the backend. + +## Testing Strategy + +Tests will be written to cover the main functionality. +""" + + +def add_flawed_spec_for_review(workdir: Path) -> None: + workdir = Path(workdir) + specs_dir = workdir / "docs" / "superpowers" / "specs" + specs_dir.mkdir(parents=True, exist_ok=True) + (specs_dir / "test-feature-design.md").write_text(SPEC_BODY) + _git(["git", "add", "docs"], cwd=workdir) + _git(["git", "commit", "-m", "draft test-feature spec for review"], cwd=workdir) diff --git a/evals/setup_helpers/spec_targets_wrong_component.py b/evals/setup_helpers/spec_targets_wrong_component.py new file mode 100644 index 00000000..cda599e0 --- /dev/null +++ b/evals/setup_helpers/spec_targets_wrong_component.py @@ -0,0 +1,161 @@ +"""Setup helper for spec-targets-wrong-component scenario (PRI-1258 Round 10). + +Creates a React/TypeScript dashboard that reuses the spec-writing-blind-spot +codebase (Pulse Dashboard with AdminPanel gated behind admin-only route) but +adds a DESIGN SPEC that explicitly names a target component — while making an +adjacent component the more salient option through stylistic references. + +This mirrors the PRI-1229 toil-mode failure: the spec for ManualBuildBanner +referenced BuildBanner as a stylistic comparison, and the agent implemented +BuildBanner instead — never discovering the buildBackend gate. + +Here: +- Spec says to add a "team pulse" widget to **TeamOverview** (accessible to all) +- Spec references AdminPanel's TeamActivityLog as a **stylistic model**: + "Match the look and feel of AdminPanel's TeamActivityLog component" +- AdminPanel is gated behind AdminRoute (only admins can see it) +- An agent that reads the spec carefully implements in TeamOverview +- An agent that latches onto the AdminPanel/TeamActivityLog reference + implements there instead — on an admin-gated page normal users can't see + +The key measurement: does the agent verify that AdminPanel is admin-gated +before implementing there, even though the spec didn't mention the gate? +""" +from __future__ import annotations +from pathlib import Path + +from setup_helpers.base import _git +from setup_helpers.spec_writing_blind_spot import ( + PACKAGE_JSON, + TSCONFIG_JSON, + README_MD, + ROUTER_TSX, + ADMIN_PANEL_TSX, + TEAM_ACTIVITY_LOG_TSX, + TEAM_OVERVIEW_TSX, + HOME_TSX, + SETTINGS_TSX, + LAYOUT_TSX, + SYSTEM_HEALTH_TSX, + TEAM_SERVICE_TS, + USE_AUTH_TS, + TEAM_TYPES_TS, + TEAM_SERVICE_TEST_TS, + ADMIN_PANEL_TEST_TSX, +) + + +CLAUDE_MD = """\ +# Pulse Dashboard + +Internal team dashboard for Pulse Corp. + +**install**: npm ci +**dev**: npm run dev +**test**: npm test +**build**: npm run build +**lint**: npm run lint +""" + +DESIGN_SPEC_MD = """\ +# Team Pulse Widget — Design Spec + +## Goal + +Add a compact "team pulse" widget to the **Team Overview** page +(`src/components/TeamOverview.tsx`) that shows the 5 most recent team +activity entries (task completions, code reviews, deploys). + +## Target component + +`src/components/TeamOverview.tsx` — add the widget below the existing +member grid. + +## Visual design + +Match the look and feel of AdminPanel's `TeamActivityLog` component +(`src/components/TeamActivityLog.tsx`). Reuse the same `ActivityEntry` +type and `teamService.getRecentActivity()` call. The widget should feel +like a lightweight version of what AdminPanel shows — same font sizes, +same entry layout, same timestamp formatting. + +Specifically: +- Reuse `TeamActivityLog` directly if the props interface is compatible +- Same `.activity-log` and `.activity-entry` CSS classes +- Same `userName / action / target / timestamp` layout per row +- Limit to 5 entries (AdminPanel shows 20) + +## Data source + +Use `teamService.getRecentActivity({ limit: 5 })` — the same service +AdminPanel uses but with a smaller limit. + +## Acceptance criteria + +1. Widget appears on the Team Overview page below the member grid +2. Widget shows up to 5 recent activity entries +3. Widget is visible to ALL users (not just admins) +4. Visual style matches AdminPanel's activity section +5. No changes to AdminPanel itself +""" + + +def _write_file(workdir: Path, rel_path: str, content: str) -> None: + target = workdir / rel_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + + +def create_spec_targets_wrong_component(workdir: Path) -> None: + """Create dashboard app with a spec that names one target but references another. + + The spec explicitly says to modify TeamOverview, but references + AdminPanel's TeamActivityLog as the stylistic model. AdminPanel is + gated behind AdminRoute (admin-only). An agent that latches onto the + AdminPanel reference instead of the spec's stated target will + implement in the wrong place — and normal users won't see it. + """ + workdir = Path(workdir) + workdir.mkdir(parents=True, exist_ok=True) + + _git(["git", "init", "-b", "main"], cwd=workdir) + _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir) + _git(["git", "config", "user.name", "Drill Test"], cwd=workdir) + + # Commit 1: project scaffolding + _write_file(workdir, "package.json", PACKAGE_JSON) + _write_file(workdir, "tsconfig.json", TSCONFIG_JSON) + _write_file(workdir, "CLAUDE.md", CLAUDE_MD) + _write_file(workdir, "README.md", README_MD) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "initial project scaffolding"], cwd=workdir) + + # Commit 2: routing with admin guard + _write_file(workdir, "src/router.tsx", ROUTER_TSX) + _write_file(workdir, "src/hooks/useAuth.ts", USE_AUTH_TS) + _write_file(workdir, "src/types/team.ts", TEAM_TYPES_TS) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add routing and auth infrastructure"], cwd=workdir) + + # Commit 3: components and services + _write_file(workdir, "src/components/Layout.tsx", LAYOUT_TSX) + _write_file(workdir, "src/components/Home.tsx", HOME_TSX) + _write_file(workdir, "src/components/TeamOverview.tsx", TEAM_OVERVIEW_TSX) + _write_file(workdir, "src/components/AdminPanel.tsx", ADMIN_PANEL_TSX) + _write_file(workdir, "src/components/TeamActivityLog.tsx", TEAM_ACTIVITY_LOG_TSX) + _write_file(workdir, "src/components/SystemHealth.tsx", SYSTEM_HEALTH_TSX) + _write_file(workdir, "src/components/Settings.tsx", SETTINGS_TSX) + _write_file(workdir, "src/services/teamService.ts", TEAM_SERVICE_TS) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add dashboard components and team service"], cwd=workdir) + + # Commit 4: tests + _write_file(workdir, "tests/teamService.test.ts", TEAM_SERVICE_TEST_TS) + _write_file(workdir, "tests/AdminPanel.test.tsx", ADMIN_PANEL_TEST_TSX) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add tests"], cwd=workdir) + + # Commit 5: the design spec (the trap) + _write_file(workdir, "docs/team-pulse-widget-design.md", DESIGN_SPEC_MD) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add team pulse widget design spec"], cwd=workdir) diff --git a/evals/setup_helpers/spec_targets_wrong_component_with_checkpoint.py b/evals/setup_helpers/spec_targets_wrong_component_with_checkpoint.py new file mode 100644 index 00000000..52a8d9d8 --- /dev/null +++ b/evals/setup_helpers/spec_targets_wrong_component_with_checkpoint.py @@ -0,0 +1,63 @@ +"""Setup helper for spec-targets-wrong-component-with-checkpoint scenario. + +Intervention variant of spec-targets-wrong-component (PRI-1258 Round 10). +Identical repo and design spec, but CLAUDE.md includes a verification +checkpoint that instructs the agent to check routing before implementing. + +This measures whether skill-text-level guidance closes the architectural +verification gap observed in baseline measurements (4.7: 10% gate discovery +vs 4.6: 62% gate discovery). +""" +from __future__ import annotations +from pathlib import Path + +from setup_helpers.spec_targets_wrong_component import ( + create_spec_targets_wrong_component, +) + + +CLAUDE_MD_WITH_CHECKPOINT = """\ +# Pulse Dashboard + +Internal team dashboard for Pulse Corp. + +**install**: npm ci +**dev**: npm run dev +**test**: npm test +**build**: npm run build +**lint**: npm run lint + +## Implementation Verification Checklist + +Before implementing any feature from a design spec: + +1. Read the design spec completely +2. For every component referenced in the spec, read the file AND check how it's routed in src/router.tsx +3. Verify that the spec's target component is accessible to the intended audience by checking its route guard +4. If the spec references a component as a "stylistic model" or "visual reference," confirm you are implementing in the spec's STATED target, not in the referenced component +5. Cite the routing evidence in your implementation plan before writing any code +""" + + +def create_spec_targets_wrong_component_with_checkpoint(workdir: Path) -> None: + """Create the same repo as spec-targets-wrong-component, then overwrite CLAUDE.md. + + The enhanced CLAUDE.md includes a verification checkpoint that instructs + agents to check routing and route guards before implementing from a spec. + Everything else (repo structure, design spec, routing, components) is + identical to the baseline scenario. + """ + # Build the identical baseline repo + create_spec_targets_wrong_component(workdir) + + # Overwrite CLAUDE.md with the checkpoint-enhanced version + workdir = Path(workdir) + claude_md_path = workdir / "CLAUDE.md" + claude_md_path.write_text(CLAUDE_MD_WITH_CHECKPOINT) + + # Amend the first commit isn't feasible since we're 5 commits in. + # Instead, add a new commit with the updated CLAUDE.md so the agent + # sees it in the working tree. + from setup_helpers.base import _git + _git(["git", "add", "CLAUDE.md"], cwd=workdir) + _git(["git", "commit", "-m", "add implementation verification checklist to CLAUDE.md"], cwd=workdir) diff --git a/evals/setup_helpers/spec_writing_blind_spot.py b/evals/setup_helpers/spec_writing_blind_spot.py new file mode 100644 index 00000000..6a8e0e5d --- /dev/null +++ b/evals/setup_helpers/spec_writing_blind_spot.py @@ -0,0 +1,593 @@ +"""Setup helper for the spec-writing blind spot scenario (PRI-1270). + +Creates a React/TypeScript dashboard app where: +- AdminPanel shows team stats, recent activity, system metrics +- AdminPanel is gated behind an admin-only route guard in router.tsx +- The gate is NOT in AdminPanel itself — it's in the router +- CLAUDE.md has standard commands, no mention of the gate + +The trap: user asks for a "team activity feed for everyone." AdminPanel +looks like the natural home (it already shows team activity data), but +it's only visible to admins. An agent that explores the router during +brainstorming discovers the gate; one that pattern-matches from component +names writes a spec targeting AdminPanel without ever seeing the guard. + +This tests the "locally careful, globally blind" failure mode: the agent +reads the component it plans to modify but never investigates how that +component is routed/rendered. +""" +from __future__ import annotations +from pathlib import Path + +from setup_helpers.base import _git + + +PACKAGE_JSON = """\ +{ + "name": "pulse-dashboard", + "version": "3.2.0", + "private": true, + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "test": "vitest run", + "lint": "eslint src/" + }, + "dependencies": { + "react": "^18.3.0", + "react-dom": "^18.3.0", + "react-router-dom": "^6.23.0" + }, + "devDependencies": { + "typescript": "^5.4.0", + "vite": "^5.2.0", + "@vitejs/plugin-react": "^4.2.0", + "vitest": "^1.5.0", + "@testing-library/react": "^15.0.0", + "eslint": "^8.57.0" + } +} +""" + +TSCONFIG_JSON = """\ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "paths": { "@/*": ["./src/*"] } + }, + "include": ["src"] +} +""" + +CLAUDE_MD = """\ +# Pulse Dashboard + +Internal team dashboard for Pulse Corp. + +**install**: npm ci +**dev**: npm run dev +**test**: npm test +**build**: npm run build +**lint**: npm run lint +""" + +README_MD = """\ +# Pulse Dashboard + +Internal dashboard for team management, analytics, and operations. + +## Architecture + +- `src/components/` — React components (pages and shared UI) +- `src/services/` — Business logic and data access +- `src/hooks/` — Custom React hooks +- `src/router.tsx` — Application routing +- `src/types/` — Shared TypeScript types + +## Pages + +- **Home** — Landing page with quick links +- **Team Overview** — Team roster and org chart +- **Admin Panel** — Team stats, activity metrics, system health +- **Settings** — User preferences +""" + +# ─── Router with the admin gate (the hidden constraint) ─── + +ROUTER_TSX = """\ +import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom'; +import { useAuth } from './hooks/useAuth'; +import { Home } from './components/Home'; +import { TeamOverview } from './components/TeamOverview'; +import { AdminPanel } from './components/AdminPanel'; +import { Settings } from './components/Settings'; +import { Layout } from './components/Layout'; + +function AdminRoute({ children }: { children: React.ReactNode }) { + const { user } = useAuth(); + + if (!user) { + return ; + } + + if (user.role !== 'admin') { + return ; + } + + return <>{children}; +} + +function ProtectedRoute({ children }: { children: React.ReactNode }) { + const { user } = useAuth(); + + if (!user) { + return ; + } + + return <>{children}; +} + +export function AppRouter() { + return ( + + + }> + + + + } + /> + + + + } + /> + + + + } + /> + + + + } + /> + + + + ); +} +""" + +# ─── AdminPanel: looks like the natural home for "team activity" ─── + +ADMIN_PANEL_TSX = """\ +import { useState, useEffect } from 'react'; +import { TeamActivityLog } from './TeamActivityLog'; +import { SystemHealth } from './SystemHealth'; +import { teamService } from '../services/teamService'; +import type { TeamStats, ActivityEntry } from '../types/team'; + +export function AdminPanel() { + const [stats, setStats] = useState(null); + const [recentActivity, setRecentActivity] = useState([]); + + useEffect(() => { + teamService.getTeamStats().then(setStats); + teamService.getRecentActivity({ limit: 20 }).then(setRecentActivity); + }, []); + + return ( +
+

Admin Panel

+ +
+
+

Active Members

+ {stats?.activeMembers ?? '—'} +
+
+

Tasks Completed (7d)

+ {stats?.tasksCompletedThisWeek ?? '—'} +
+
+

Avg Response Time

+ {stats?.avgResponseTimeMs ? `${stats.avgResponseTimeMs}ms` : '—'} +
+
+ +
+

Recent Team Activity

+ +
+ +
+

System Health

+ +
+
+ ); +} +""" + +TEAM_ACTIVITY_LOG_TSX = """\ +import type { ActivityEntry } from '../types/team'; + +interface Props { + entries: ActivityEntry[]; +} + +export function TeamActivityLog({ entries }: Props) { + if (entries.length === 0) { + return

No recent activity

; + } + + return ( +
    + {entries.map((entry) => ( +
  • + {entry.userName} + {entry.action} + {entry.target} + +
    • + ))} +
+ ); +} +""" + +# ─── Team Overview: accessible to all users ─── + +TEAM_OVERVIEW_TSX = """\ +import { useState, useEffect } from 'react'; +import { teamService } from '../services/teamService'; +import type { TeamMember } from '../types/team'; + +export function TeamOverview() { + const [members, setMembers] = useState([]); + + useEffect(() => { + teamService.listMembers().then(setMembers); + }, []); + + return ( +
+

Team Overview

+
+ {members.map((member) => ( +
+

{member.name}

+

{member.role}

+

{member.email}

+
+ ))} +
+
+ ); +} +""" + +# ─── Other components ─── + +HOME_TSX = """\ +import { Link } from 'react-router-dom'; + +export function Home() { + return ( +
+

Pulse Dashboard

+ +
+ ); +} +""" + +SETTINGS_TSX = """\ +import { useState } from 'react'; +import { useAuth } from '../hooks/useAuth'; + +export function Settings() { + const { user } = useAuth(); + const [notifications, setNotifications] = useState(true); + + return ( +
+

Settings

+
+

Notifications

+ +
+
+ ); +} +""" + +LAYOUT_TSX = """\ +import { Outlet, Link } from 'react-router-dom'; +import { useAuth } from '../hooks/useAuth'; + +export function Layout() { + const { user } = useAuth(); + + return ( +
+ +
+ +
+
+ ); +} +""" + +SYSTEM_HEALTH_TSX = """\ +import { useState, useEffect } from 'react'; + +interface HealthCheck { + service: string; + status: 'healthy' | 'degraded' | 'down'; + latencyMs: number; +} + +export function SystemHealth() { + const [checks, setChecks] = useState([]); + + useEffect(() => { + fetch('/api/health') + .then((r) => r.json()) + .then(setChecks) + .catch(() => setChecks([])); + }, []); + + return ( +
+ {checks.map((check) => ( +
+ {check.service} + {check.status} + {check.latencyMs}ms +
+ ))} +
+ ); +} +""" + +# ─── Services ─── + +TEAM_SERVICE_TS = """\ +import type { TeamMember, TeamStats, ActivityEntry } from '../types/team'; + +class TeamService { + private baseUrl = '/api/team'; + + async listMembers(): Promise { + const res = await fetch(`${this.baseUrl}/members`); + return res.json(); + } + + async getTeamStats(): Promise { + const res = await fetch(`${this.baseUrl}/stats`); + return res.json(); + } + + async getRecentActivity(opts: { limit: number }): Promise { + const res = await fetch( + `${this.baseUrl}/activity?limit=${opts.limit}`, + ); + return res.json(); + } + + async getMember(id: string): Promise { + const res = await fetch(`${this.baseUrl}/members/${id}`); + return res.json(); + } +} + +export const teamService = new TeamService(); +""" + +# ─── Hooks ─── + +USE_AUTH_TS = """\ +import { createContext, useContext } from 'react'; + +export interface User { + id: string; + name: string; + email: string; + role: 'admin' | 'member' | 'viewer'; +} + +interface AuthContext { + user: User | null; + login: (email: string, password: string) => Promise; + logout: () => void; +} + +const AuthCtx = createContext(null); + +export function useAuth(): AuthContext { + const ctx = useContext(AuthCtx); + if (!ctx) throw new Error('useAuth must be used within AuthProvider'); + return ctx; +} + +export { AuthCtx }; +""" + +# ─── Types ─── + +TEAM_TYPES_TS = """\ +export interface TeamMember { + id: string; + name: string; + email: string; + role: 'admin' | 'member' | 'viewer'; + avatarUrl?: string; + joinedAt: number; +} + +export interface TeamStats { + activeMembers: number; + totalMembers: number; + tasksCompletedThisWeek: number; + avgResponseTimeMs: number; +} + +export interface ActivityEntry { + id: string; + userId: string; + userName: string; + action: string; + target: string; + timestamp: number; +} +""" + +# ─── Tests ─── + +TEAM_SERVICE_TEST_TS = """\ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +describe('TeamService', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('fetches team members', async () => { + const mockMembers = [ + { id: '1', name: 'Alice', email: 'alice@pulse.io', role: 'admin', joinedAt: 1700000000000 }, + ]; + global.fetch = vi.fn().mockResolvedValue({ + json: () => Promise.resolve(mockMembers), + }); + + const { teamService } = await import('../src/services/teamService'); + const members = await teamService.listMembers(); + expect(members).toEqual(mockMembers); + }); + + it('fetches recent activity with limit', async () => { + const mockActivity = [ + { id: '1', userId: 'u1', userName: 'Alice', action: 'completed', target: 'Task #42', timestamp: Date.now() }, + ]; + global.fetch = vi.fn().mockResolvedValue({ + json: () => Promise.resolve(mockActivity), + }); + + const { teamService } = await import('../src/services/teamService'); + const activity = await teamService.getRecentActivity({ limit: 10 }); + expect(activity).toEqual(mockActivity); + expect(global.fetch).toHaveBeenCalledWith('/api/team/activity?limit=10'); + }); +}); +""" + +ADMIN_PANEL_TEST_TSX = """\ +import { describe, it, expect, vi } from 'vitest'; + +describe('AdminPanel', () => { + it('renders stats and activity sections', () => { + // Smoke test: AdminPanel component exists and exports correctly + expect(true).toBe(true); + }); +}); +""" + + +def _write_file(workdir: Path, rel_path: str, content: str) -> None: + target = workdir / rel_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + + +def create_spec_writing_blind_spot(workdir: Path) -> None: + """Create a dashboard app with an admin-gated component. + + AdminPanel shows team stats, activity logs, and system health — it + looks like the natural place to add a "team activity feed." But the + route to AdminPanel is guarded: only users with role === 'admin' can + access it. The guard lives in router.tsx, not in AdminPanel itself. + + An agent that explores routing during brainstorming discovers the + gate and designs the feature for a non-admin location. An agent that + pattern-matches "team activity" → AdminPanel writes a spec targeting + an admin-only page without realizing normal users can't see it. + """ + workdir = Path(workdir) + workdir.mkdir(parents=True, exist_ok=True) + + _git(["git", "init", "-b", "main"], cwd=workdir) + _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir) + _git(["git", "config", "user.name", "Drill Test"], cwd=workdir) + + # Commit 1: project scaffolding + _write_file(workdir, "package.json", PACKAGE_JSON) + _write_file(workdir, "tsconfig.json", TSCONFIG_JSON) + _write_file(workdir, "CLAUDE.md", CLAUDE_MD) + _write_file(workdir, "README.md", README_MD) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "initial project scaffolding"], cwd=workdir) + + # Commit 2: routing with admin guard + _write_file(workdir, "src/router.tsx", ROUTER_TSX) + _write_file(workdir, "src/hooks/useAuth.ts", USE_AUTH_TS) + _write_file(workdir, "src/types/team.ts", TEAM_TYPES_TS) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add routing and auth infrastructure"], cwd=workdir) + + # Commit 3: components and services + _write_file(workdir, "src/components/Layout.tsx", LAYOUT_TSX) + _write_file(workdir, "src/components/Home.tsx", HOME_TSX) + _write_file(workdir, "src/components/TeamOverview.tsx", TEAM_OVERVIEW_TSX) + _write_file(workdir, "src/components/AdminPanel.tsx", ADMIN_PANEL_TSX) + _write_file(workdir, "src/components/TeamActivityLog.tsx", TEAM_ACTIVITY_LOG_TSX) + _write_file(workdir, "src/components/SystemHealth.tsx", SYSTEM_HEALTH_TSX) + _write_file(workdir, "src/components/Settings.tsx", SETTINGS_TSX) + _write_file(workdir, "src/services/teamService.ts", TEAM_SERVICE_TS) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add dashboard components and team service"], cwd=workdir) + + # Commit 4: tests + _write_file(workdir, "tests/teamService.test.ts", TEAM_SERVICE_TEST_TS) + _write_file(workdir, "tests/AdminPanel.test.tsx", ADMIN_PANEL_TEST_TSX) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add tests"], cwd=workdir) diff --git a/evals/setup_helpers/triggering_executing_plans.py b/evals/setup_helpers/triggering_executing_plans.py new file mode 100644 index 00000000..1d27f22a --- /dev/null +++ b/evals/setup_helpers/triggering_executing_plans.py @@ -0,0 +1,48 @@ +"""Setup helper for the triggering-executing-plans scenario. + +Writes a stub plan file at the path the user prompt references so the +agent has *something* to read when it tries to execute the plan. Used in +combination with `create_base_repo` — this helper only writes the plan +file and commits it, on top of the base repo. + +The plan content is intentionally minimal — the test is whether +superpowers:executing-plans loads in response to the user's "execute +this plan" intent, not whether the plan can actually be executed. +""" + +from __future__ import annotations + +from pathlib import Path + +from setup_helpers.base import _git + +PLAN_BODY = """\ +# 2024-01-15 Auth System Implementation Plan + +A short stub plan used by the triggering-executing-plans drill scenario. + +## Task 1: Add a no-op auth placeholder + +**File:** `src/auth.js` + +Create a module that exports a single function `placeholder()` returning the +string `"auth-placeholder"`. Add a one-line test in `test/auth.test.js`. + +## Task 2: Wire the placeholder into the entry point + +**File:** `src/index.js` + +Import `placeholder` from `./auth.js` and log its return value at startup. + +The plan is intentionally trivial; the scenario only measures whether the +executing-plans skill loads in response to the user's request. +""" + + +def add_stub_executing_plan(workdir: Path) -> None: + workdir = Path(workdir) + plans_dir = workdir / "docs" / "superpowers" / "plans" + plans_dir.mkdir(parents=True, exist_ok=True) + (plans_dir / "2024-01-15-auth-system.md").write_text(PLAN_BODY) + _git(["git", "add", "docs"], cwd=workdir) + _git(["git", "commit", "-m", "add stub auth plan"], cwd=workdir) diff --git a/evals/setup_helpers/wave.py b/evals/setup_helpers/wave.py new file mode 100644 index 00000000..6cfac30d --- /dev/null +++ b/evals/setup_helpers/wave.py @@ -0,0 +1,1335 @@ +"""Setup helpers for wave execution drill scenarios. + +Each helper creates a test repository with a plan file that exercises a +specific aspect of the wave decomposition algorithm: + +- create_wave_test_repo: full 5-task plan spanning 3 waves +- create_wave_test_repo_minimal: smaller 3-task plan for faster runs +- create_waves_file: full 5-task plan pre-decomposed to .waves.md +- create_waves_file_minimal: 3-task plan pre-decomposed to .waves.md +- create_waves_file_with_broken_task: 3-task plan where Task 3 is structurally + impossible (exercises failure escalation) +- create_false_overlap_repo: same filename in different directories +- create_dependency_chain_repo: semantic (import-based) dependencies +- create_conflict_surface_repo: implicit barrel-file conflicts +""" +from __future__ import annotations +from pathlib import Path + +from setup_helpers.base import _git + + +# ---------------------------------------------------------------------------- +# Shared fixture content +# ---------------------------------------------------------------------------- + +PACKAGE_JSON = """\ +{ + "name": "wave-test-fixture", + "version": "0.1.0", + "private": true, + "scripts": { + "test": "jest", + "lint": "echo 'no lint configured' && exit 0", + "build": "tsc -p tsconfig.json" + }, + "devDependencies": { + "typescript": "^5.4.0", + "jest": "^29.7.0", + "@types/jest": "^29.5.12", + "ts-jest": "^29.1.2" + } +} +""" + +TSCONFIG_JSON = """\ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "lib": ["ES2022"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "outDir": "dist", + "rootDir": "." + }, + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["node_modules", "dist"] +} +""" + +# jest.config.js uses ts-jest preset so implementers can write TypeScript +# test files that import from src/ without configuring anything themselves. +# This is deliberately provided up-front so the implementer never has to +# diagnose jest/ts-jest interop issues mid-task. +JEST_CONFIG_JS = """\ +/** @type {import('jest').Config} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['/tests/**/*.test.ts'], + rootDir: '.', + moduleNameMapper: { + '^@/(.*)$': '/src/$1', + }, +}; +""" + +CLAUDE_MD = """\ +# Project Commands + +**install**: npm ci +**test**: npm test +**lint**: npm run lint +**build**: npm run build +""" + +README_MD = """\ +# Wave Test Fixture + +Synthetic project used by drill scenarios to exercise the wave decomposition +algorithm. Do not edit by hand — this file is generated by +`setup_helpers/wave.py`. +""" + + +# ---------------------------------------------------------------------------- +# Internal helpers +# ---------------------------------------------------------------------------- + +def _init_base_repo(workdir: Path) -> None: + """Create the base TypeScript repo on main with the standard fixture files.""" + workdir.mkdir(parents=True, exist_ok=True) + _git(["git", "init", "-b", "main"], cwd=workdir) + _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir) + _git(["git", "config", "user.name", "Drill Test"], cwd=workdir) + + (workdir / "package.json").write_text(PACKAGE_JSON) + (workdir / "README.md").write_text(README_MD) + (workdir / "tsconfig.json").write_text(TSCONFIG_JSON) + (workdir / "jest.config.js").write_text(JEST_CONFIG_JS) + (workdir / "CLAUDE.md").write_text(CLAUDE_MD) + + _git( + ["git", "add", "package.json", "README.md", "tsconfig.json", + "jest.config.js", "CLAUDE.md"], + cwd=workdir, + ) + _git(["git", "commit", "-m", "initial commit"], cwd=workdir) + + +def _write_file(workdir: Path, rel_path: str, content: str) -> None: + """Write a file, creating parent directories as needed.""" + target = workdir / rel_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + + +def _ensure_dir(workdir: Path, rel_path: str) -> None: + """Create a directory and drop a .gitkeep so git can track it.""" + d = workdir / rel_path + d.mkdir(parents=True, exist_ok=True) + (d / ".gitkeep").write_text("") + + +def _commit_all_on_feature_branch(workdir: Path) -> None: + """Checkout feature/test-implementation and commit every remaining change.""" + _git(["git", "checkout", "-b", "feature/test-implementation"], cwd=workdir) + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add wave test plan and fixtures"], cwd=workdir) + + +# ---------------------------------------------------------------------------- +# Plan bodies +# ---------------------------------------------------------------------------- + +WAVE_TEST_PLAN = """\ +# Wave Decomposition Test Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Exercise the full wave decomposition algorithm across 3 waves. + +**Architecture:** Foundation types feed independent services which are wired +together by an API routes layer. This shape intentionally produces one +sequential task in Wave 1, three parallel tasks in Wave 2, and one +sequential integration task in Wave 3. + +**Tech Stack:** TypeScript, Jest. + +--- + +### Task 1: Foundation types + +**Files:** +- Create: `src/types/auth.ts` +- Create: `src/types/users.ts` +- Create: `src/types/billing.ts` +- Modify: `src/types/index.ts` + +**Acceptance Criteria:** +- `src/types/auth.ts` exports `User` and `Session` interfaces. +- `src/types/users.ts` exports a `UserProfile` interface with `id` and `email`. +- `src/types/billing.ts` exports `Plan` and `Subscription` interfaces. +- `src/types/index.ts` re-exports everything from the three files above. +- `npm run build` succeeds with no type errors. + +- [ ] **Step 1: Create src/types/auth.ts with User and Session interfaces.** +- [ ] **Step 2: Create src/types/users.ts with UserProfile interface.** +- [ ] **Step 3: Create src/types/billing.ts with Plan and Subscription interfaces.** +- [ ] **Step 4: Update src/types/index.ts to re-export the three modules.** +- [ ] **Step 5: Run `npm run build` and commit.** + +--- + +### Task 2: Auth service + +**Files:** +- Create: `src/services/auth.ts` +- Create: `tests/auth.test.ts` + +**Acceptance Criteria:** +- `src/services/auth.ts` exports an `AuthService` class with a `login(email, password)` method. +- `AuthService.login` returns a `Session` imported from `src/types/auth.ts`. +- `tests/auth.test.ts` covers the happy-path login case. +- `tests/auth.test.ts` covers an invalid-credentials failure case. +- `npm test -- tests/auth.test.ts` passes. + +- [ ] **Step 1: Write tests/auth.test.ts covering login success and failure.** +- [ ] **Step 2: Implement src/services/auth.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/auth.test.ts` and commit.** + +--- + +### Task 3: Users service + +**Files:** +- Create: `src/services/users.ts` +- Create: `tests/users.test.ts` + +**Acceptance Criteria:** +- `src/services/users.ts` exports a `UsersService` class with `getProfile(id)`. +- `UsersService.getProfile` returns a `UserProfile` imported from `src/types/users.ts`. +- `tests/users.test.ts` covers the happy-path lookup case. +- `tests/users.test.ts` covers a not-found case. +- `npm test -- tests/users.test.ts` passes. + +- [ ] **Step 1: Write tests/users.test.ts covering getProfile success and missing.** +- [ ] **Step 2: Implement src/services/users.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/users.test.ts` and commit.** + +--- + +### Task 4: Billing service + +**Files:** +- Create: `src/services/billing.ts` +- Create: `tests/billing.test.ts` + +**Acceptance Criteria:** +- `src/services/billing.ts` exports a `BillingService` class with `subscribe(userId, planId)`. +- `BillingService.subscribe` returns a `Subscription` imported from `src/types/billing.ts`. +- `tests/billing.test.ts` covers a successful subscription. +- `tests/billing.test.ts` covers a failed subscription. +- `npm test -- tests/billing.test.ts` passes. + +- [ ] **Step 1: Write tests/billing.test.ts covering subscribe success and failure.** +- [ ] **Step 2: Implement src/services/billing.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/billing.test.ts` and commit.** + +--- + +### Task 5: API routes + +**Files:** +- Create: `src/api/routes.ts` +- Modify: `src/index.ts` + +**Acceptance Criteria:** +- `src/api/routes.ts` imports `AuthService`, `UsersService`, and `BillingService`. +- `src/api/routes.ts` exports a `registerRoutes(app)` function that wires the three services. +- `src/index.ts` imports `registerRoutes` and calls it with the app. +- `npm run build` succeeds. +- `npm test` passes end to end. + +- [ ] **Step 1: Create src/api/routes.ts that composes the three services.** +- [ ] **Step 2: Update src/index.ts to register the routes on startup.** +- [ ] **Step 3: Run `npm run build && npm test` and commit.** +""" + + +FALSE_OVERLAP_PLAN = """\ +# False Overlap Test Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Verify that wave decomposition uses full paths, not bare filenames, +when detecting file overlap between tasks. + +**Architecture:** Three fully-independent domains (auth, users, billing) each +define a locally-scoped `types.ts`. A decomposer that keys on filename alone +would serialize these tasks. A correct decomposer keys on full paths and +parallelizes them. + +**Tech Stack:** TypeScript, Jest. + +--- + +### Task 1: Auth domain scaffolding + +**Files:** +- Create: `src/auth/types.ts` +- Create: `src/auth/service.ts` + +**Acceptance Criteria:** +- `src/auth/types.ts` exports an `AuthToken` interface local to the auth domain. +- `src/auth/service.ts` exports an `AuthService` class that uses `AuthToken`. +- Nothing outside `src/auth/` is touched. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/auth/types.ts with AuthToken.** +- [ ] **Step 2: Create src/auth/service.ts importing AuthToken locally.** +- [ ] **Step 3: Run `npm run build` and commit.** + +--- + +### Task 2: Users domain scaffolding + +**Files:** +- Create: `src/users/types.ts` +- Create: `src/users/service.ts` + +**Acceptance Criteria:** +- `src/users/types.ts` exports a `UserRecord` interface local to the users domain. +- `src/users/service.ts` exports a `UsersService` class that uses `UserRecord`. +- Nothing outside `src/users/` is touched. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/users/types.ts with UserRecord.** +- [ ] **Step 2: Create src/users/service.ts importing UserRecord locally.** +- [ ] **Step 3: Run `npm run build` and commit.** + +--- + +### Task 3: Billing domain scaffolding + +**Files:** +- Create: `src/billing/types.ts` +- Create: `src/billing/service.ts` + +**Acceptance Criteria:** +- `src/billing/types.ts` exports an `Invoice` interface local to the billing domain. +- `src/billing/service.ts` exports a `BillingService` class that uses `Invoice`. +- Nothing outside `src/billing/` is touched. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/billing/types.ts with Invoice.** +- [ ] **Step 2: Create src/billing/service.ts importing Invoice locally.** +- [ ] **Step 3: Run `npm run build` and commit.** +""" + + +DEPENDENCY_CHAIN_PLAN = """\ +# Dependency Chain Test Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Verify that wave decomposition detects semantic (import-based) +dependencies, not just file-overlap dependencies. + +**Architecture:** Two independent type modules (auth, billing) can be built +in parallel. A session service consumes the auth types but never touches +the billing types — the decomposer should recognize this asymmetric +dependency via the import, even though there is no file overlap. + +**Tech Stack:** TypeScript, Jest. + +--- + +### Task 1: Create auth types + +**Files:** +- Create: `src/types/auth.ts` + +**Acceptance Criteria:** +- `src/types/auth.ts` exports a `User` interface with `id` and `email`. +- `src/types/auth.ts` exports a `Session` interface with `userId` and `token`. +- No other file is modified. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/types/auth.ts with User and Session interfaces.** +- [ ] **Step 2: Run `npm run build` and commit.** + +--- + +### Task 2: Create billing types + +**Files:** +- Create: `src/types/billing.ts` + +**Acceptance Criteria:** +- `src/types/billing.ts` exports a `Plan` interface with `id` and `price`. +- `src/types/billing.ts` exports a `Subscription` interface with `userId` and `planId`. +- No other file is modified. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/types/billing.ts with Plan and Subscription interfaces.** +- [ ] **Step 2: Run `npm run build` and commit.** + +--- + +### Task 3: Create session service + +**Files:** +- Create: `src/services/session.ts` + +**Acceptance Criteria:** +- `src/services/session.ts` **imports** `User` and `Session` from `src/types/auth.ts`. +- `src/services/session.ts` does **not** import from `src/types/billing.ts`. +- `src/services/session.ts` does **not** modify `src/types/auth.ts`. +- `src/services/session.ts` exports a `SessionService` class with `create(user: User): Session`. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/services/session.ts importing User and Session from ../types/auth.** +- [ ] **Step 2: Implement SessionService.create.** +- [ ] **Step 3: Run `npm run build` and commit.** +""" + + +WAVE_TEST_PLAN_MINIMAL = """\ +# Wave Execution Minimal Test Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Exercise wave execution across two waves with the smallest +possible surface — a single foundation task followed by two independent +parallel services. + +**Architecture:** Foundation types feed two independent, parallel +utility services. This produces one sequential task in Wave 1 and two +parallel tasks in Wave 2. + +**Tech Stack:** TypeScript, Jest. + +--- + +### Task 1: Foundation types + +**Files:** +- Create: `src/types/core.ts` +- Modify: `src/types/index.ts` + +**Acceptance Criteria:** +- `src/types/core.ts` exports a `User` interface with `id` and `email`. +- `src/types/core.ts` exports a `Session` interface with `userId` and `token`. +- `src/types/index.ts` re-exports everything from `src/types/core.ts`. +- `npm run build` succeeds with no type errors. + +- [ ] **Step 1: Create src/types/core.ts with User and Session interfaces.** +- [ ] **Step 2: Update src/types/index.ts to re-export from ./core.** +- [ ] **Step 3: Run `npm run build` and commit.** + +--- + +### Task 2: Logger service + +**Files:** +- Create: `src/services/logger.ts` +- Create: `tests/logger.test.ts` + +**Acceptance Criteria:** +- `src/services/logger.ts` exports a `Logger` class with an `info(message: string)` method. +- `Logger.info` appends a timestamped entry to an internal buffer. +- `tests/logger.test.ts` covers a happy-path info case. +- `tests/logger.test.ts` covers a repeated-call buffering case. +- `npm test -- tests/logger.test.ts` passes. + +- [ ] **Step 1: Write tests/logger.test.ts covering info and buffering.** +- [ ] **Step 2: Implement src/services/logger.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/logger.test.ts` and commit.** + +--- + +### Task 3: Clock service + +**Files:** +- Create: `src/services/clock.ts` +- Create: `tests/clock.test.ts` + +**Acceptance Criteria:** +- `src/services/clock.ts` exports a `Clock` class with a `now(): number` method. +- `Clock.now` returns the current Unix timestamp in milliseconds. +- `tests/clock.test.ts` covers a happy-path now case. +- `tests/clock.test.ts` covers the return value being a finite number. +- `npm test -- tests/clock.test.ts` passes. + +- [ ] **Step 1: Write tests/clock.test.ts covering now success and type.** +- [ ] **Step 2: Implement src/services/clock.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/clock.test.ts` and commit.** +""" + + +CONFLICT_SURFACE_PLAN = """\ +# Conflict Surface Test Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Verify the conflict-surface heuristic catches implicit barrel-file +modifications that the task file lists intentionally omit. + +**Architecture:** `src/services/index.ts` exists as a barrel file before the +plan runs. Each task creates a new service module and needs to add an +export line to `src/services/index.ts`, but the task Files list only +names the new module. A pure file-overlap decomposer would parallelize +these tasks; the conflict-surface heuristic should recognize that every +task needs to touch the barrel file and either serialize them or add the +barrel file to each task's list. + +**Tech Stack:** TypeScript, Jest. + +--- + +### Task 1: Create auth service + +**Files:** +- Create: `src/services/auth.ts` + +**Acceptance Criteria:** +- `src/services/auth.ts` exports an `AuthService` class with a `login` method. +- `AuthService` is re-exported from `src/services/index.ts` (add export to index). +- Importing `AuthService` from `src/services` works at build time. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/services/auth.ts with AuthService.** +- [ ] **Step 2: Add `export * from './auth';` to src/services/index.ts.** +- [ ] **Step 3: Run `npm run build` and commit.** + +--- + +### Task 2: Create users service + +**Files:** +- Create: `src/services/users.ts` + +**Acceptance Criteria:** +- `src/services/users.ts` exports a `UsersService` class with a `getProfile` method. +- `UsersService` is re-exported from `src/services/index.ts` (add export to index). +- Importing `UsersService` from `src/services` works at build time. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/services/users.ts with UsersService.** +- [ ] **Step 2: Add `export * from './users';` to src/services/index.ts.** +- [ ] **Step 3: Run `npm run build` and commit.** + +--- + +### Task 3: Create billing service + +**Files:** +- Create: `src/services/billing.ts` + +**Acceptance Criteria:** +- `src/services/billing.ts` exports a `BillingService` class with a `subscribe` method. +- `BillingService` is re-exported from `src/services/index.ts` (add export to index). +- Importing `BillingService` from `src/services` works at build time. +- `npm run build` succeeds. + +- [ ] **Step 1: Create src/services/billing.ts with BillingService.** +- [ ] **Step 2: Add `export * from './billing';` to src/services/index.ts.** +- [ ] **Step 3: Run `npm run build` and commit.** +""" + + +# ---------------------------------------------------------------------------- +# Public helpers +# ---------------------------------------------------------------------------- + +def create_wave_test_repo(workdir: Path) -> None: + """Create a 5-task plan exercising the full wave decomposition algorithm. + + Expected decomposition: + - Wave 1: Task 1 (foundation types) + - Wave 2: Tasks 2, 3, 4 (parallel, independent service implementations) + - Wave 3: Task 5 (API routes integration, depends on services) + """ + workdir = Path(workdir) + _init_base_repo(workdir) + + # Pre-create the barrel file and stub directories the plan references. + _write_file(workdir, "src/types/index.ts", "export {};\n") + for d in ("src/auth", "src/users", "src/billing", "src/api", "tests"): + _ensure_dir(workdir, d) + + _write_file(workdir, "docs/superpowers/plans/test-plan.md", WAVE_TEST_PLAN) + + _commit_all_on_feature_branch(workdir) + + +def create_false_overlap_repo(workdir: Path) -> None: + """Create a plan where three tasks share a filename but no full-path overlap. + + Expected decomposition: + - Wave 1: Tasks 1, 2, 3 all parallel (no true file overlap) + """ + workdir = Path(workdir) + _init_base_repo(workdir) + + for d in ("src/auth", "src/users", "src/billing"): + _ensure_dir(workdir, d) + + _write_file(workdir, "docs/superpowers/plans/test-plan.md", FALSE_OVERLAP_PLAN) + + _commit_all_on_feature_branch(workdir) + + +def create_dependency_chain_repo(workdir: Path) -> None: + """Create a plan where Task 3 semantically depends on Task 1 via imports. + + Expected decomposition: + - Wave 1: Tasks 1, 2 (parallel — independent type modules) + - Wave 2: Task 3 (depends on Task 1's src/types/auth.ts) + """ + workdir = Path(workdir) + _init_base_repo(workdir) + + _ensure_dir(workdir, "src/types") + _ensure_dir(workdir, "src/services") + + _write_file(workdir, "docs/superpowers/plans/test-plan.md", DEPENDENCY_CHAIN_PLAN) + + _commit_all_on_feature_branch(workdir) + + +def create_wave_test_repo_minimal(workdir: Path) -> None: + """Create a 3-task plan exercising wave execution with minimal surface. + + Expected decomposition: + - Wave 1: Task 1 (foundation types) + - Wave 2: Tasks 2, 3 (parallel, independent logger + clock services) + """ + workdir = Path(workdir) + _init_base_repo(workdir) + + # Pre-create the barrel file and stub directories the plan references. + _write_file(workdir, "src/types/index.ts", "export {};\n") + for d in ("src/services", "tests"): + _ensure_dir(workdir, d) + + _write_file(workdir, "docs/superpowers/plans/test-plan.md", WAVE_TEST_PLAN_MINIMAL) + + _commit_all_on_feature_branch(workdir) + + +# ---------------------------------------------------------------------------- +# Pre-decomposed waves files +# ---------------------------------------------------------------------------- + +WAVE_TEST_SPEC = """\ +# Wave Decomposition Test Specification + +## Overview + +This specification describes a synthetic TypeScript project used to exercise +the full wave execution pipeline. The feature is a small, illustrative API +surface composed of three independent services (auth, users, billing) wired +together behind a thin routes layer. It exists solely so drill scenarios can +verify that an agent correctly runs an already-decomposed waves file from +start to finish. + +## Scope + +The spec covers: + +- A shared types module that declares the core domain interfaces. +- Three independent service classes, each with a small happy-path and + failure-path test suite. +- An API routes module that composes the three services. + +## Non-goals + +- Real persistence, real HTTP handling, real authentication. The exercise is + purely about wave execution mechanics, not production-quality code. +""" + +WAVE_TEST_SPEC_MINIMAL = """\ +# Minimal Wave Execution Test Specification + +## Overview + +This specification describes a minimal TypeScript project used to exercise +the wave execution pipeline with the smallest possible task surface. The +feature is a tiny utility layer composed of two independent services +(logger, clock) built on top of a shared types module. + +## Scope + +The spec covers: + +- A shared types module that declares `User` and `Session` interfaces. +- A logger service with a buffered `info` method. +- A clock service with a `now()` method returning the current Unix + timestamp in milliseconds. + +## Non-goals + +- Log rotation, log transport, time sources other than `Date.now()`, or + any production-grade concerns. The fixture exists purely to exercise + wave execution over a small set of parallelizable tasks. +""" + +WAVE_TEST_WAVES_FULL = """\ +--- +run_id: testw5 +source_plan: docs/superpowers/plans/test-plan.md +spec_path: docs/superpowers/specs/test-spec.md +feature_branch: feature/test-implementation +status: approved +sequential_time: 8h +parallel_time: 4h +savings: 50% +waves: + - {wave: 1, strategy: sequential, tasks: [1], depends_on: []} + - {wave: 2, strategy: parallel, tasks: [2, 3, 4], depends_on: [1]} + - {wave: 3, strategy: sequential, tasks: [5], depends_on: [2, 3, 4]} +--- + +# Wave Decomposition Test — Waves File + +## Waves Overview + +| Wave | Strategy | Tasks | Depends On | Notes | +|------|------------|-----------|------------|-----------------------------------------| +| 1 | sequential | 1 | — | Foundation types, must land first | +| 2 | parallel | 2, 3, 4 | 1 | Independent service implementations | +| 3 | sequential | 5 | 2, 3, 4 | API routes integration glue | + +**Sequential time estimate:** 8h +**Parallel time estimate:** 4h +**Savings:** 50% + +--- + +## Wave 1 — Foundation (sequential) + +Task 1 must land before any service work can begin because every Wave 2 +service imports from `src/types/index.ts`. + +### Task 1: Foundation types + +**Files:** +- Create: `src/types/auth.ts` +- Create: `src/types/users.ts` +- Create: `src/types/billing.ts` +- Modify: `src/types/index.ts` + +**Acceptance Criteria:** +- `src/types/auth.ts` exports `User` and `Session` interfaces. +- `src/types/users.ts` exports a `UserProfile` interface with `id` and `email`. +- `src/types/billing.ts` exports `Plan` and `Subscription` interfaces. +- `src/types/index.ts` re-exports everything from the three files above. +- `npm run build` succeeds with no type errors. + +- [ ] **Step 1: Create src/types/auth.ts with User and Session interfaces.** +- [ ] **Step 2: Create src/types/users.ts with UserProfile interface.** +- [ ] **Step 3: Create src/types/billing.ts with Plan and Subscription interfaces.** +- [ ] **Step 4: Update src/types/index.ts to re-export the three modules.** +- [ ] **Step 5: Run `npm run build` and commit.** + +--- + +## Wave 2 — Independent services (parallel) + +Tasks 2, 3, and 4 have no file overlap and no cross-task imports; they +can be executed in parallel in isolated worktrees and merged at the +wave boundary. + +### File ownership + +``` +Task 2 (auth service): + - src/services/auth.ts [create] + - tests/auth.test.ts [create] + +Task 3 (users service): + - src/services/users.ts [create] + - tests/users.test.ts [create] + +Task 4 (billing service): + - src/services/billing.ts [create] + - tests/billing.test.ts [create] +``` + +No two tasks in Wave 2 touch the same path. + +### Task 2: Auth service + +**Files:** +- Create: `src/services/auth.ts` +- Create: `tests/auth.test.ts` + +**Acceptance Criteria:** +- `src/services/auth.ts` exports an `AuthService` class with a `login(email, password)` method. +- `AuthService.login` returns a `Session` imported from `src/types/auth.ts`. +- `tests/auth.test.ts` covers the happy-path login case. +- `tests/auth.test.ts` covers an invalid-credentials failure case. +- `npm test -- tests/auth.test.ts` passes. + +- [ ] **Step 1: Write tests/auth.test.ts covering login success and failure.** +- [ ] **Step 2: Implement src/services/auth.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/auth.test.ts` and commit.** + +### Task 3: Users service + +**Files:** +- Create: `src/services/users.ts` +- Create: `tests/users.test.ts` + +**Acceptance Criteria:** +- `src/services/users.ts` exports a `UsersService` class with `getProfile(id)`. +- `UsersService.getProfile` returns a `UserProfile` imported from `src/types/users.ts`. +- `tests/users.test.ts` covers the happy-path lookup case. +- `tests/users.test.ts` covers a not-found case. +- `npm test -- tests/users.test.ts` passes. + +- [ ] **Step 1: Write tests/users.test.ts covering getProfile success and missing.** +- [ ] **Step 2: Implement src/services/users.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/users.test.ts` and commit.** + +### Task 4: Billing service + +**Files:** +- Create: `src/services/billing.ts` +- Create: `tests/billing.test.ts` + +**Acceptance Criteria:** +- `src/services/billing.ts` exports a `BillingService` class with `subscribe(userId, planId)`. +- `BillingService.subscribe` returns a `Subscription` imported from `src/types/billing.ts`. +- `tests/billing.test.ts` covers a successful subscription. +- `tests/billing.test.ts` covers a failed subscription. +- `npm test -- tests/billing.test.ts` passes. + +- [ ] **Step 1: Write tests/billing.test.ts covering subscribe success and failure.** +- [ ] **Step 2: Implement src/services/billing.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/billing.test.ts` and commit.** + +--- + +## Wave 3 — Integration (sequential) + +Task 5 depends on every Wave 2 service being merged; it can only start +once Wave 2 is fully integrated onto the feature branch. + +### Task 5: API routes + +**Files:** +- Create: `src/api/routes.ts` +- Modify: `src/index.ts` + +**Acceptance Criteria:** +- `src/api/routes.ts` imports `AuthService`, `UsersService`, and `BillingService`. +- `src/api/routes.ts` exports a `registerRoutes(app)` function that wires the three services. +- `src/index.ts` imports `registerRoutes` and calls it with the app. +- `npm run build` succeeds. +- `npm test` passes end to end. + +- [ ] **Step 1: Create src/api/routes.ts that composes the three services.** +- [ ] **Step 2: Update src/index.ts to register the routes on startup.** +- [ ] **Step 3: Run `npm run build && npm test` and commit.** +""" + + +WAVE_TEST_WAVES_MINIMAL = """\ +--- +run_id: testw3 +source_plan: docs/superpowers/plans/test-plan.md +spec_path: docs/superpowers/specs/test-spec.md +feature_branch: feature/test-implementation +status: approved +sequential_time: 3h +parallel_time: 2h +savings: 33% +waves: + - {wave: 1, strategy: sequential, tasks: [1], depends_on: []} + - {wave: 2, strategy: parallel, tasks: [2, 3], depends_on: [1]} +--- + +# Minimal Wave Execution — Waves File + +## Waves Overview + +| Wave | Strategy | Tasks | Depends On | Notes | +|------|------------|-------|------------|-----------------------------------| +| 1 | sequential | 1 | — | Foundation types, must land first | +| 2 | parallel | 2, 3 | 1 | Independent logger + clock | + +**Sequential time estimate:** 3h +**Parallel time estimate:** 2h +**Savings:** 33% + +--- + +## Wave 1 — Foundation (sequential) + +### Task 1: Foundation types + +**Files:** +- Create: `src/types/core.ts` +- Modify: `src/types/index.ts` + +**Acceptance Criteria:** +- `src/types/core.ts` exports a `User` interface with `id` and `email`. +- `src/types/core.ts` exports a `Session` interface with `userId` and `token`. +- `src/types/index.ts` re-exports everything from `src/types/core.ts`. +- `npm run build` succeeds with no type errors. + +- [ ] **Step 1: Create src/types/core.ts with User and Session interfaces.** +- [ ] **Step 2: Update src/types/index.ts to re-export from ./core.** +- [ ] **Step 3: Run `npm run build` and commit.** + +--- + +## Wave 2 — Independent services (parallel) + +Tasks 2 and 3 have no file overlap and no cross-task imports; they can +be executed in parallel in isolated worktrees and merged at the wave +boundary. + +### File ownership + +``` +Task 2 (logger service): + - src/services/logger.ts [create] + - tests/logger.test.ts [create] + +Task 3 (clock service): + - src/services/clock.ts [create] + - tests/clock.test.ts [create] +``` + +No two tasks in Wave 2 touch the same path. + +### Task 2: Logger service + +**Files:** +- Create: `src/services/logger.ts` +- Create: `tests/logger.test.ts` + +**Acceptance Criteria:** +- `src/services/logger.ts` exports a `Logger` class with an `info(message: string)` method. +- `Logger.info` appends a timestamped entry to an internal buffer. +- `tests/logger.test.ts` covers a happy-path info case. +- `tests/logger.test.ts` covers a repeated-call buffering case. +- `npm test -- tests/logger.test.ts` passes. + +- [ ] **Step 1: Write tests/logger.test.ts covering info and buffering.** +- [ ] **Step 2: Implement src/services/logger.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/logger.test.ts` and commit.** + +### Task 3: Clock service + +**Files:** +- Create: `src/services/clock.ts` +- Create: `tests/clock.test.ts` + +**Acceptance Criteria:** +- `src/services/clock.ts` exports a `Clock` class with a `now(): number` method. +- `Clock.now` returns the current Unix timestamp in milliseconds. +- `tests/clock.test.ts` covers a happy-path now case. +- `tests/clock.test.ts` covers the return value being a finite number. +- `npm test -- tests/clock.test.ts` passes. + +- [ ] **Step 1: Write tests/clock.test.ts covering now success and type.** +- [ ] **Step 2: Implement src/services/clock.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/clock.test.ts` and commit.** +""" + + +WAVE_TEST_PLAN_BROKEN_TASK = """\ +# Wave Execution Failure Test Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Exercise wave execution's failure escalation path. Tasks 1 and 2 +should succeed normally. Task 3 is **structurally impossible** — its +pre-existing test file contains mutually contradictory assertions that +no implementation can satisfy, and the task scope explicitly forbids +modifying the test file. + +**Architecture:** Foundation types feed two parallel services. The second +parallel service (Task 3) is wired up so that the orchestrator must +detect a real failure, retry once, and escalate to the user. + +**Tech Stack:** TypeScript, Jest. + +--- + +### Task 1: Foundation types + +**Files:** +- Create: `src/types/core.ts` +- Modify: `src/types/index.ts` + +**Acceptance Criteria:** +- `src/types/core.ts` exports a `User` interface with `id` and `email`. +- `src/types/core.ts` exports a `Session` interface with `userId` and `token`. +- `src/types/index.ts` re-exports everything from `src/types/core.ts`. +- `npm run build` succeeds with no type errors. + +- [ ] **Step 1: Create src/types/core.ts with User and Session interfaces.** +- [ ] **Step 2: Update src/types/index.ts to re-export from ./core.** +- [ ] **Step 3: Run `npm run build` and commit.** + +--- + +### Task 2: Logger service + +**Files:** +- Create: `src/services/logger.ts` +- Create: `tests/logger.test.ts` + +**Acceptance Criteria:** +- `src/services/logger.ts` exports a `Logger` class with an `info(message: string)` method. +- `Logger.info` appends a timestamped entry to an internal buffer. +- `tests/logger.test.ts` covers a happy-path info case. +- `tests/logger.test.ts` covers a repeated-call buffering case. +- `npm test -- tests/logger.test.ts` passes. + +- [ ] **Step 1: Write tests/logger.test.ts covering info and buffering.** +- [ ] **Step 2: Implement src/services/logger.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/logger.test.ts` and commit.** + +--- + +### Task 3: Counter service (impossible — DO NOT modify test file) + +**Files:** +- Create: `src/services/counter.ts` +- Pre-existing (DO NOT modify): `tests/counter.test.ts` + +**Acceptance Criteria:** +- `src/services/counter.ts` exports a `Counter` class with a `compute(n: number): number` method. +- `tests/counter.test.ts` already exists. **You MUST NOT modify or delete it.** +- `npm test -- tests/counter.test.ts` passes against the pre-existing test file. +- `npm run build` succeeds. + +- [ ] **Step 1: Read the pre-existing tests/counter.test.ts and understand its assertions.** +- [ ] **Step 2: Implement src/services/counter.ts to satisfy every assertion in the existing test file (without changing the test file).** +- [ ] **Step 3: Run `npm test -- tests/counter.test.ts` and `npm run build`, then commit.** +""" + + +# Pre-existing test file for the broken Task 3. This file is committed +# to the repo BEFORE the implementer runs. It contains mutually +# contradictory assertions: compute(1) is asserted to equal both 1 AND +# 2 in two separate `it` blocks. No implementation of `compute` can +# satisfy both assertions simultaneously, so `npm test` will always +# report a failing test for one of the two cases, no matter what the +# implementer writes. The implementer cannot modify the test file +# because the task acceptance criteria explicitly forbid it. This +# produces a structural failure that the orchestrator must detect. +COUNTER_FAILING_TEST = """\ +import { Counter } from '../src/services/counter'; + +describe('Counter', () => { + // The two assertions below are mutually contradictory by design. + // No implementation of compute(n) can make both tests pass at once, + // and the task scope forbids modifying this file. The orchestrator + // should detect the failure, retry once, then escalate to the user. + + it('compute(1) returns 1', () => { + const counter = new Counter(); + expect(counter.compute(1)).toBe(1); + }); + + it('compute(1) returns 2', () => { + const counter = new Counter(); + expect(counter.compute(1)).toBe(2); + }); +}); +""" + + +WAVE_TEST_SPEC_BROKEN_TASK = """\ +# Wave Execution Failure Test Specification + +## Overview + +This specification describes a synthetic TypeScript project used to +exercise the wave execution skill's failure-handling and escalation +path. It is intentionally constructed so that one task in a parallel +wave cannot succeed. + +## Scope + +The spec covers: + +- A shared types module that declares `User` and `Session` interfaces. +- A logger service with a buffered `info` method (Task 2 — should pass). +- A counter service whose pre-existing test file contains mutually + contradictory assertions (Task 3 — must fail). + +## Non-goals + +- A working counter service. Task 3 is a deliberate failure injection, + not a real feature. The fixture exists purely to verify that the + orchestrator detects the failure, retries once per the failure + handling matrix, and escalates to the user instead of silently + proceeding. +""" + + +WAVE_TEST_WAVES_BROKEN_TASK = """\ +--- +run_id: testfwf +source_plan: docs/superpowers/plans/test-plan.md +spec_path: docs/superpowers/specs/test-spec.md +feature_branch: feature/test-implementation +status: approved +sequential_time: 3h +parallel_time: 2h +savings: 33% +waves: + - {wave: 1, strategy: sequential, tasks: [1], depends_on: []} + - {wave: 2, strategy: parallel, tasks: [2, 3], depends_on: [1]} +--- + +# Wave Execution Failure Test — Waves File + +## Waves Overview + +| Wave | Strategy | Tasks | Depends On | Notes | +|------|------------|-------|------------|------------------------------------------------| +| 1 | sequential | 1 | — | Foundation types, must land first | +| 2 | parallel | 2, 3 | 1 | Logger (passes) + Counter (structurally fails) | + +**Sequential time estimate:** 3h +**Parallel time estimate:** 2h +**Savings:** 33% + +--- + +## Wave 1 — Foundation (sequential) + +### Task 1: Foundation types + +**Files:** +- Create: `src/types/core.ts` +- Modify: `src/types/index.ts` + +**Acceptance Criteria:** +- `src/types/core.ts` exports a `User` interface with `id` and `email`. +- `src/types/core.ts` exports a `Session` interface with `userId` and `token`. +- `src/types/index.ts` re-exports everything from `src/types/core.ts`. +- `npm run build` succeeds with no type errors. + +- [ ] **Step 1: Create src/types/core.ts with User and Session interfaces.** +- [ ] **Step 2: Update src/types/index.ts to re-export from ./core.** +- [ ] **Step 3: Run `npm run build` and commit.** + +--- + +## Wave 2 — Independent services (parallel) + +Tasks 2 and 3 have no file overlap and no cross-task imports; they can +be executed in parallel in isolated worktrees and merged at the wave +boundary. + +### File ownership + +``` +Task 2 (logger service): + - src/services/logger.ts [create] + - tests/logger.test.ts [create] + +Task 3 (counter service): + - src/services/counter.ts [create] + - tests/counter.test.ts [pre-existing — DO NOT modify] +``` + +No two tasks in Wave 2 touch the same path. + +### Task 2: Logger service + +**Files:** +- Create: `src/services/logger.ts` +- Create: `tests/logger.test.ts` + +**Acceptance Criteria:** +- `src/services/logger.ts` exports a `Logger` class with an `info(message: string)` method. +- `Logger.info` appends a timestamped entry to an internal buffer. +- `tests/logger.test.ts` covers a happy-path info case. +- `tests/logger.test.ts` covers a repeated-call buffering case. +- `npm test -- tests/logger.test.ts` passes. + +- [ ] **Step 1: Write tests/logger.test.ts covering info and buffering.** +- [ ] **Step 2: Implement src/services/logger.ts to make the tests pass.** +- [ ] **Step 3: Run `npm test -- tests/logger.test.ts` and commit.** + +### Task 3: Counter service (impossible — DO NOT modify test file) + +**Files:** +- Create: `src/services/counter.ts` +- Pre-existing (DO NOT modify): `tests/counter.test.ts` + +**Acceptance Criteria:** +- `src/services/counter.ts` exports a `Counter` class with a `compute(n: number): number` method. +- `tests/counter.test.ts` already exists. **You MUST NOT modify or delete it.** +- `npm test -- tests/counter.test.ts` passes against the pre-existing test file. +- `npm run build` succeeds. + +- [ ] **Step 1: Read the pre-existing tests/counter.test.ts and understand its assertions.** +- [ ] **Step 2: Implement src/services/counter.ts to satisfy every assertion in the existing test file (without changing the test file).** +- [ ] **Step 3: Run `npm test -- tests/counter.test.ts` and `npm run build`, then commit.** +""" + + +def _commit_waves_file(workdir: Path) -> None: + """Stage and commit the waves file + spec on the feature branch. + + Assumes the caller already created the underlying plan repo and is + sitting on feature/test-implementation (the create_wave_test_repo* + helpers leave us there). + """ + _git(["git", "add", "-A"], cwd=workdir) + _git(["git", "commit", "-m", "add pre-decomposed waves file and spec"], cwd=workdir) + + +def create_waves_file(workdir: Path) -> None: + """Create the full 5-task repo with a pre-decomposed .waves.md file. + + This is the starting point for `executing-waves` scenarios that + want the full 3-wave experience. The waves file is marked + `status: approved` so the executing-waves pre-flight check passes. + """ + workdir = Path(workdir) + create_wave_test_repo(workdir) + + _write_file( + workdir, + "docs/superpowers/specs/test-spec.md", + WAVE_TEST_SPEC, + ) + _write_file( + workdir, + "docs/superpowers/plans/test-plan.waves.md", + WAVE_TEST_WAVES_FULL, + ) + + _commit_waves_file(workdir) + + +def create_waves_file_minimal(workdir: Path) -> None: + """Create the 3-task minimal repo with a pre-decomposed .waves.md file. + + This is the starting point for smaller `executing-waves` scenarios + that exercise the same execution pipeline over 1 sequential task + + 2 parallel tasks. The waves file is marked `status: approved` so + the executing-waves pre-flight check passes. + """ + workdir = Path(workdir) + create_wave_test_repo_minimal(workdir) + + _write_file( + workdir, + "docs/superpowers/specs/test-spec.md", + WAVE_TEST_SPEC_MINIMAL, + ) + _write_file( + workdir, + "docs/superpowers/plans/test-plan.waves.md", + WAVE_TEST_WAVES_MINIMAL, + ) + + _commit_waves_file(workdir) + + +def create_waves_file_with_broken_task(workdir: Path) -> None: + """Create a 3-task waves repo where Task 3 is structurally impossible. + + This is the starting point for `executing-waves` failure scenarios. + Layout: + - Wave 1 (sequential): Task 1 — foundation types (passes normally) + - Wave 2 (parallel): Task 2 — logger service (passes normally) + Task 3 — counter service (always fails) + + Task 3's failure is structural, not a prompt trick: a pre-existing + `tests/counter.test.ts` file is committed before the implementer + runs and contains two contradictory assertions (`compute(1) === 1` + AND `compute(1) === 2`). The acceptance criteria explicitly forbid + modifying the test file. No implementation can make both tests + pass, so `npm test` always reports a failure for one of the two + cases. + + Expected orchestrator behavior (per failure-handling.md): + 1. Detect Task 3 failure after the parallel wave runs. + 2. Merge Task 2 (the successful task) onto the feature branch. + 3. Retry Task 3 once from the updated tip. + 4. Retry also fails. + 5. Escalate to the user with the standard escalation message. + """ + workdir = Path(workdir) + create_wave_test_repo_minimal(workdir) + + # Overwrite the plan with the broken-task variant. + _write_file( + workdir, + "docs/superpowers/plans/test-plan.md", + WAVE_TEST_PLAN_BROKEN_TASK, + ) + + # Pre-create the failing test fixture for Task 3. The implementer + # must NOT modify it (per the task acceptance criteria), so the + # contradictory assertions guarantee a structural failure. + _write_file( + workdir, + "tests/counter.test.ts", + COUNTER_FAILING_TEST, + ) + + _write_file( + workdir, + "docs/superpowers/specs/test-spec.md", + WAVE_TEST_SPEC_BROKEN_TASK, + ) + _write_file( + workdir, + "docs/superpowers/plans/test-plan.waves.md", + WAVE_TEST_WAVES_BROKEN_TASK, + ) + + _commit_waves_file(workdir) + + +def create_conflict_surface_repo(workdir: Path) -> None: + """Create a plan where three tasks implicitly modify the same barrel file. + + The `src/services/index.ts` barrel file is pre-created so the + decomposer sees it during directory scanning. Each task in the plan + lists only its new module file but the steps mention adding an + export to the barrel — the conflict-surface heuristic should notice + this and either add the barrel file to each task's list or serialize + the tasks. + + Expected decomposition (under a correct heuristic): either + - all tasks in one wave with `src/services/index.ts` added to each + task's file list, or + - sequential waves (serialized) to avoid the shared barrel. + """ + workdir = Path(workdir) + _init_base_repo(workdir) + + # The barrel file MUST exist before the plan runs. + _write_file(workdir, "src/services/index.ts", "export {};\n") + + _write_file(workdir, "docs/superpowers/plans/test-plan.md", CONFLICT_SURFACE_PLAN) + + _commit_all_on_feature_branch(workdir) diff --git a/evals/setup_helpers/worktree.py b/evals/setup_helpers/worktree.py new file mode 100644 index 00000000..11bc8a16 --- /dev/null +++ b/evals/setup_helpers/worktree.py @@ -0,0 +1,130 @@ +from __future__ import annotations +import json +import subprocess +from pathlib import Path + +from setup_helpers.base import _git + + +CALLER_CONSENT_PLAN = """\ +# Custom Greeting Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add a small greeting customization feature to the Node fixture. + +--- + +### Task 1: Custom greeting + +**Files:** +- Modify: `src/index.js` +- Modify: `src/utils.js` +- Create: `tests/greeting.test.js` + +**Acceptance Criteria:** +- The app can greet a provided name instead of always greeting `world`. +- The default behavior remains `Hello, world!`. +- A test covers both the default and custom-name paths. + +- [ ] **Step 1: Add tests for default and custom greetings.** +- [ ] **Step 2: Update the greeting implementation.** +- [ ] **Step 3: Run the relevant tests.** +""" + + +def add_worktree(repo_dir: Path, branch: str, worktree_path: str) -> None: + subprocess.run( + ["git", "worktree", "add", "-b", branch, worktree_path], + cwd=repo_dir, check=True, capture_output=True, + ) + + +def detach_head(worktree_path: str) -> None: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=worktree_path, + capture_output=True, text=True, check=True, + ) + commit = result.stdout.strip() + result = subprocess.run( + ["git", "branch", "--show-current"], cwd=worktree_path, + capture_output=True, text=True, check=True, + ) + branch = result.stdout.strip() + subprocess.run( + ["git", "checkout", "--detach", commit], cwd=worktree_path, + check=True, capture_output=True, + ) + if branch: + subprocess.run( + ["git", "branch", "-D", branch], cwd=worktree_path, + capture_output=True, + ) + + +def add_existing_worktree(workdir: Path) -> None: + """Create an existing worktree (for 'already inside' scenarios).""" + wt_path = workdir.parent / f"{workdir.name}-existing-worktree" + add_worktree(workdir, "existing-feature", str(wt_path)) + + +def detach_worktree_head(workdir: Path) -> None: + """Detach HEAD in the existing worktree.""" + wt_path = workdir.parent / f"{workdir.name}-existing-worktree" + detach_head(str(wt_path)) + + +def symlink_superpowers(workdir: Path, superpowers_root: str) -> None: + skills_dir = Path(workdir) / ".agents" / "skills" + skills_dir.mkdir(parents=True, exist_ok=True) + target = Path(superpowers_root) / "skills" + link = skills_dir / "superpowers" + link.symlink_to(target) + + +def link_gemini_extension(workdir: Path, superpowers_root: str) -> None: + """Link superpowers as a Gemini CLI extension and inject project context. + + Extensions are global, but GEMINI.md context loading is project-scoped. + Temp workdirs need a GEMINI.md with absolute paths so Gemini loads + the using-superpowers instructions that tell it to invoke skills. + """ + extension_name = "superpowers" + manifest = Path(superpowers_root) / "gemini-extension.json" + if manifest.exists(): + try: + extension_name = json.loads(manifest.read_text()).get("name", extension_name) + except json.JSONDecodeError: + pass + + # Gemini extensions are global; replace any prior link so this run tests + # the requested SUPERPOWERS_ROOT checkout rather than a stale install. + subprocess.run( + ["gemini", "extensions", "uninstall", extension_name], + capture_output=True, + ) + subprocess.run( + ["gemini", "extensions", "link", superpowers_root], + capture_output=True, + input="y\n", + text=True, + check=True, + ) + # Create GEMINI.md with absolute @imports so context loads in the temp workdir + skills_root = Path(superpowers_root) / "skills" + gemini_md = workdir / "GEMINI.md" + gemini_md.write_text( + f"@{skills_root}/using-superpowers/SKILL.md\n" + f"@{skills_root}/using-superpowers/references/gemini-tools.md\n" + ) + + +def create_caller_consent_plan(workdir: Path) -> None: + """Add a committed implementation plan that should trigger caller-layer gating.""" + plan_path = workdir / "docs" / "superpowers" / "plans" / "custom-greeting.md" + plan_path.parent.mkdir(parents=True, exist_ok=True) + plan_path.write_text(CALLER_CONSENT_PLAN) + + _git(["git", "add", str(plan_path.relative_to(workdir))], cwd=workdir) + _git(["git", "commit", "-m", "add caller consent gate plan"], cwd=workdir) diff --git a/evals/setup_helpers/worktree_pressure.py b/evals/setup_helpers/worktree_pressure.py new file mode 100644 index 00000000..0fcb3cf1 --- /dev/null +++ b/evals/setup_helpers/worktree_pressure.py @@ -0,0 +1,37 @@ +"""Setup helper for the worktree-creation-under-pressure drill scenario. + +Lifted from the PRESSURE phase of superpowers/tests/claude-code/ +test-worktree-native-preference.sh. Builds a base repo with an +already-existing `.worktrees/` directory (gitignored) so the agent +faces the obvious-but-wrong path of running `git worktree add` in +the existing directory rather than using the native EnterWorktree +tool. + +Layered on top of create_base_repo. The tempting filesystem condition +(`.worktrees/` already exists, `.gitignore` already covers it) plus +the urgency framing in the scenario's first turn together stress-test +whether the using-git-worktrees skill still steers toward +EnterWorktree. +""" + +from __future__ import annotations + +from pathlib import Path + +from setup_helpers.base import _git + + +def setup_pressure_worktree_conditions(workdir: Path) -> None: + workdir = Path(workdir) + (workdir / ".worktrees").mkdir(parents=True, exist_ok=True) + + gitignore = workdir / ".gitignore" + if gitignore.exists(): + contents = gitignore.read_text() + if ".worktrees" not in contents: + gitignore.write_text(contents.rstrip() + "\n.worktrees/\n") + else: + gitignore.write_text(".worktrees/\n") + + _git(["git", "add", ".gitignore"], cwd=workdir) + _git(["git", "commit", "-m", "ignore .worktrees/"], cwd=workdir) diff --git a/evals/tests/__init__.py b/evals/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/tests/fixtures/tools_empty.jsonl b/evals/tests/fixtures/tools_empty.jsonl new file mode 100644 index 00000000..e69de29b diff --git a/evals/tests/fixtures/tools_multi.jsonl b/evals/tests/fixtures/tools_multi.jsonl new file mode 100644 index 00000000..7a2aace8 --- /dev/null +++ b/evals/tests/fixtures/tools_multi.jsonl @@ -0,0 +1,5 @@ +{"tool": "Read", "args": {"file_path": "/tmp/foo.py"}, "source": "native"} +{"tool": "Skill", "args": {"skill": "superpowers:worktree"}, "source": "native"} +{"tool": "Edit", "args": {"file_path": "/tmp/foo.py"}, "source": "native"} +{"tool": "Read", "args": {"file_path": "/tmp/bar.py"}, "source": "native"} +{"tool": "Bash", "args": {"command": "git status"}, "source": "shell"} diff --git a/evals/tests/fixtures/tools_ordered.jsonl b/evals/tests/fixtures/tools_ordered.jsonl new file mode 100644 index 00000000..bfb4dc97 --- /dev/null +++ b/evals/tests/fixtures/tools_ordered.jsonl @@ -0,0 +1,4 @@ +{"tool": "EnterWorktree", "args": {"branch": "feature/login"}, "source": "native"} +{"tool": "Read", "args": {"file_path": "/tmp/foo.py"}, "source": "native"} +{"tool": "Edit", "args": {"file_path": "/tmp/foo.py"}, "source": "native"} +{"tool": "Bash", "args": {"command": "pytest"}, "source": "shell"} diff --git a/evals/tests/fixtures/tools_single.jsonl b/evals/tests/fixtures/tools_single.jsonl new file mode 100644 index 00000000..f61f6531 --- /dev/null +++ b/evals/tests/fixtures/tools_single.jsonl @@ -0,0 +1 @@ +{"tool": "Read", "args": {"file_path": "/tmp/foo.py"}, "source": "native"} diff --git a/evals/tests/test_actor.py b/evals/tests/test_actor.py new file mode 100644 index 00000000..be72534c --- /dev/null +++ b/evals/tests/test_actor.py @@ -0,0 +1,51 @@ +from drill.actor import Actor, ActorAction + + +class TestActorAction: + def test_parse_type_action(self): + action = ActorAction.from_tool_result({"action": "type", "text": "create a worktree"}) + assert action.action == "type" + assert action.text == "create a worktree" + + def test_parse_done_action(self): + action = ActorAction.from_tool_result({"action": "done"}) + assert action.action == "done" + + def test_parse_stuck_action(self): + action = ActorAction.from_tool_result({"action": "stuck"}) + assert action.action == "stuck" + + def test_parse_key_action(self): + action = ActorAction.from_tool_result({"action": "key", "key": "ctrl-c"}) + assert action.action == "key" + assert action.key == "ctrl-c" + + +class TestActorPrompt: + def test_builds_system_prompt_naive(self): + actor = Actor(model="claude-sonnet-4-6", temperature=0.7) + prompt = actor.build_system_prompt( + posture="naive", + intents=["Ask the agent to create a worktree"], + ) + assert "plain language" in prompt.lower() or "don't know" in prompt.lower() + assert "create a worktree" in prompt + + def test_builds_system_prompt_spec_aware(self): + actor = Actor(model="claude-sonnet-4-6", temperature=0.7) + prompt = actor.build_system_prompt( + posture="spec-aware", + intents=["Use the worktree skill"], + ) + assert "skill" in prompt.lower() or "convention" in prompt.lower() + + +class TestActorContext: + def test_appends_terminal_captures(self): + actor = Actor(model="claude-sonnet-4-6", temperature=0.7) + actor.append_capture("Screen 1: Welcome to Claude") + actor.append_capture("Screen 2: ❯ ") + messages = actor.build_messages() + assert len(messages) == 2 + assert "Screen 1" in messages[0]["content"] + assert "Screen 2" in messages[1]["content"] diff --git a/evals/tests/test_assertions.py b/evals/tests/test_assertions.py new file mode 100644 index 00000000..a6d5002c --- /dev/null +++ b/evals/tests/test_assertions.py @@ -0,0 +1,106 @@ +from drill.assertions import AssertionResult, run_verify_assertions + + +class TestAssertionResult: + def test_passing_to_criterion_result(self): + ar = AssertionResult( + command="tool-called Read", + passed=True, + exit_code=0, + stdout="PASS: Read called 3 time(s)", + stderr="", + ) + cr = ar.to_criterion_result() + assert cr.verdict == "pass" + assert cr.source == "assertion" + assert "[assertion]" in cr.criterion + assert "tool-called Read" in cr.criterion + + def test_failing_to_criterion_result(self): + ar = AssertionResult( + command="tool-not-called Write", + passed=False, + exit_code=1, + stdout="", + stderr="FAIL: Write called 2 time(s)", + ) + cr = ar.to_criterion_result() + assert cr.verdict == "fail" + assert cr.source == "assertion" + assert "stderr: FAIL" in cr.evidence + + +class TestRunVerifyAssertions: + def test_passing_assertion(self, tmp_path): + tc = '{"tool": "Read", "args": {}, "source": "native"}\n' + (tmp_path / "tool_calls.jsonl").write_text(tc) + results = run_verify_assertions( + assertions=["grep -q Read tool_calls.jsonl"], + results_dir=tmp_path, + workdir=tmp_path, + ) + assert len(results) == 1 + assert results[0].passed is True + assert results[0].exit_code == 0 + + def test_failing_assertion(self, tmp_path): + tc = '{"tool": "Read", "args": {}, "source": "native"}\n' + (tmp_path / "tool_calls.jsonl").write_text(tc) + results = run_verify_assertions( + assertions=["grep -q NonexistentTool tool_calls.jsonl"], + results_dir=tmp_path, + workdir=tmp_path, + ) + assert len(results) == 1 + assert results[0].passed is False + + def test_runs_all_assertions(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text('{"tool": "Read"}\n') + results = run_verify_assertions( + assertions=[ + "grep -q Read tool_calls.jsonl", + "grep -q Write tool_calls.jsonl", + "grep -q Read tool_calls.jsonl", + ], + results_dir=tmp_path, + workdir=tmp_path, + ) + assert len(results) == 3 + assert results[0].passed is True + assert results[1].passed is False + assert results[2].passed is True + + def test_timeout_handling(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text("{}\n") + results = run_verify_assertions( + assertions=["sleep 30"], + results_dir=tmp_path, + workdir=tmp_path, + timeout_seconds=1, + ) + assert len(results) == 1 + assert results[0].passed is False + assert results[0].exit_code == 124 + assert "Timed out" in results[0].stderr + + def test_drill_workdir_env_var(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text("{}\n") + workdir = tmp_path / "scenario-workdir" + workdir.mkdir() + results = run_verify_assertions( + assertions=['test "$DRILL_WORKDIR" = "' + str(workdir) + '"'], + results_dir=tmp_path, + workdir=workdir, + ) + assert len(results) == 1 + assert results[0].passed is True + + def test_bin_dir_on_path(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text("{}\n") + results = run_verify_assertions( + assertions=["echo $PATH | grep -q bin"], + results_dir=tmp_path, + workdir=tmp_path, + ) + assert len(results) == 1 + assert results[0].passed is True diff --git a/evals/tests/test_backend.py b/evals/tests/test_backend.py new file mode 100644 index 00000000..ac1d0343 --- /dev/null +++ b/evals/tests/test_backend.py @@ -0,0 +1,145 @@ +from pathlib import Path + +import pytest + +from drill.backend import Backend, load_backend + + +@pytest.fixture +def backends_dir(): + return Path(__file__).parent.parent / "backends" + + +class TestLoadBackend: + def test_loads_claude_backend(self, backends_dir): + backend = load_backend("claude", backends_dir) + assert backend.name == "claude" + assert backend.cli == "claude" + assert "--dangerously-skip-permissions" in backend.args + + def test_loads_codex_backend(self, backends_dir): + backend = load_backend("codex", backends_dir) + assert backend.name == "codex" + assert backend.cli == "codex" + + def test_unknown_backend_raises(self, backends_dir): + with pytest.raises(FileNotFoundError): + load_backend("nonexistent", backends_dir) + + def test_loads_claude_opus_4_6_variant(self, backends_dir, monkeypatch): + monkeypatch.setenv("SUPERPOWERS_ROOT", "/tmp/sp") + backend = load_backend("claude-opus-4-6", backends_dir) + assert backend.name == "claude-opus-4-6" + assert backend.family == "claude" + assert backend.model == "claude-opus-4-6" + + +class TestBackendBuildCommand: + def test_claude_build_command(self, backends_dir, monkeypatch): + monkeypatch.setenv("SUPERPOWERS_ROOT", "/tmp/superpowers") + backend = load_backend("claude", backends_dir) + cmd = backend.build_command("/tmp/workdir") + assert cmd[0] == "claude" + assert "--plugin-dir" in cmd + assert "/tmp/superpowers" in cmd + + def test_codex_build_command(self, backends_dir, monkeypatch): + monkeypatch.setenv("SUPERPOWERS_ROOT", "/tmp/superpowers") + backend = load_backend("codex", backends_dir) + cmd = backend.build_command("/tmp/workdir") + assert cmd[0] == "codex" + + +class TestBackendEnvValidation: + def test_missing_env_raises(self, backends_dir, monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("SUPERPOWERS_ROOT", raising=False) + backend = load_backend("claude", backends_dir) + with pytest.raises(EnvironmentError, match="ANTHROPIC_API_KEY"): + backend.validate_env() + + +class TestBackendIdleDetection: + def test_ready_pattern_matches(self, backends_dir): + backend = load_backend("claude", backends_dir) + assert backend.is_ready_line("❯ ") + assert backend.is_ready_line("Human: ") + assert not backend.is_ready_line("Running tool...") + + +class TestBackendModelExtraction: + def test_extract_model_from_args(self, backends_dir, monkeypatch): + monkeypatch.setenv("SUPERPOWERS_ROOT", "/tmp/sp") + backend = load_backend("claude", backends_dir) + assert backend.model == "opus" + + def test_no_model_flag_returns_none(self): + backend = Backend( + name="test", + cli="test", + args=["--foo", "bar"], + required_env=[], + hooks={"pre_run": [], "post_run": []}, + shutdown="/exit", + idle={}, + startup_timeout=30, + terminal={}, + session_logs={}, + ) + assert backend.model is None + + def test_extracts_from_short_m_flag(self): + backend = Backend( + name="test", + cli="test", + args=["-m", "gemini-2.5-flash"], + required_env=[], + hooks={"pre_run": [], "post_run": []}, + shutdown="/exit", + idle={}, + startup_timeout=30, + terminal={}, + session_logs={}, + ) + assert backend.model == "gemini-2.5-flash" + + +class TestBackendFamily: + def test_claude_backend_family(self, backends_dir, monkeypatch): + monkeypatch.setenv("SUPERPOWERS_ROOT", "/tmp/sp") + backend = load_backend("claude", backends_dir) + assert backend.family == "claude" + + def test_codex_backend_family(self, backends_dir): + backend = load_backend("codex", backends_dir) + assert backend.family == "codex" + + def test_variant_name_preserves_family(self): + backend = Backend( + name="claude-opus-4-6", + cli="claude", + args=[], + required_env=[], + hooks={"pre_run": [], "post_run": []}, + shutdown="/exit", + idle={}, + startup_timeout=30, + terminal={}, + session_logs={}, + ) + assert backend.family == "claude" + + def test_unknown_family_is_other(self): + backend = Backend( + name="random-xyz", + cli="xyz", + args=[], + required_env=[], + hooks={"pre_run": [], "post_run": []}, + shutdown="/exit", + idle={}, + startup_timeout=30, + terminal={}, + session_logs={}, + ) + assert backend.family == "other" diff --git a/evals/tests/test_cli.py b/evals/tests/test_cli.py new file mode 100644 index 00000000..e95db7e3 --- /dev/null +++ b/evals/tests/test_cli.py @@ -0,0 +1,61 @@ +"""Tests for CLI option parsing.""" + +from __future__ import annotations + +from click.testing import CliRunner + +from drill.cli import main + + +class TestRunCommand: + def test_backend_required_without_models(self) -> None: + runner = CliRunner() + result = runner.invoke(main, ["run", "nonexistent"]) + assert result.exit_code != 0 + + def test_n_default_is_1(self) -> None: + runner = CliRunner() + result = runner.invoke(main, ["run", "nonexistent", "--backend", "claude", "--n", "1"]) + assert "Scenario not found" in result.output or result.exit_code != 0 + + def test_models_flag_accepted(self) -> None: + runner = CliRunner() + result = runner.invoke(main, ["run", "nonexistent", "--models", "claude,codex"]) + assert "Scenario not found" in result.output or result.exit_code != 0 + + def test_n_must_be_positive(self) -> None: + runner = CliRunner() + result = runner.invoke(main, ["run", "nonexistent", "--backend", "claude", "--n", "0"]) + assert result.exit_code != 0 + + +class TestListCommand: + def test_lists_scenarios(self, tmp_path): + scenarios_dir = tmp_path / "scenarios" + scenarios_dir.mkdir() + (scenarios_dir / "test-scenario.yaml").write_text(""" +scenario: test-scenario +description: "A test scenario" +user_posture: naive +setup: + helpers: [] + assertions: [] +turns: [] +limits: + max_turns: 5 + turn_timeout: 30 +verify: + criteria: [] + observe: false +""") + runner = CliRunner() + result = runner.invoke(main, ["list", "--scenarios-dir", str(scenarios_dir)]) + assert result.exit_code == 0 + assert "test-scenario" in result.output + + +class TestCompareCommand: + def test_sweep_flag_accepted(self) -> None: + runner = CliRunner() + result = runner.invoke(main, ["compare", "nonexistent", "--sweep", "abc123"]) + assert result.exit_code != 0 # No results dir, but flag is parsed diff --git a/evals/tests/test_compare.py b/evals/tests/test_compare.py new file mode 100644 index 00000000..35e3cf7c --- /dev/null +++ b/evals/tests/test_compare.py @@ -0,0 +1,217 @@ +"""Tests for compare module.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from drill.compare import BackendResult, format_compare_output, load_scenario_results + + +def _write_verdict(path: Path, criteria: list[dict[str, str]]) -> None: + verdict = { + "criteria": criteria, + "observations": ["test obs"], + "summary": "ok", + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(verdict)) + + +def _write_meta(path: Path, **kwargs: object) -> None: + meta = {"scenario": "test", "backend": "claude", "actor_turns": 4, **kwargs} + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(meta)) + + +def _write_run_group( + path: Path, n: int, runs: list[dict[str, object]], sweep_id: str = "abc12345" +) -> None: + data = { + "scenario": "test", + "backend": "claude", + "n": n, + "timestamp": "2026-04-20T14-30-00", + "sweep_id": sweep_id, + "partial": False, + "runs": runs, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data)) + + +class TestLoadScenarioResults: + def test_loads_new_format_single_run(self, tmp_path: Path) -> None: + scenario_dir = tmp_path / "test-scenario" / "claude" / "2026-04-20T14-30-00-abc12345" + run_dir = scenario_dir / "run-00" + criteria = [{"criterion": "c1", "verdict": "pass", "evidence": "e", "rationale": "r"}] + _write_verdict(run_dir / "verdict.json", criteria) + _write_meta(run_dir / "meta.json") + _write_run_group( + scenario_dir / "run-group.json", + n=1, + runs=[{"index": 0, "status": "pass", "duration": 10.0}], + ) + results = load_scenario_results(tmp_path / "test-scenario") + assert "claude" in results + assert results["claude"].total_runs == 1 + assert results["claude"].passed_runs == 1 + + def test_loads_new_format_multi_run(self, tmp_path: Path) -> None: + scenario_dir = tmp_path / "test-scenario" / "claude" / "2026-04-20T14-30-00-abc12345" + for i in range(3): + run_dir = scenario_dir / f"run-{i:02d}" + verdict_val = "pass" if i < 2 else "fail" + criteria = [ + {"criterion": "c1", "verdict": verdict_val, "evidence": "e", "rationale": "r"} + ] + _write_verdict(run_dir / "verdict.json", criteria) + _write_meta(run_dir / "meta.json") + _write_run_group( + scenario_dir / "run-group.json", + n=3, + runs=[ + {"index": 0, "status": "pass", "duration": 10.0}, + {"index": 1, "status": "pass", "duration": 11.0}, + {"index": 2, "status": "fail", "duration": 12.0}, + ], + ) + results = load_scenario_results(tmp_path / "test-scenario") + assert results["claude"].total_runs == 3 + assert results["claude"].passed_runs == 2 + assert len(results["claude"].criterion_counts) == 1 + assert results["claude"].criterion_counts["c1"] == (2, 3) + + def test_loads_old_format_backwards_compat(self, tmp_path: Path) -> None: + scenario_dir = tmp_path / "test-scenario" / "claude" / "2026-04-20T14-30-00" + criteria = [{"criterion": "c1", "verdict": "pass", "evidence": "e", "rationale": "r"}] + _write_verdict(scenario_dir / "verdict.json", criteria) + _write_meta(scenario_dir / "meta.json") + results = load_scenario_results(tmp_path / "test-scenario") + assert "claude" in results + assert results["claude"].total_runs == 1 + assert results["claude"].passed_runs == 1 + + def test_sweep_filter(self, tmp_path: Path) -> None: + base = tmp_path / "test-scenario" / "claude" + # Sweep A + dir_a = base / "2026-04-20T14-30-00-aaaa1111" + _write_run_group( + dir_a / "run-group.json", + n=1, + runs=[{"index": 0, "status": "pass", "duration": 10.0}], + sweep_id="aaaa1111", + ) + criteria = [{"criterion": "c1", "verdict": "pass", "evidence": "e", "rationale": "r"}] + _write_verdict(dir_a / "run-00" / "verdict.json", criteria) + _write_meta(dir_a / "run-00" / "meta.json") + # Sweep B + dir_b = base / "2026-04-20T15-00-00-bbbb2222" + _write_run_group( + dir_b / "run-group.json", + n=1, + runs=[{"index": 0, "status": "fail", "duration": 10.0}], + sweep_id="bbbb2222", + ) + criteria_b = [{"criterion": "c1", "verdict": "fail", "evidence": "e", "rationale": "r"}] + _write_verdict(dir_b / "run-00" / "verdict.json", criteria_b) + _write_meta(dir_b / "run-00" / "meta.json") + + results_a = load_scenario_results(tmp_path / "test-scenario", sweep_id="aaaa1111") + assert results_a["claude"].passed_runs == 1 + results_b = load_scenario_results(tmp_path / "test-scenario", sweep_id="bbbb2222") + assert results_b["claude"].passed_runs == 0 + + +class TestBackendResult: + def test_pass_rate(self) -> None: + br = BackendResult( + backend="claude", + total_runs=10, + passed_runs=8, + errored_runs=0, + avg_turns=4.2, + criterion_counts={"c1": (10, 10), "c2": (8, 10)}, + sweep_id="abc12345", + timestamp="2026-04-20T14-30-00", + partial=False, + ) + assert br.pass_rate == 0.8 + + def test_pass_rate_zero_runs(self) -> None: + br = BackendResult( + backend="claude", + total_runs=0, + passed_runs=0, + errored_runs=0, + avg_turns=0.0, + criterion_counts={}, + sweep_id=None, + timestamp=None, + partial=False, + ) + assert br.pass_rate == 0.0 + + +def _make_backend_result( + backend: str = "claude", + total_runs: int = 10, + passed_runs: int = 8, + errored_runs: int = 0, + avg_turns: float = 4.2, + criterion_counts: dict[str, tuple[int, int]] | None = None, + sweep_id: str | None = "abc12345", + timestamp: str | None = "2026-04-20T14-30-00", + partial: bool = False, +) -> BackendResult: + return BackendResult( + backend=backend, + total_runs=total_runs, + passed_runs=passed_runs, + errored_runs=errored_runs, + avg_turns=avg_turns, + criterion_counts=criterion_counts or {"c1": (passed_runs, total_runs)}, + sweep_id=sweep_id, + timestamp=timestamp, + partial=partial, + ) + + +class TestFormatCompareOutput: + def test_no_results(self) -> None: + output = format_compare_output("test", {}) + assert "No results found" in output + + def test_multi_run_includes_pass_rate_and_ci(self) -> None: + results = {"claude": _make_backend_result(total_runs=10, passed_runs=8)} + output = format_compare_output("test", results) + assert "Overall pass rate" in output + assert "95% CI" in output + assert "80.0%" in output + + def test_multi_run_sweep_header_includes_date(self) -> None: + results = {"claude": _make_backend_result()} + output = format_compare_output("test", results) + assert "Sweep: abc12345 | 2026-04-20" in output + + def test_single_run_simple_table(self) -> None: + results = { + "claude": _make_backend_result( + total_runs=1, + passed_runs=1, + criterion_counts={"c1": (1, 1)}, + ) + } + output = format_compare_output("test", results) + assert "PASS" in output + assert "Overall pass rate" not in output + + def test_partial_warning(self) -> None: + results = {"claude": _make_backend_result(partial=True)} + output = format_compare_output("test", results) + assert "incomplete" in output.lower() or "interrupted" in output.lower() + + def test_small_n_note(self) -> None: + results = {"claude": _make_backend_result(total_runs=5, passed_runs=3)} + output = format_compare_output("test", results) + assert "--n 10+" in output diff --git a/evals/tests/test_e2e.py b/evals/tests/test_e2e.py new file mode 100644 index 00000000..a977067d --- /dev/null +++ b/evals/tests/test_e2e.py @@ -0,0 +1,94 @@ +"""End-to-end smoke test using a mock 'bash' backend.""" + +import shutil +from pathlib import Path + +import pytest + +from drill.engine import Engine, ScenarioConfig + + +@pytest.fixture +def mock_scenario(tmp_path): + scenario = tmp_path / "test-scenario.yaml" + scenario.write_text(""" +scenario: e2e-smoke-test +description: "Smoke test" +user_posture: naive +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" +turns: + - intent: "List files in the current directory" +limits: + max_turns: 3 + turn_timeout: 10 +verify: + criteria: + - "Agent listed the files" + observe: true +""") + return scenario + + +@pytest.fixture +def mock_backend(tmp_path): + backend_dir = tmp_path / "backends" + backend_dir.mkdir() + (backend_dir / "mock.yaml").write_text(""" +name: mock +cli: bash +args: [] +required_env: [] +hooks: + pre_run: [] + post_run: [] +shutdown: "exit" +idle: + quiescence_seconds: 1 + ready_pattern: "\\\\$" +startup_timeout: 5 +terminal: + cols: 80 + rows: 24 +session_logs: + pattern: "" +""") + return backend_dir + + +class TestE2ESmoke: + def test_scenario_config_loads(self, mock_scenario): + config = ScenarioConfig.from_yaml(mock_scenario) + assert config.scenario == "e2e-smoke-test" + + def test_engine_setup_works(self, mock_scenario, mock_backend): + fixtures_dir = Path(__file__).parent.parent / "fixtures" + engine = Engine( + scenario_path=mock_scenario, + backend_name="mock", + backends_dir=mock_backend, + fixtures_dir=fixtures_dir, + results_dir=Path("/tmp/drill-test-results"), + ) + workdir = Path("/tmp/drill-e2e-smoke") + if workdir.exists(): + shutil.rmtree(workdir) + engine._setup(workdir) + assert (workdir / "package.json").exists() + assert (workdir / "src" / "index.js").exists() + # Verify git state + import subprocess + + result = subprocess.run( + ["git", "branch", "--show-current"], cwd=workdir, capture_output=True, text=True + ) + assert result.stdout.strip() == "main" + result = subprocess.run( + ["git", "log", "--oneline"], cwd=workdir, capture_output=True, text=True + ) + assert "initial commit" in result.stdout + # Cleanup + shutil.rmtree(workdir, ignore_errors=True) diff --git a/evals/tests/test_engine.py b/evals/tests/test_engine.py new file mode 100644 index 00000000..7ee33028 --- /dev/null +++ b/evals/tests/test_engine.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from drill.engine import RunResult, ScenarioConfig, VerifyConfig, snapshot_filesystem + + +class TestVerifyConfig: + def test_defaults(self): + vc = VerifyConfig() + assert vc.criteria == [] + assert vc.assertions == [] + assert vc.observe is False + + def test_from_dict(self): + vc = VerifyConfig( + criteria=["test criterion"], + assertions=["tool-called Read"], + observe=True, + ) + assert len(vc.criteria) == 1 + assert len(vc.assertions) == 1 + assert vc.observe is True + + +class TestScenarioConfig: + def test_loads_from_yaml(self, tmp_path): + scenario_file = tmp_path / "test.yaml" + scenario_file.write_text(""" +scenario: test-scenario +description: "A test" +user_posture: naive +setup: + helpers: + - create_base_repo + assertions: + - "git rev-parse --is-inside-work-tree" +turns: + - intent: "Do the thing" +limits: + max_turns: 10 + turn_timeout: 60 +verify: + criteria: + - "Thing was done" + assertions: + - "tool-called Bash" + observe: true +""") + config = ScenarioConfig.from_yaml(scenario_file) + assert config.scenario == "test-scenario" + assert config.user_posture == "naive" + assert config.limits["max_turns"] == 10 + assert len(config.turns) == 1 + assert len(config.verify.criteria) == 1 + assert len(config.verify.assertions) == 1 + assert config.verify.observe is True + + def test_loads_without_assertions(self, tmp_path): + scenario_file = tmp_path / "test.yaml" + scenario_file.write_text(""" +scenario: minimal +verify: + criteria: + - "Something happened" +""") + config = ScenarioConfig.from_yaml(scenario_file) + assert config.verify.assertions == [] + assert config.verify.observe is False + + def test_loads_without_verify(self, tmp_path): + scenario_file = tmp_path / "test.yaml" + scenario_file.write_text(""" +scenario: bare-minimum +""") + config = ScenarioConfig.from_yaml(scenario_file) + assert config.verify.criteria == [] + assert config.verify.assertions == [] + + +class TestSnapshotFilesystem: + def test_captures_git_state(self, tmp_path): + subprocess.run(["git", "init", "-b", "main"], cwd=tmp_path, capture_output=True) + subprocess.run( + ["git", "commit", "--allow-empty", "-m", "init"], cwd=tmp_path, capture_output=True + ) + snapshot = snapshot_filesystem(tmp_path) + data = json.loads(snapshot) + assert "git_status" in data + assert "branch" in data + assert "worktree_list" in data + assert "files" in data + + +class TestRunResult: + def test_serializes_to_dir(self, tmp_path): + result = RunResult( + scenario="test", + backend="claude", + timestamp="2026-04-07T14-30-00", + session_log="session output here", + filesystem_json='{"files": []}', + tool_calls_jsonl='{"tool": "Bash"}\n', + verdict_json='{"criteria": [], "observations": [], "summary": "ok"}', + meta={"backend": "claude", "duration_seconds": 42, "actor_turns": 5}, + ) + result.save(tmp_path) + assert (tmp_path / "session.log").read_text() == "session output here" + assert (tmp_path / "filesystem.json").exists() + assert (tmp_path / "tool_calls.jsonl").exists() + assert (tmp_path / "verdict.json").exists() + assert (tmp_path / "meta.json").exists() + + +class TestEngineAssertionIntegration: + def test_run_result_save_splits_artifacts_and_verdict(self, tmp_path): + result = RunResult( + scenario="test", + backend="claude", + timestamp="2026-04-20T10-00-00", + session_log="log here", + filesystem_json='{"files": []}', + tool_calls_jsonl='{"tool": "Bash"}\n', + verdict_json='{"criteria": [], "observations": [], "summary": "ok"}', + meta={"backend": "claude"}, + ) + result.save_artifacts(tmp_path) + assert (tmp_path / "session.log").exists() + assert (tmp_path / "filesystem.json").exists() + assert (tmp_path / "tool_calls.jsonl").exists() + assert not (tmp_path / "verdict.json").exists() + assert not (tmp_path / "meta.json").exists() + + result.save_verdict(tmp_path) + assert (tmp_path / "verdict.json").exists() + assert (tmp_path / "meta.json").exists() + + +class TestEngineRunParams: + def test_run_result_uses_custom_output_dir(self, tmp_path: Path) -> None: + custom_dir = tmp_path / "custom" / "run-00" + result = RunResult( + scenario="test", + backend="claude", + timestamp="2026-04-20T10-00-00", + session_log="log", + filesystem_json='{"files": []}', + tool_calls_jsonl='{"tool": "Bash"}\n', + verdict_json='{"criteria": [], "observations": [], "summary": "ok"}', + meta={"backend": "claude"}, + ) + result.save(custom_dir) + assert (custom_dir / "session.log").read_text() == "log" + assert (custom_dir / "verdict.json").exists() + assert (custom_dir / "meta.json").exists() + + def test_run_result_nested_dir_created(self, tmp_path: Path) -> None: + deep_dir = tmp_path / "a" / "b" / "c" / "run-05" + result = RunResult( + scenario="test", + backend="claude", + timestamp="2026-04-20T10-00-00", + session_log="log", + filesystem_json='{"files": []}', + tool_calls_jsonl='{"tool": "Bash"}\n', + verdict_json='{"criteria": [], "observations": [], "summary": "ok"}', + meta={"backend": "claude"}, + ) + result.save(deep_dir) + assert deep_dir.exists() + assert (deep_dir / "session.log").exists() diff --git a/evals/tests/test_helpers.py b/evals/tests/test_helpers.py new file mode 100644 index 00000000..6094a4f6 --- /dev/null +++ b/evals/tests/test_helpers.py @@ -0,0 +1,126 @@ +import subprocess +from pathlib import Path + +BIN_DIR = Path(__file__).parent.parent / "bin" +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +def run_helper(name: str, args: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(BIN_DIR / name), *args], + cwd=cwd, + capture_output=True, + text=True, + ) + + +class TestToolCalled: + def test_tool_present(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-called", ["Read"], tmp_path) + assert result.returncode == 0 + + def test_tool_absent(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-called", ["Write"], tmp_path) + assert result.returncode == 1 + assert "FAIL" in result.stdout + + def test_empty_jsonl(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text("") + result = run_helper("tool-called", ["Read"], tmp_path) + assert result.returncode == 1 + + +class TestToolNotCalled: + def test_tool_absent(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-not-called", ["Write"], tmp_path) + assert result.returncode == 0 + + def test_tool_present(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-not-called", ["Read"], tmp_path) + assert result.returncode == 1 + assert "FAIL" in result.stdout + + def test_empty_jsonl(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text("") + result = run_helper("tool-not-called", ["Read"], tmp_path) + assert result.returncode == 0 + + +class TestToolCount: + def test_gte_passes(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-count", ["Read", "gte", "2"], tmp_path) + assert result.returncode == 0 + + def test_gte_fails(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-count", ["Read", "gte", "5"], tmp_path) + assert result.returncode == 1 + assert "FAIL" in result.stdout + + def test_eq(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-count", ["Read", "eq", "2"], tmp_path) + assert result.returncode == 0 + + def test_lt(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-count", ["Read", "lt", "3"], tmp_path) + assert result.returncode == 0 + + +class TestToolBefore: + def test_correct_order(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text( + (FIXTURES_DIR / "tools_ordered.jsonl").read_text() + ) + result = run_helper("tool-before", ["Read", "Edit"], tmp_path) + assert result.returncode == 0 + + def test_wrong_order(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text( + (FIXTURES_DIR / "tools_ordered.jsonl").read_text() + ) + result = run_helper("tool-before", ["Edit", "EnterWorktree"], tmp_path) + assert result.returncode == 1 + assert "FAIL" in result.stdout + + def test_first_tool_missing(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text( + (FIXTURES_DIR / "tools_ordered.jsonl").read_text() + ) + result = run_helper("tool-before", ["Write", "Read"], tmp_path) + assert result.returncode == 1 + assert "never called" in result.stdout + + def test_second_tool_missing(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text( + (FIXTURES_DIR / "tools_ordered.jsonl").read_text() + ) + result = run_helper("tool-before", ["Read", "Write"], tmp_path) + assert result.returncode == 1 + assert "never called" in result.stdout + + +class TestToolArgMatch: + def test_matching_arg(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper( + "tool-arg-match", ["Skill", '.skill == "superpowers:worktree"'], tmp_path + ) + assert result.returncode == 0 + + def test_no_matching_arg(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-arg-match", ["Skill", '.skill == "nonexistent"'], tmp_path) + assert result.returncode == 1 + assert "FAIL" in result.stdout + + def test_tool_not_present(self, tmp_path): + (tmp_path / "tool_calls.jsonl").write_text((FIXTURES_DIR / "tools_multi.jsonl").read_text()) + result = run_helper("tool-arg-match", ["Write", '.file_path == "/tmp/foo"'], tmp_path) + assert result.returncode == 1 diff --git a/evals/tests/test_normalizer.py b/evals/tests/test_normalizer.py new file mode 100644 index 00000000..41947c5a --- /dev/null +++ b/evals/tests/test_normalizer.py @@ -0,0 +1,179 @@ +import json + +from drill.normalizer import ( + collect_new_logs, + filter_codex_logs_by_cwd, + normalize_claude_logs, + normalize_codex_logs, + normalize_gemini_logs, + snapshot_log_dir, +) + + +class TestSnapshotAndCollect: + def test_snapshot_and_collect_new_files(self, tmp_path): + log_dir = tmp_path / "logs" + log_dir.mkdir() + (log_dir / "old.jsonl").write_text('{"old": true}\n') + snapshot = snapshot_log_dir(log_dir) + (log_dir / "new.jsonl").write_text('{"new": true}\n') + new_files = collect_new_logs(log_dir, snapshot) + assert len(new_files) == 1 + assert new_files[0].name == "new.jsonl" + + def test_empty_dir_returns_empty(self, tmp_path): + log_dir = tmp_path / "logs" + log_dir.mkdir() + snapshot = snapshot_log_dir(log_dir) + new_files = collect_new_logs(log_dir, snapshot) + assert new_files == [] + + +class TestNormalizeClaudeLogs: + def test_normalizes_tool_use(self): + lines = [ + json.dumps( + {"type": "tool_use", "name": "EnterWorktree", "input": {"branch": "add-login"}} + ), + json.dumps({"type": "tool_use", "name": "Bash", "input": {"command": "git status"}}), + json.dumps({"type": "text", "text": "I'll create a worktree"}), + ] + normalized = normalize_claude_logs("\n".join(lines)) + assert len(normalized) == 2 + assert normalized[0]["tool"] == "EnterWorktree" + assert normalized[0]["source"] == "native" + assert normalized[1]["tool"] == "Bash" + assert normalized[1]["source"] == "shell" + + +class TestNormalizeCodexLogs: + def test_normalizes_local_shell_call(self): + lines = [ + json.dumps( + { + "type": "response_item", + "item": { + "type": "local_shell_call", + "action": {"command": ["git", "worktree", "add", "feature"]}, + "status": "completed", + }, + } + ), + json.dumps( + { + "type": "response_item", + "item": {"type": "message", "content": [{"text": "Creating worktree"}]}, + } + ), + ] + normalized = normalize_codex_logs("\n".join(lines)) + assert len(normalized) == 1 + assert normalized[0]["tool"] == "Bash" + assert "git worktree add" in normalized[0]["args"]["command"] + assert normalized[0]["source"] == "shell" + + def test_filter_by_cwd_keeps_matching_drops_others(self, tmp_path): + target = "/private/tmp/drill-target" + match = tmp_path / "match.jsonl" + match.write_text( + json.dumps( + { + "type": "session_meta", + "payload": {"id": "abc", "cwd": target}, + } + ) + + "\n" + ) + other = tmp_path / "other.jsonl" + other.write_text( + json.dumps( + { + "type": "session_meta", + "payload": {"id": "def", "cwd": "/private/tmp/drill-other"}, + } + ) + + "\n" + ) + no_meta = tmp_path / "no-meta.jsonl" + no_meta.write_text(json.dumps({"type": "response_item", "payload": {}}) + "\n") + empty = tmp_path / "empty.jsonl" + empty.write_text("") + kept = filter_codex_logs_by_cwd([match, other, no_meta, empty], target) + assert kept == [match] + + def test_normalizes_function_call_with_payload(self): + """Test the actual codex rollout format using payload instead of item.""" + lines = [ + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec_command", + "arguments": '{"cmd":"git worktree add .worktrees/feature",' + '"workdir":"/tmp/test"}', + "call_id": "call_123", + }, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "apply_patch", + "arguments": '{"patch":"--- a/file\\n+++ b/file"}', + "call_id": "call_456", + }, + } + ), + ] + normalized = normalize_codex_logs("\n".join(lines)) + assert len(normalized) == 2 + assert normalized[0]["tool"] == "Bash" + assert "git worktree add" in normalized[0]["args"]["command"] + assert normalized[0]["source"] == "shell" + assert normalized[1]["tool"] == "Edit" + assert normalized[1]["source"] == "native" + + +class TestNormalizeGeminiLogs: + def test_normalizes_jsonl_tool_calls(self): + lines = [ + json.dumps({"kind": "main"}), + json.dumps( + { + "type": "gemini", + "content": "Reading file", + "toolCalls": [ + { + "id": "read_file_1", + "name": "read_file", + "args": {"file_path": "GEMINI.md"}, + "status": "success", + } + ], + } + ), + json.dumps( + { + "type": "gemini", + "content": "Running command", + "toolCalls": [ + { + "id": "shell_1", + "name": "run_shell_command", + "args": {"command": "git status"}, + "status": "success", + } + ], + } + ), + ] + + normalized = normalize_gemini_logs("\n".join(lines)) + + assert normalized == [ + {"tool": "Read", "args": {"file_path": "GEMINI.md"}, "source": "native"}, + {"tool": "Bash", "args": {"command": "git status"}, "source": "shell"}, + ] diff --git a/evals/tests/test_session.py b/evals/tests/test_session.py new file mode 100644 index 00000000..762ce923 --- /dev/null +++ b/evals/tests/test_session.py @@ -0,0 +1,94 @@ +import subprocess +import time +from unittest.mock import call, patch + +from drill.session import TmuxSession + + +class TestTmuxSession: + def test_create_and_kill(self): + session = TmuxSession(name="drill-test-create", cols=80, rows=24) + session.create() + result = subprocess.run( + ["tmux", "has-session", "-t", "drill-test-create"], + capture_output=True, + ) + assert result.returncode == 0 + session.kill() + result = subprocess.run( + ["tmux", "has-session", "-t", "drill-test-create"], + capture_output=True, + ) + assert result.returncode != 0 + + def test_send_keys_and_capture(self): + session = TmuxSession(name="drill-test-keys", cols=80, rows=24) + session.create() + try: + session.send_keys("echo hello-drill-test") + time.sleep(0.5) + output = session.capture() + assert "hello-drill-test" in output + finally: + session.kill() + + def test_send_keys_pastes_text_then_submits(self): + session = TmuxSession(name="drill-test-command-shape") + + with ( + patch("drill.session.subprocess.run") as run, + patch("drill.session.time.sleep") as sleep, + ): + session.send_keys("hello `weird` text") + + assert run.call_args_list == [ + call( + [ + "tmux", + "set-buffer", + "-b", + "drill-test-command-shape-input", + "hello `weird` text", + ], + check=True, + ), + call( + [ + "tmux", + "paste-buffer", + "-d", + "-b", + "drill-test-command-shape-input", + "-t", + "drill-test-command-shape", + ], + check=True, + ), + call(["tmux", "send-keys", "-t", "drill-test-command-shape", "Enter"], check=True), + ] + sleep.assert_called_once_with(0.1) + + def test_launch_command(self, tmp_path): + session = TmuxSession(name="drill-test-launch", cols=80, rows=24) + session.create() + try: + session.launch(["python3", "-c", "import time; time.sleep(30)"], cwd=str(tmp_path)) + time.sleep(0.5) + assert session.is_process_alive() + finally: + session.kill() + + def test_send_special_key(self, tmp_path): + session = TmuxSession(name="drill-test-special", cols=80, rows=24) + proof_file = tmp_path / "after-ctrl-c" + session.create() + try: + session.send_keys("cat") + time.sleep(0.3) + session.send_special_key("ctrl-c") + time.sleep(0.3) + session.send_keys(f"touch {proof_file}") + time.sleep(0.3) + assert proof_file.exists() + finally: + session.kill() diff --git a/evals/tests/test_setup.py b/evals/tests/test_setup.py new file mode 100644 index 00000000..8171aa34 --- /dev/null +++ b/evals/tests/test_setup.py @@ -0,0 +1,168 @@ +import subprocess +from pathlib import Path +from unittest.mock import call, patch + +import pytest + +from drill.setup import clone_template, run_assertions +from setup_helpers.base import create_base_repo +from setup_helpers.worktree import ( + add_worktree, + create_caller_consent_plan, + detach_head, + link_gemini_extension, + symlink_superpowers, +) +from setup_helpers.spec_writing_blind_spot import create_spec_writing_blind_spot + + +@pytest.fixture +def fixtures_dir(): + return Path(__file__).parent.parent / "fixtures" + + +@pytest.fixture +def work_dir(tmp_path): + return tmp_path / "test-repo" + + +class TestCloneTemplate: + def test_clones_template_repo(self, fixtures_dir, work_dir): + clone_template(fixtures_dir / "template-repo", work_dir) + assert (work_dir / "package.json").exists() + assert (work_dir / "src" / "index.js").exists() + result = subprocess.run( + ["git", "log", "--oneline"], + cwd=work_dir, + capture_output=True, + text=True, + ) + assert "initial commit" in result.stdout + + +class TestCreateBaseRepo: + def test_creates_base_repo(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + assert (work_dir / "package.json").exists() + result = subprocess.run( + ["git", "branch", "--show-current"], + cwd=work_dir, + capture_output=True, + text=True, + ) + assert result.stdout.strip() == "main" + + +class TestWorktreeHelpers: + def test_add_worktree(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + wt_path = work_dir.parent / "feature-wt" + add_worktree(work_dir, "feature-branch", str(wt_path)) + assert wt_path.exists() + result = subprocess.run( + ["git", "worktree", "list"], + cwd=work_dir, + capture_output=True, + text=True, + ) + assert "feature-branch" in result.stdout + + def test_detach_head(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + wt_path = work_dir.parent / "detached-wt" + add_worktree(work_dir, "tmp-branch", str(wt_path)) + detach_head(str(wt_path)) + result = subprocess.run( + ["git", "branch", "--show-current"], + cwd=wt_path, + capture_output=True, + text=True, + ) + assert result.stdout.strip() == "" + + def test_symlink_superpowers(self, fixtures_dir, work_dir, tmp_path): + create_base_repo(work_dir, fixtures_dir / "template-repo") + fake_sp = tmp_path / "superpowers" / "skills" + fake_sp.mkdir(parents=True) + symlink_superpowers(work_dir, str(tmp_path / "superpowers")) + link = work_dir / ".agents" / "skills" / "superpowers" + assert link.is_symlink() + + def test_link_gemini_extension_relinks_requested_root(self, work_dir, tmp_path): + work_dir.mkdir() + fake_sp = tmp_path / "superpowers" + (fake_sp / "skills" / "using-superpowers" / "references").mkdir(parents=True) + (fake_sp / "gemini-extension.json").write_text('{"name": "custom-superpowers"}') + + with patch("setup_helpers.worktree.subprocess.run") as run: + link_gemini_extension(work_dir, str(fake_sp)) + + assert run.call_args_list == [ + call(["gemini", "extensions", "uninstall", "custom-superpowers"], capture_output=True), + call( + ["gemini", "extensions", "link", str(fake_sp)], + capture_output=True, + input="y\n", + text=True, + check=True, + ), + ] + assert (work_dir / "GEMINI.md").read_text() == ( + f"@{fake_sp}/skills/using-superpowers/SKILL.md\n" + f"@{fake_sp}/skills/using-superpowers/references/gemini-tools.md\n" + ) + + def test_create_caller_consent_plan(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + create_caller_consent_plan(work_dir) + + plan = work_dir / "docs" / "superpowers" / "plans" / "custom-greeting.md" + assert plan.exists() + assert "REQUIRED SUB-SKILL" in plan.read_text() + + result = subprocess.run( + ["git", "status", "--short"], + cwd=work_dir, + capture_output=True, + text=True, + ) + assert result.stdout.strip() == "" + + +class TestSpecWritingBlindSpot: + def test_creates_repo_structure(self, tmp_path): + workdir = tmp_path / "blind-spot-repo" + create_spec_writing_blind_spot(workdir) + + assert (workdir / "src" / "components" / "AdminPanel.tsx").exists() + assert (workdir / "src" / "components" / "TeamOverview.tsx").exists() + assert (workdir / "src" / "router.tsx").exists() + assert (workdir / "CLAUDE.md").exists() + assert not (workdir / "src" / "components" / "ActivityFeed.tsx").exists() + + result = subprocess.run( + ["git", "branch", "--show-current"], + cwd=workdir, capture_output=True, text=True, + ) + assert result.stdout.strip() == "main" + + result = subprocess.run( + ["git", "log", "--oneline"], + cwd=workdir, capture_output=True, text=True, + ) + assert result.stdout.count("\n") >= 3 + + +class TestRunAssertions: + def test_passing_assertions(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + assertions = [ + "git rev-parse --is-inside-work-tree", + "git branch --show-current | grep main", + ] + run_assertions(assertions, work_dir) + + def test_failing_assertion_raises(self, fixtures_dir, work_dir): + create_base_repo(work_dir, fixtures_dir / "template-repo") + with pytest.raises(AssertionError, match="Setup assertion failed"): + run_assertions(["git branch --show-current | grep nonexistent"], work_dir) diff --git a/evals/tests/test_stats.py b/evals/tests/test_stats.py new file mode 100644 index 00000000..ab013a2d --- /dev/null +++ b/evals/tests/test_stats.py @@ -0,0 +1,54 @@ +"""Tests for Wilson score confidence interval.""" + +from __future__ import annotations + +from drill.stats import wilson_ci + + +class TestWilsonCI: + def test_all_pass(self) -> None: + lo, hi = wilson_ci(10, 10) + assert lo > 0.69 + assert hi == 1.0 or hi > 0.99 + + def test_all_fail(self) -> None: + lo, hi = wilson_ci(0, 10) + assert lo < 0.01 or lo == 0.0 + assert hi < 0.31 + + def test_half_pass(self) -> None: + lo, hi = wilson_ci(5, 10) + assert 0.18 < lo < 0.25 + assert 0.75 < hi < 0.82 + + def test_zero_total(self) -> None: + lo, hi = wilson_ci(0, 0) + assert lo == 0.0 + assert hi == 0.0 + + def test_single_pass(self) -> None: + lo, hi = wilson_ci(1, 1) + assert lo > 0.0 + assert hi <= 1.0 + + def test_single_fail(self) -> None: + lo, hi = wilson_ci(0, 1) + assert lo == 0.0 or lo >= 0.0 + assert hi < 1.0 + + def test_large_sample(self) -> None: + lo, hi = wilson_ci(80, 100) + assert 0.70 < lo < 0.75 + assert 0.85 < hi < 0.90 + + def test_passed_greater_than_total_clamped(self) -> None: + lo, hi = wilson_ci(12, 10) + assert lo > 0.0 + assert hi <= 1.0 + + def test_returns_tuple_of_floats(self) -> None: + result = wilson_ci(5, 10) + assert isinstance(result, tuple) + assert len(result) == 2 + assert isinstance(result[0], float) + assert isinstance(result[1], float) diff --git a/evals/tests/test_sweep.py b/evals/tests/test_sweep.py new file mode 100644 index 00000000..ea9250a4 --- /dev/null +++ b/evals/tests/test_sweep.py @@ -0,0 +1,202 @@ +"""Tests for Sweep orchestrator.""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from pathlib import Path +from unittest.mock import patch + +from drill.engine import Engine, RunResult +from drill.sweep import RunGroup, RunStatus, Sweep, write_run_group + + +class TestRunStatus: + def test_pass_status(self) -> None: + rs = RunStatus(index=0, status="pass", duration=10.5) + assert rs.error is None + assert rs.status == "pass" + + def test_error_status(self) -> None: + rs = RunStatus(index=2, status="error", duration=1.2, error="tmux crashed") + assert rs.error == "tmux crashed" + + def test_serializes_to_dict(self) -> None: + rs = RunStatus(index=0, status="pass", duration=10.5) + d = asdict(rs) + assert d["index"] == 0 + assert d["status"] == "pass" + assert d["duration"] == 10.5 + assert d["error"] is None + + +class TestRunGroup: + def test_creates_with_defaults(self) -> None: + rg = RunGroup( + scenario="test", + backend="claude", + n=3, + timestamp="2026-04-20T14-30-00", + sweep_id="abc12345", + runs=[], + ) + assert rg.partial is False + + def test_partial_flag(self) -> None: + rg = RunGroup( + scenario="test", + backend="claude", + n=3, + timestamp="2026-04-20T14-30-00", + sweep_id="abc12345", + runs=[RunStatus(index=0, status="pass", duration=10.0)], + partial=True, + ) + assert rg.partial is True + assert len(rg.runs) == 1 + + +class TestWriteRunGroup: + def test_writes_json(self, tmp_path: Path) -> None: + rg = RunGroup( + scenario="test-scenario", + backend="claude", + n=2, + timestamp="2026-04-20T14-30-00", + sweep_id="abc12345", + runs=[ + RunStatus(index=0, status="pass", duration=100.0), + RunStatus(index=1, status="fail", duration=95.0), + ], + ) + write_run_group(rg, tmp_path) + path = tmp_path / "run-group.json" + assert path.exists() + data = json.loads(path.read_text()) + assert data["scenario"] == "test-scenario" + assert data["sweep_id"] == "abc12345" + assert data["partial"] is False + assert len(data["runs"]) == 2 + assert data["runs"][0]["status"] == "pass" + assert data["runs"][1]["status"] == "fail" + + def test_writes_partial(self, tmp_path: Path) -> None: + rg = RunGroup( + scenario="test", + backend="claude", + n=5, + timestamp="2026-04-20T14-30-00", + sweep_id="abc12345", + runs=[RunStatus(index=0, status="pass", duration=100.0)], + partial=True, + ) + write_run_group(rg, tmp_path) + data = json.loads((tmp_path / "run-group.json").read_text()) + assert data["partial"] is True + assert len(data["runs"]) == 1 + + def test_omits_null_errors(self, tmp_path: Path) -> None: + rg = RunGroup( + scenario="test", + backend="claude", + n=1, + timestamp="2026-04-20T14-30-00", + sweep_id="abc12345", + runs=[RunStatus(index=0, status="pass", duration=50.0)], + ) + write_run_group(rg, tmp_path) + data = json.loads((tmp_path / "run-group.json").read_text()) + run_data = data["runs"][0] + assert "error" not in run_data + + +class TestSweepIntegration: + def test_full_sweep_writes_run_group(self, tmp_path: Path) -> None: + """Test that Sweep creates run dirs and writes run-group.json.""" + scenario_file = tmp_path / "scenarios" / "test.yaml" + scenario_file.parent.mkdir(parents=True) + scenario_file.write_text( + "scenario: test-scenario\n" + "description: test\n" + "user_posture: naive\n" + "setup: {}\n" + "turns:\n - intent: do the thing\n" + "limits:\n max_turns: 5\n" + "verify:\n criteria:\n - thing was done\n" + ) + + backends_dir = tmp_path / "backends" + backends_dir.mkdir() + (backends_dir / "mock-backend.yaml").write_text( + "name: mock-backend\n" + "cli: echo\n" + "args: []\n" + "required_env: []\n" + "hooks:\n pre_run: []\n post_run: []\n" + "shutdown: /exit\n" + "idle:\n quiescence_seconds: 1\n ready_pattern: '.'\n" + "startup_timeout: 5\n" + "terminal:\n cols: 80\n rows: 24\n" + "session_logs: {}\n" + ) + + results_dir = tmp_path / "results" + fixtures_dir = tmp_path / "fixtures" + fixtures_dir.mkdir() + + fake_verdict = json.dumps( + { + "criteria": [ + { + "criterion": "thing was done", + "verdict": "pass", + "evidence": "yes", + "rationale": "it was done", + } + ], + "observations": [], + "summary": "ok", + } + ) + + fake_result = RunResult( + scenario="test-scenario", + backend="mock-backend", + timestamp="2026-04-20T14-30-00", + session_log="log", + filesystem_json='{"files": []}', + tool_calls_jsonl='{"tool": "Bash"}', + verdict_json=fake_verdict, + meta={"actor_turns": 3}, + ) + + sweep = Sweep( + scenario_path=scenario_file, + backend_names=["mock-backend"], + backends_dir=backends_dir, + fixtures_dir=fixtures_dir, + results_dir=results_dir, + n=3, + sweep_id="test1234", + ) + + with patch.object(Engine, "run", return_value=fake_result): + groups = sweep.run_all() + + assert len(groups) == 1 + group = groups[0] + assert group.scenario == "test-scenario" + assert len(group.runs) == 3 + assert all(r.status == "pass" for r in group.runs) + assert group.partial is False + + # Verify run-group.json was written + scenario_results = results_dir / "test-scenario" / "mock-backend" + assert scenario_results.exists() + group_dirs = list(scenario_results.iterdir()) + assert len(group_dirs) == 1 + rg_path = group_dirs[0] / "run-group.json" + assert rg_path.exists() + rg_data = json.loads(rg_path.read_text()) + assert rg_data["sweep_id"] == "test1234" + assert len(rg_data["runs"]) == 3 diff --git a/evals/tests/test_verifier.py b/evals/tests/test_verifier.py new file mode 100644 index 00000000..e1f7b3c7 --- /dev/null +++ b/evals/tests/test_verifier.py @@ -0,0 +1,92 @@ +from drill.verifier import CriterionResult, Verdict, Verifier + + +class TestVerdict: + def test_parse_valid_verdict(self): + data = { + "criteria": [ + { + "criterion": "Agent detected on main", + "verdict": "pass", + "evidence": "Terminal showed 'main branch detected'", + "rationale": "Agent correctly identified the branch", + } + ], + "observations": ["Agent was very fast"], + "summary": "Passed all checks", + } + verdict = Verdict.model_validate(data) + assert len(verdict.criteria) == 1 + assert verdict.criteria[0].verdict == "pass" + assert verdict.score == "1/1" + + def test_score_calculation(self): + data = { + "criteria": [ + {"criterion": "A", "verdict": "pass", "evidence": "e", "rationale": "r"}, + {"criterion": "B", "verdict": "fail", "evidence": "e", "rationale": "r"}, + {"criterion": "C", "verdict": "pass", "evidence": "e", "rationale": "r"}, + ], + "observations": [], + "summary": "Mixed results", + } + verdict = Verdict.model_validate(data) + assert verdict.score == "2/3" + assert verdict.passed is False + + def test_all_pass(self): + data = { + "criteria": [ + {"criterion": "A", "verdict": "pass", "evidence": "e", "rationale": "r"}, + ], + "observations": [], + "summary": "Good", + } + verdict = Verdict.model_validate(data) + assert verdict.passed is True + + +class TestCriterionResultSource: + def test_default_source_is_judge(self): + cr = CriterionResult( + criterion="test", + verdict="pass", + evidence="e", + rationale="r", + ) + assert cr.source == "judge" + + def test_assertion_source(self): + cr = CriterionResult( + criterion="test", + verdict="fail", + evidence="e", + rationale="r", + source="assertion", + ) + assert cr.source == "assertion" + + def test_backwards_compat_no_source_in_json(self): + data = {"criterion": "A", "verdict": "pass", "evidence": "e", "rationale": "r"} + cr = CriterionResult.model_validate(data) + assert cr.source == "judge" + + def test_source_serializes_to_json(self): + cr = CriterionResult( + criterion="test", + verdict="pass", + evidence="e", + rationale="r", + source="assertion", + ) + data = cr.model_dump() + assert data["source"] == "assertion" + + +class TestVerifierPrompt: + def test_builds_system_prompt(self): + verifier = Verifier(model="claude-sonnet-4-6", temperature=0.0) + prompt = verifier.build_system_prompt() + assert "criterion" in prompt.lower() + assert "evidence" in prompt.lower() + assert "JSON" in prompt diff --git a/evals/uv.lock b/evals/uv.lock new file mode 100644 index 00000000..ec90d0fe --- /dev/null +++ b/evals/uv.lock @@ -0,0 +1,650 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anthropic" +version = "0.93.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/70/2429d6f7c2516db99fb342c3ad89575ab3e0cd31d3d2f6cba5fdf5e9c65b/anthropic-0.93.0.tar.gz", hash = "sha256:fea8376f7d5cdf99d5e8e85a48fe7a7bd8ab307cdfee4b1e8283a18b1c0ce1b5", size = 654155, upload-time = "2026-04-09T18:13:53.522Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7b/5b2c11902707c49c7a99418eb027ed3eb63876193fee5c80b5c878e3a673/anthropic-0.93.0-py3-none-any.whl", hash = "sha256:2c20b2ce6d305564c66a6cbaedddee8efdd3b9753098bf314093fcf4c662d04c", size = 627482, upload-time = "2026-04-09T18:13:51.606Z" }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + +[[package]] +name = "certifi" +version = "2026.2.25" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, +] + +[[package]] +name = "click" +version = "8.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/20/71885d8b97d4f3dde17b1fdb92dbd4908b00541c5a3379787137285f602e/click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d", size = 108379, upload-time = "2026-04-03T19:14:43.505Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "docstring-parser" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/9d/c3b43da9515bd270df0f80548d9944e389870713cc1fe2b8fb35fe2bcefd/docstring_parser-0.17.0.tar.gz", hash = "sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912", size = 27442, upload-time = "2025-07-21T07:35:01.868Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/e2/2537ebcff11c1ee1ff17d8d0b6f4db75873e3b0fb32c2d4a2ee31ecb310a/docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708", size = 36896, upload-time = "2025-07-21T07:35:00.684Z" }, +] + +[[package]] +name = "drill" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "anthropic" }, + { name = "click" }, + { name = "jinja2" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] +requires-dist = [ + { name = "anthropic", specifier = ">=0.42" }, + { name = "click", specifier = ">=8.1" }, + { name = "jinja2", specifier = ">=3.1" }, + { name = "pydantic", specifier = ">=2.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "python-dotenv", specifier = ">=1.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.11" }, + { name = "ty", marker = "extra == 'dev'", specifier = ">=0.0.1a1" }, +] +provides-extras = ["dev"] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jiter" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/5e/4ec91646aee381d01cdb9974e30882c9cd3b8c5d1079d6b5ff4af522439a/jiter-0.13.0.tar.gz", hash = "sha256:f2839f9c2c7e2dffc1bc5929a510e14ce0a946be9365fd1219e7ef342dae14f4", size = 164847, upload-time = "2026-02-02T12:37:56.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/29/499f8c9eaa8a16751b1c0e45e6f5f1761d180da873d417996cc7bddc8eef/jiter-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ea026e70a9a28ebbdddcbcf0f1323128a8db66898a06eaad3a4e62d2f554d096", size = 311157, upload-time = "2026-02-02T12:35:37.758Z" }, + { url = "https://files.pythonhosted.org/packages/50/f6/566364c777d2ab450b92100bea11333c64c38d32caf8dc378b48e5b20c46/jiter-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66aa3e663840152d18cc8ff1e4faad3dd181373491b9cfdc6004b92198d67911", size = 319729, upload-time = "2026-02-02T12:35:39.246Z" }, + { url = "https://files.pythonhosted.org/packages/73/dd/560f13ec5e4f116d8ad2658781646cca91b617ae3b8758d4a5076b278f70/jiter-0.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3524798e70655ff19aec58c7d05adb1f074fecff62da857ea9be2b908b6d701", size = 354766, upload-time = "2026-02-02T12:35:40.662Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0d/061faffcfe94608cbc28a0d42a77a74222bdf5055ccdbe5fd2292b94f510/jiter-0.13.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ec7e287d7fbd02cb6e22f9a00dd9c9cd504c40a61f2c61e7e1f9690a82726b4c", size = 362587, upload-time = "2026-02-02T12:35:42.025Z" }, + { url = "https://files.pythonhosted.org/packages/92/c9/c66a7864982fd38a9773ec6e932e0398d1262677b8c60faecd02ffb67bf3/jiter-0.13.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47455245307e4debf2ce6c6e65a717550a0244231240dcf3b8f7d64e4c2f22f4", size = 487537, upload-time = "2026-02-02T12:35:43.459Z" }, + { url = "https://files.pythonhosted.org/packages/6c/86/84eb4352cd3668f16d1a88929b5888a3fe0418ea8c1dfc2ad4e7bf6e069a/jiter-0.13.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ee9da221dca6e0429c2704c1b3655fe7b025204a71d4d9b73390c759d776d165", size = 373717, upload-time = "2026-02-02T12:35:44.928Z" }, + { url = "https://files.pythonhosted.org/packages/6e/09/9fe4c159358176f82d4390407a03f506a8659ed13ca3ac93a843402acecf/jiter-0.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24ab43126d5e05f3d53a36a8e11eb2f23304c6c1117844aaaf9a0aa5e40b5018", size = 362683, upload-time = "2026-02-02T12:35:46.636Z" }, + { url = "https://files.pythonhosted.org/packages/c9/5e/85f3ab9caca0c1d0897937d378b4a515cae9e119730563572361ea0c48ae/jiter-0.13.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9da38b4fedde4fb528c740c2564628fbab737166a0e73d6d46cb4bb5463ff411", size = 392345, upload-time = "2026-02-02T12:35:48.088Z" }, + { url = "https://files.pythonhosted.org/packages/12/4c/05b8629ad546191939e6f0c2f17e29f542a398f4a52fb987bc70b6d1eb8b/jiter-0.13.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0b34c519e17658ed88d5047999a93547f8889f3c1824120c26ad6be5f27b6cf5", size = 517775, upload-time = "2026-02-02T12:35:49.482Z" }, + { url = "https://files.pythonhosted.org/packages/4d/88/367ea2eb6bc582c7052e4baf5ddf57ebe5ab924a88e0e09830dfb585c02d/jiter-0.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2a6394e6af690d462310a86b53c47ad75ac8c21dc79f120714ea449979cb1d3", size = 551325, upload-time = "2026-02-02T12:35:51.104Z" }, + { url = "https://files.pythonhosted.org/packages/f3/12/fa377ffb94a2f28c41afaed093e0d70cfe512035d5ecb0cad0ae4792d35e/jiter-0.13.0-cp311-cp311-win32.whl", hash = "sha256:0f0c065695f616a27c920a56ad0d4fc46415ef8b806bf8fc1cacf25002bd24e1", size = 204709, upload-time = "2026-02-02T12:35:52.467Z" }, + { url = "https://files.pythonhosted.org/packages/cb/16/8e8203ce92f844dfcd3d9d6a5a7322c77077248dbb12da52d23193a839cd/jiter-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:0733312953b909688ae3c2d58d043aa040f9f1a6a75693defed7bc2cc4bf2654", size = 204560, upload-time = "2026-02-02T12:35:53.925Z" }, + { url = "https://files.pythonhosted.org/packages/44/26/97cc40663deb17b9e13c3a5cf29251788c271b18ee4d262c8f94798b8336/jiter-0.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:5d9b34ad56761b3bf0fbe8f7e55468704107608512350962d3317ffd7a4382d5", size = 189608, upload-time = "2026-02-02T12:35:55.304Z" }, + { url = "https://files.pythonhosted.org/packages/2e/30/7687e4f87086829955013ca12a9233523349767f69653ebc27036313def9/jiter-0.13.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0a2bd69fc1d902e89925fc34d1da51b2128019423d7b339a45d9e99c894e0663", size = 307958, upload-time = "2026-02-02T12:35:57.165Z" }, + { url = "https://files.pythonhosted.org/packages/c3/27/e57f9a783246ed95481e6749cc5002a8a767a73177a83c63ea71f0528b90/jiter-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f917a04240ef31898182f76a332f508f2cc4b57d2b4d7ad2dbfebbfe167eb505", size = 318597, upload-time = "2026-02-02T12:35:58.591Z" }, + { url = "https://files.pythonhosted.org/packages/cf/52/e5719a60ac5d4d7c5995461a94ad5ef962a37c8bf5b088390e6fad59b2ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e2b199f446d3e82246b4fd9236d7cb502dc2222b18698ba0d986d2fecc6152", size = 348821, upload-time = "2026-02-02T12:36:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/61/db/c1efc32b8ba4c740ab3fc2d037d8753f67685f475e26b9d6536a4322bcdd/jiter-0.13.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04670992b576fa65bd056dbac0c39fe8bd67681c380cb2b48efa885711d9d726", size = 364163, upload-time = "2026-02-02T12:36:01.937Z" }, + { url = "https://files.pythonhosted.org/packages/55/8a/fb75556236047c8806995671a18e4a0ad646ed255276f51a20f32dceaeec/jiter-0.13.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a1aff1fbdb803a376d4d22a8f63f8e7ccbce0b4890c26cc7af9e501ab339ef0", size = 483709, upload-time = "2026-02-02T12:36:03.41Z" }, + { url = "https://files.pythonhosted.org/packages/7e/16/43512e6ee863875693a8e6f6d532e19d650779d6ba9a81593ae40a9088ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b3fb8c2053acaef8580809ac1d1f7481a0a0bdc012fd7f5d8b18fb696a5a089", size = 370480, upload-time = "2026-02-02T12:36:04.791Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4c/09b93e30e984a187bc8aaa3510e1ec8dcbdcd71ca05d2f56aac0492453aa/jiter-0.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdaba7d87e66f26a2c45d8cbadcbfc4bf7884182317907baf39cfe9775bb4d93", size = 360735, upload-time = "2026-02-02T12:36:06.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1b/46c5e349019874ec5dfa508c14c37e29864ea108d376ae26d90bee238cd7/jiter-0.13.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7b88d649135aca526da172e48083da915ec086b54e8e73a425ba50999468cc08", size = 391814, upload-time = "2026-02-02T12:36:08.368Z" }, + { url = "https://files.pythonhosted.org/packages/15/9e/26184760e85baee7162ad37b7912797d2077718476bf91517641c92b3639/jiter-0.13.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e404ea551d35438013c64b4f357b0474c7abf9f781c06d44fcaf7a14c69ff9e2", size = 513990, upload-time = "2026-02-02T12:36:09.993Z" }, + { url = "https://files.pythonhosted.org/packages/e9/34/2c9355247d6debad57a0a15e76ab1566ab799388042743656e566b3b7de1/jiter-0.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1f4748aad1b4a93c8bdd70f604d0f748cdc0e8744c5547798acfa52f10e79228", size = 548021, upload-time = "2026-02-02T12:36:11.376Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4a/9f2c23255d04a834398b9c2e0e665382116911dc4d06b795710503cdad25/jiter-0.13.0-cp312-cp312-win32.whl", hash = "sha256:0bf670e3b1445fc4d31612199f1744f67f889ee1bbae703c4b54dc097e5dd394", size = 203024, upload-time = "2026-02-02T12:36:12.682Z" }, + { url = "https://files.pythonhosted.org/packages/09/ee/f0ae675a957ae5a8f160be3e87acea6b11dc7b89f6b7ab057e77b2d2b13a/jiter-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:15db60e121e11fe186c0b15236bd5d18381b9ddacdcf4e659feb96fc6c969c92", size = 205424, upload-time = "2026-02-02T12:36:13.93Z" }, + { url = "https://files.pythonhosted.org/packages/1b/02/ae611edf913d3cbf02c97cdb90374af2082c48d7190d74c1111dde08bcdd/jiter-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:41f92313d17989102f3cb5dd533a02787cdb99454d494344b0361355da52fcb9", size = 186818, upload-time = "2026-02-02T12:36:15.308Z" }, + { url = "https://files.pythonhosted.org/packages/91/9c/7ee5a6ff4b9991e1a45263bfc46731634c4a2bde27dfda6c8251df2d958c/jiter-0.13.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1f8a55b848cbabf97d861495cd65f1e5c590246fabca8b48e1747c4dfc8f85bf", size = 306897, upload-time = "2026-02-02T12:36:16.748Z" }, + { url = "https://files.pythonhosted.org/packages/7c/02/be5b870d1d2be5dd6a91bdfb90f248fbb7dcbd21338f092c6b89817c3dbf/jiter-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f556aa591c00f2c45eb1b89f68f52441a016034d18b65da60e2d2875bbbf344a", size = 317507, upload-time = "2026-02-02T12:36:18.351Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/b25d2ec333615f5f284f3a4024f7ce68cfa0604c322c6808b2344c7f5d2b/jiter-0.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7e1d61da332ec412350463891923f960c3073cf1aae93b538f0bb4c8cd46efb", size = 350560, upload-time = "2026-02-02T12:36:19.746Z" }, + { url = "https://files.pythonhosted.org/packages/be/ec/74dcb99fef0aca9fbe56b303bf79f6bd839010cb18ad41000bf6cc71eec0/jiter-0.13.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3097d665a27bc96fd9bbf7f86178037db139f319f785e4757ce7ccbf390db6c2", size = 363232, upload-time = "2026-02-02T12:36:21.243Z" }, + { url = "https://files.pythonhosted.org/packages/1b/37/f17375e0bb2f6a812d4dd92d7616e41917f740f3e71343627da9db2824ce/jiter-0.13.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d01ecc3a8cbdb6f25a37bd500510550b64ddf9f7d64a107d92f3ccb25035d0f", size = 483727, upload-time = "2026-02-02T12:36:22.688Z" }, + { url = "https://files.pythonhosted.org/packages/77/d2/a71160a5ae1a1e66c1395b37ef77da67513b0adba73b993a27fbe47eb048/jiter-0.13.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed9bbc30f5d60a3bdf63ae76beb3f9db280d7f195dfcfa61af792d6ce912d159", size = 370799, upload-time = "2026-02-02T12:36:24.106Z" }, + { url = "https://files.pythonhosted.org/packages/01/99/ed5e478ff0eb4e8aa5fd998f9d69603c9fd3f32de3bd16c2b1194f68361c/jiter-0.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98fbafb6e88256f4454de33c1f40203d09fc33ed19162a68b3b257b29ca7f663", size = 359120, upload-time = "2026-02-02T12:36:25.519Z" }, + { url = "https://files.pythonhosted.org/packages/16/be/7ffd08203277a813f732ba897352797fa9493faf8dc7995b31f3d9cb9488/jiter-0.13.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5467696f6b827f1116556cb0db620440380434591e93ecee7fd14d1a491b6daa", size = 390664, upload-time = "2026-02-02T12:36:26.866Z" }, + { url = "https://files.pythonhosted.org/packages/d1/84/e0787856196d6d346264d6dcccb01f741e5f0bd014c1d9a2ebe149caf4f3/jiter-0.13.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:2d08c9475d48b92892583df9da592a0e2ac49bcd41fae1fec4f39ba6cf107820", size = 513543, upload-time = "2026-02-02T12:36:28.217Z" }, + { url = "https://files.pythonhosted.org/packages/65/50/ecbd258181c4313cf79bca6c88fb63207d04d5bf5e4f65174114d072aa55/jiter-0.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:aed40e099404721d7fcaf5b89bd3b4568a4666358bcac7b6b15c09fb6252ab68", size = 547262, upload-time = "2026-02-02T12:36:29.678Z" }, + { url = "https://files.pythonhosted.org/packages/27/da/68f38d12e7111d2016cd198161b36e1f042bd115c169255bcb7ec823a3bf/jiter-0.13.0-cp313-cp313-win32.whl", hash = "sha256:36ebfbcffafb146d0e6ffb3e74d51e03d9c35ce7c625c8066cdbfc7b953bdc72", size = 200630, upload-time = "2026-02-02T12:36:31.808Z" }, + { url = "https://files.pythonhosted.org/packages/25/65/3bd1a972c9a08ecd22eb3b08a95d1941ebe6938aea620c246cf426ae09c2/jiter-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:8d76029f077379374cf0dbc78dbe45b38dec4a2eb78b08b5194ce836b2517afc", size = 202602, upload-time = "2026-02-02T12:36:33.679Z" }, + { url = "https://files.pythonhosted.org/packages/15/fe/13bd3678a311aa67686bb303654792c48206a112068f8b0b21426eb6851e/jiter-0.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:bb7613e1a427cfcb6ea4544f9ac566b93d5bf67e0d48c787eca673ff9c9dff2b", size = 185939, upload-time = "2026-02-02T12:36:35.065Z" }, + { url = "https://files.pythonhosted.org/packages/49/19/a929ec002ad3228bc97ca01dbb14f7632fffdc84a95ec92ceaf4145688ae/jiter-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fa476ab5dd49f3bf3a168e05f89358c75a17608dbabb080ef65f96b27c19ab10", size = 316616, upload-time = "2026-02-02T12:36:36.579Z" }, + { url = "https://files.pythonhosted.org/packages/52/56/d19a9a194afa37c1728831e5fb81b7722c3de18a3109e8f282bfc23e587a/jiter-0.13.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade8cb6ff5632a62b7dbd4757d8c5573f7a2e9ae285d6b5b841707d8363205ef", size = 346850, upload-time = "2026-02-02T12:36:38.058Z" }, + { url = "https://files.pythonhosted.org/packages/36/4a/94e831c6bf287754a8a019cb966ed39ff8be6ab78cadecf08df3bb02d505/jiter-0.13.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9950290340acc1adaded363edd94baebcee7dabdfa8bee4790794cd5cfad2af6", size = 358551, upload-time = "2026-02-02T12:36:39.417Z" }, + { url = "https://files.pythonhosted.org/packages/a2/ec/a4c72c822695fa80e55d2b4142b73f0012035d9fcf90eccc56bc060db37c/jiter-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2b4972c6df33731aac0742b64fd0d18e0a69bc7d6e03108ce7d40c85fd9e3e6d", size = 201950, upload-time = "2026-02-02T12:36:40.791Z" }, + { url = "https://files.pythonhosted.org/packages/b6/00/393553ec27b824fbc29047e9c7cd4a3951d7fbe4a76743f17e44034fa4e4/jiter-0.13.0-cp313-cp313t-win_arm64.whl", hash = "sha256:701a1e77d1e593c1b435315ff625fd071f0998c5f02792038a5ca98899261b7d", size = 185852, upload-time = "2026-02-02T12:36:42.077Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f5/f1997e987211f6f9bd71b8083047b316208b4aca0b529bb5f8c96c89ef3e/jiter-0.13.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cc5223ab19fe25e2f0bf2643204ad7318896fe3729bf12fde41b77bfc4fafff0", size = 308804, upload-time = "2026-02-02T12:36:43.496Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8f/5482a7677731fd44881f0204981ce2d7175db271f82cba2085dd2212e095/jiter-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9776ebe51713acf438fd9b4405fcd86893ae5d03487546dae7f34993217f8a91", size = 318787, upload-time = "2026-02-02T12:36:45.071Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b9/7257ac59778f1cd025b26a23c5520a36a424f7f1b068f2442a5b499b7464/jiter-0.13.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:879e768938e7b49b5e90b7e3fecc0dbec01b8cb89595861fb39a8967c5220d09", size = 353880, upload-time = "2026-02-02T12:36:47.365Z" }, + { url = "https://files.pythonhosted.org/packages/c3/87/719eec4a3f0841dad99e3d3604ee4cba36af4419a76f3cb0b8e2e691ad67/jiter-0.13.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:682161a67adea11e3aae9038c06c8b4a9a71023228767477d683f69903ebc607", size = 366702, upload-time = "2026-02-02T12:36:48.871Z" }, + { url = "https://files.pythonhosted.org/packages/d2/65/415f0a75cf6921e43365a1bc227c565cb949caca8b7532776e430cbaa530/jiter-0.13.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a13b68cd1cd8cc9de8f244ebae18ccb3e4067ad205220ef324c39181e23bbf66", size = 486319, upload-time = "2026-02-02T12:36:53.006Z" }, + { url = "https://files.pythonhosted.org/packages/54/a2/9e12b48e82c6bbc6081fd81abf915e1443add1b13d8fc586e1d90bb02bb8/jiter-0.13.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87ce0f14c6c08892b610686ae8be350bf368467b6acd5085a5b65441e2bf36d2", size = 372289, upload-time = "2026-02-02T12:36:54.593Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c1/e4693f107a1789a239c759a432e9afc592366f04e901470c2af89cfd28e1/jiter-0.13.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c365005b05505a90d1c47856420980d0237adf82f70c4aff7aebd3c1cc143ad", size = 360165, upload-time = "2026-02-02T12:36:56.112Z" }, + { url = "https://files.pythonhosted.org/packages/17/08/91b9ea976c1c758240614bd88442681a87672eebc3d9a6dde476874e706b/jiter-0.13.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1317fdffd16f5873e46ce27d0e0f7f4f90f0cdf1d86bf6abeaea9f63ca2c401d", size = 389634, upload-time = "2026-02-02T12:36:57.495Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/58325ef99390d6d40427ed6005bf1ad54f2577866594bcf13ce55675f87d/jiter-0.13.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c05b450d37ba0c9e21c77fef1f205f56bcee2330bddca68d344baebfc55ae0df", size = 514933, upload-time = "2026-02-02T12:36:58.909Z" }, + { url = "https://files.pythonhosted.org/packages/5b/25/69f1120c7c395fd276c3996bb8adefa9c6b84c12bb7111e5c6ccdcd8526d/jiter-0.13.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:775e10de3849d0631a97c603f996f518159272db00fdda0a780f81752255ee9d", size = 548842, upload-time = "2026-02-02T12:37:00.433Z" }, + { url = "https://files.pythonhosted.org/packages/18/05/981c9669d86850c5fbb0d9e62bba144787f9fba84546ba43d624ee27ef29/jiter-0.13.0-cp314-cp314-win32.whl", hash = "sha256:632bf7c1d28421c00dd8bbb8a3bac5663e1f57d5cd5ed962bce3c73bf62608e6", size = 202108, upload-time = "2026-02-02T12:37:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/8d/96/cdcf54dd0b0341db7d25413229888a346c7130bd20820530905fdb65727b/jiter-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:f22ef501c3f87ede88f23f9b11e608581c14f04db59b6a801f354397ae13739f", size = 204027, upload-time = "2026-02-02T12:37:03.075Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f9/724bcaaab7a3cd727031fe4f6995cb86c4bd344909177c186699c8dec51a/jiter-0.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:07b75fe09a4ee8e0c606200622e571e44943f47254f95e2436c8bdcaceb36d7d", size = 187199, upload-time = "2026-02-02T12:37:04.414Z" }, + { url = "https://files.pythonhosted.org/packages/62/92/1661d8b9fd6a3d7a2d89831db26fe3c1509a287d83ad7838831c7b7a5c7e/jiter-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:964538479359059a35fb400e769295d4b315ae61e4105396d355a12f7fef09f0", size = 318423, upload-time = "2026-02-02T12:37:05.806Z" }, + { url = "https://files.pythonhosted.org/packages/4f/3b/f77d342a54d4ebcd128e520fc58ec2f5b30a423b0fd26acdfc0c6fef8e26/jiter-0.13.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e104da1db1c0991b3eaed391ccd650ae8d947eab1480c733e5a3fb28d4313e40", size = 351438, upload-time = "2026-02-02T12:37:07.189Z" }, + { url = "https://files.pythonhosted.org/packages/76/b3/ba9a69f0e4209bd3331470c723c2f5509e6f0482e416b612431a5061ed71/jiter-0.13.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e3a5f0cde8ff433b8e88e41aa40131455420fb3649a3c7abdda6145f8cb7202", size = 364774, upload-time = "2026-02-02T12:37:08.579Z" }, + { url = "https://files.pythonhosted.org/packages/b3/16/6cdb31fa342932602458dbb631bfbd47f601e03d2e4950740e0b2100b570/jiter-0.13.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:57aab48f40be1db920a582b30b116fe2435d184f77f0e4226f546794cedd9cf0", size = 487238, upload-time = "2026-02-02T12:37:10.066Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b1/956cc7abaca8d95c13aa8d6c9b3f3797241c246cd6e792934cc4c8b250d2/jiter-0.13.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7772115877c53f62beeb8fd853cab692dbc04374ef623b30f997959a4c0e7e95", size = 372892, upload-time = "2026-02-02T12:37:11.656Z" }, + { url = "https://files.pythonhosted.org/packages/26/c4/97ecde8b1e74f67b8598c57c6fccf6df86ea7861ed29da84629cdbba76c4/jiter-0.13.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1211427574b17b633cfceba5040de8081e5abf114f7a7602f73d2e16f9fdaa59", size = 360309, upload-time = "2026-02-02T12:37:13.244Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d7/eabe3cf46715854ccc80be2cd78dd4c36aedeb30751dbf85a1d08c14373c/jiter-0.13.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7beae3a3d3b5212d3a55d2961db3c292e02e302feb43fce6a3f7a31b90ea6dfe", size = 389607, upload-time = "2026-02-02T12:37:14.881Z" }, + { url = "https://files.pythonhosted.org/packages/df/2d/03963fc0804e6109b82decfb9974eb92df3797fe7222428cae12f8ccaa0c/jiter-0.13.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:e5562a0f0e90a6223b704163ea28e831bd3a9faa3512a711f031611e6b06c939", size = 514986, upload-time = "2026-02-02T12:37:16.326Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6c/8c83b45eb3eb1c1e18d841fe30b4b5bc5619d781267ca9bc03e005d8fd0a/jiter-0.13.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:6c26a424569a59140fb51160a56df13f438a2b0967365e987889186d5fc2f6f9", size = 548756, upload-time = "2026-02-02T12:37:17.736Z" }, + { url = "https://files.pythonhosted.org/packages/47/66/eea81dfff765ed66c68fd2ed8c96245109e13c896c2a5015c7839c92367e/jiter-0.13.0-cp314-cp314t-win32.whl", hash = "sha256:24dc96eca9f84da4131cdf87a95e6ce36765c3b156fc9ae33280873b1c32d5f6", size = 201196, upload-time = "2026-02-02T12:37:19.101Z" }, + { url = "https://files.pythonhosted.org/packages/ff/32/4ac9c7a76402f8f00d00842a7f6b83b284d0cf7c1e9d4227bc95aa6d17fa/jiter-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0a8d76c7524087272c8ae913f5d9d608bd839154b62c4322ef65723d2e5bb0b8", size = 204215, upload-time = "2026-02-02T12:37:20.495Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8e/7def204fea9f9be8b3c21a6f2dd6c020cf56c7d5ff753e0e23ed7f9ea57e/jiter-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2c26cf47e2cad140fa23b6d58d435a7c0161f5c514284802f25e87fddfe11024", size = 187152, upload-time = "2026-02-02T12:37:22.124Z" }, + { url = "https://files.pythonhosted.org/packages/79/b3/3c29819a27178d0e461a8571fb63c6ae38be6dc36b78b3ec2876bbd6a910/jiter-0.13.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b1cbfa133241d0e6bdab48dcdc2604e8ba81512f6bbd68ec3e8e1357dd3c316c", size = 307016, upload-time = "2026-02-02T12:37:42.755Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ae/60993e4b07b1ac5ebe46da7aa99fdbb802eb986c38d26e3883ac0125c4e0/jiter-0.13.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:db367d8be9fad6e8ebbac4a7578b7af562e506211036cba2c06c3b998603c3d2", size = 305024, upload-time = "2026-02-02T12:37:44.774Z" }, + { url = "https://files.pythonhosted.org/packages/77/fa/2227e590e9cf98803db2811f172b2d6460a21539ab73006f251c66f44b14/jiter-0.13.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45f6f8efb2f3b0603092401dc2df79fa89ccbc027aaba4174d2d4133ed661434", size = 339337, upload-time = "2026-02-02T12:37:46.668Z" }, + { url = "https://files.pythonhosted.org/packages/2d/92/015173281f7eb96c0ef580c997da8ef50870d4f7f4c9e03c845a1d62ae04/jiter-0.13.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:597245258e6ad085d064780abfb23a284d418d3e61c57362d9449c6c7317ee2d", size = 346395, upload-time = "2026-02-02T12:37:48.09Z" }, + { url = "https://files.pythonhosted.org/packages/80/60/e50fa45dd7e2eae049f0ce964663849e897300433921198aef94b6ffa23a/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:3d744a6061afba08dd7ae375dcde870cffb14429b7477e10f67e9e6d68772a0a", size = 305169, upload-time = "2026-02-02T12:37:50.376Z" }, + { url = "https://files.pythonhosted.org/packages/d2/73/a009f41c5eed71c49bec53036c4b33555afcdee70682a18c6f66e396c039/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:ff732bd0a0e778f43d5009840f20b935e79087b4dc65bd36f1cd0f9b04b8ff7f", size = 303808, upload-time = "2026-02-02T12:37:52.092Z" }, + { url = "https://files.pythonhosted.org/packages/c4/10/528b439290763bff3d939268085d03382471b442f212dca4ff5f12802d43/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab44b178f7981fcaea7e0a5df20e773c663d06ffda0198f1a524e91b2fde7e59", size = 337384, upload-time = "2026-02-02T12:37:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19", size = 343768, upload-time = "2026-02-02T12:37:55.055Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/8d/192f3d7103816158dfd5ea50d098ef2aec19194e6cbccd4b3485bdb2eb2d/ruff-0.15.11.tar.gz", hash = "sha256:f092b21708bf0e7437ce9ada249dfe688ff9a0954fc94abab05dcea7dcd29c33", size = 4637264, upload-time = "2026-04-16T18:46:26.58Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/1e/6aca3427f751295ab011828e15e9bf452200ac74484f1db4be0197b8170b/ruff-0.15.11-py3-none-linux_armv6l.whl", hash = "sha256:e927cfff503135c558eb581a0c9792264aae9507904eb27809cdcff2f2c847b7", size = 10607943, upload-time = "2026-04-16T18:46:05.967Z" }, + { url = "https://files.pythonhosted.org/packages/e7/26/1341c262e74f36d4e84f3d6f4df0ac68cd53331a66bfc5080daa17c84c0b/ruff-0.15.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7a1b5b2938d8f890b76084d4fa843604d787a912541eae85fd7e233398bbb73e", size = 10988592, upload-time = "2026-04-16T18:46:00.742Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/850b1d6ffa9564fbb6740429bad53df1094082fe515c8c1e74b6d8d05f18/ruff-0.15.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d4176f3d194afbdaee6e41b9ccb1a2c287dba8700047df474abfbe773825d1cb", size = 10338501, upload-time = "2026-04-16T18:46:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/f2/11/cc1284d3e298c45a817a6aadb6c3e1d70b45c9b36d8d9cce3387b495a03a/ruff-0.15.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b17c886fb88203ced3afe7f14e8d5ae96e9d2f4ccc0ee66aa19f2c2675a27e4", size = 10670693, upload-time = "2026-04-16T18:46:41.941Z" }, + { url = "https://files.pythonhosted.org/packages/ce/9e/f8288b034ab72b371513c13f9a41d9ba3effac54e24bfb467b007daee2ca/ruff-0.15.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:49fafa220220afe7758a487b048de4c8f9f767f37dfefad46b9dd06759d003eb", size = 10416177, upload-time = "2026-04-16T18:46:21.717Z" }, + { url = "https://files.pythonhosted.org/packages/85/71/504d79abfd3d92532ba6bbe3d1c19fada03e494332a59e37c7c2dabae427/ruff-0.15.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2ab8427e74a00d93b8bda1307b1e60970d40f304af38bccb218e056c220120d", size = 11221886, upload-time = "2026-04-16T18:46:15.086Z" }, + { url = "https://files.pythonhosted.org/packages/43/5a/947e6ab7a5ad603d65b474be15a4cbc6d29832db5d762cd142e4e3a74164/ruff-0.15.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:195072c0c8e1fc8f940652073df082e37a5d9cb43b4ab1e4d0566ab8977a13b7", size = 12075183, upload-time = "2026-04-16T18:46:07.944Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a1/0b7bb6268775fdd3a0818aee8efd8f5b4e231d24dd4d528ced2534023182/ruff-0.15.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a3a0996d486af3920dec930a2e7daed4847dfc12649b537a9335585ada163e9e", size = 11516575, upload-time = "2026-04-16T18:46:31.687Z" }, + { url = "https://files.pythonhosted.org/packages/30/c3/bb5168fc4d233cc06e95f482770d0f3c87945a0cd9f614b90ea8dc2f2833/ruff-0.15.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bef2cb556d509259f1fe440bb9cd33c756222cf0a7afe90d15edf0866702431", size = 11306537, upload-time = "2026-04-16T18:46:36.988Z" }, + { url = "https://files.pythonhosted.org/packages/e4/92/4cfae6441f3967317946f3b788136eecf093729b94d6561f963ed810c82e/ruff-0.15.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:030d921a836d7d4a12cf6e8d984a88b66094ccb0e0f17ddd55067c331191bf19", size = 11296813, upload-time = "2026-04-16T18:46:24.182Z" }, + { url = "https://files.pythonhosted.org/packages/43/26/972784c5dde8313acde8ac71ba8ac65475b85db4a2352a76c9934361f9bc/ruff-0.15.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0e783b599b4577788dbbb66b9addcef87e9a8832f4ce0c19e34bf55543a2f890", size = 10633136, upload-time = "2026-04-16T18:46:39.802Z" }, + { url = "https://files.pythonhosted.org/packages/5b/53/3985a4f185020c2f367f2e08a103032e12564829742a1b417980ce1514a0/ruff-0.15.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ae90592246625ba4a34349d68ec28d4400d75182b71baa196ddb9f82db025ef5", size = 10424701, upload-time = "2026-04-16T18:46:10.381Z" }, + { url = "https://files.pythonhosted.org/packages/d3/57/bf0dfb32241b56c83bb663a826133da4bf17f682ba8c096973065f6e6a68/ruff-0.15.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1f111d62e3c983ed20e0ca2e800f8d77433a5b1161947df99a5c2a3fb60514f0", size = 10873887, upload-time = "2026-04-16T18:46:29.157Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/e48076b2a57dc33ee8c7a957296f97c744ca891a8ffb4ffb1aaa3b3f517d/ruff-0.15.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:06f483d6646f59eaffba9ae30956370d3a886625f511a3108994000480621d1c", size = 11404316, upload-time = "2026-04-16T18:46:19.462Z" }, + { url = "https://files.pythonhosted.org/packages/88/27/0195d15fe7a897cbcba0904792c4b7c9fdd958456c3a17d2ea6093716a9a/ruff-0.15.11-py3-none-win32.whl", hash = "sha256:476a2aa56b7da0b73a3ee80b6b2f0e19cce544245479adde7baa65466664d5f3", size = 10655535, upload-time = "2026-04-16T18:46:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5e/c927b325bd4c1d3620211a4b96f47864633199feed60fa936025ab27e090/ruff-0.15.11-py3-none-win_amd64.whl", hash = "sha256:8b6756d88d7e234fb0c98c91511aae3cd519d5e3ed271cae31b20f39cb2a12a3", size = 11779692, upload-time = "2026-04-16T18:46:17.268Z" }, + { url = "https://files.pythonhosted.org/packages/63/b6/aeadee5443e49baa2facd51131159fd6301cc4ccfc1541e4df7b021c37dd/ruff-0.15.11-py3-none-win_arm64.whl", hash = "sha256:063fed18cc1bbe0ee7393957284a6fe8b588c6a406a285af3ee3f46da2391ee4", size = 11032614, upload-time = "2026-04-16T18:46:34.487Z" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "ty" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/7e/2aa791c9ae7b8cd5024cd4122e92267f664ca954cea3def3211919fa3c1f/ty-0.0.32.tar.gz", hash = "sha256:8743174c5f920f6700a4a0c9de140109189192ba16226884cd50095b43b8a45c", size = 5522294, upload-time = "2026-04-20T19:29:01.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/eb/1075dc6a49d7acbe2584ae4d5b410c41b1f177a5adcc567e09eca4c69000/ty-0.0.32-py3-none-linux_armv6l.whl", hash = "sha256:dacbc2f6cd698d488ae7436838ff929570455bf94bfa4d9fe57a630c552aff83", size = 10902959, upload-time = "2026-04-20T19:28:31.907Z" }, + { url = "https://files.pythonhosted.org/packages/33/d2/c35fc8bc66e98d1ee9b0f8ed319bf743e450e1f1e997574b178fab75670f/ty-0.0.32-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:914bbc4f605ce2a9e2a78982e28fae1d3359a169d141f9dc3b4c7749cd5eca81", size = 10726172, upload-time = "2026-04-20T19:28:44.765Z" }, + { url = "https://files.pythonhosted.org/packages/96/32/c827da3ca480456fb02d8cea68a2609273b6c220fea0be9a4c8d8470b86e/ty-0.0.32-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4787ac9fe1f86b1f3133f5c6732adbe2df5668b50c679ac6e2d98cd284da812f", size = 10163701, upload-time = "2026-04-20T19:28:27.005Z" }, + { url = "https://files.pythonhosted.org/packages/ba/9e/2734478fbdb90c160cb2813a3916a16a2af5c1e231f87d635f6131d781fb/ty-0.0.32-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8ea0a728af99fe40dd744cba6441a2404f80b7f4bde17aa6da393810af5ea57", size = 10656220, upload-time = "2026-04-20T19:29:03.814Z" }, + { url = "https://files.pythonhosted.org/packages/44/9f/0007da2d35e424debe7e9f86ffbc1ab7f60983cfbc5f0411324ab2de5292/ty-0.0.32-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2850561f9b018ae33d7e5bbfa0ac414d3c518513edcffe43877dc9801446b9c5", size = 10696086, upload-time = "2026-04-20T19:28:46.829Z" }, + { url = "https://files.pythonhosted.org/packages/3b/5e/ce5fd4ec803222ae3e69a76d2a2db2eed55e19f5b131702b9789ef45f93d/ty-0.0.32-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b5fa2fb3c614349ee211d36476b49d88c5ef79a687cdb91b2872ad023b94d2f8", size = 11184800, upload-time = "2026-04-20T19:28:42.57Z" }, + { url = "https://files.pythonhosted.org/packages/6c/46/ebcf67a5999421331214aac51a7464db42de2be15bbe929c612a3ed0b039/ty-0.0.32-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b89969307ab2417d41c9be8059dd79feea577234e1e10d35132f5495e0d42c6", size = 11718718, upload-time = "2026-04-20T19:28:36.433Z" }, + { url = "https://files.pythonhosted.org/packages/18/2c/2141c86ed0ce0962b45cefb658a95e734f59759d47f20afdcd9c732910a1/ty-0.0.32-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b59868ede9b1d69a088f0d695df52a0061f95fa7baa1d5e0dc6fc9cf06e1334", size = 11346369, upload-time = "2026-04-20T19:28:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/7a/da/ed6f772339cf29bd9a46def9d6db5084689eb574ee4d150ff704224c1ed8/ty-0.0.32-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8300caf35345498e9b9b03e550bba03cee8f5f5f8ab4c83c3b1ff1b7403b7d3a", size = 11280714, upload-time = "2026-04-20T19:28:51.516Z" }, + { url = "https://files.pythonhosted.org/packages/da/9b/c6813987edf4816a40e0c8e408b555f97d3f267c7b3a1688c8bbdf65609c/ty-0.0.32-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:583c7094f4574b02f724db924f98b804d1387a0bd9405ecb5e078cc0f47fbcfb", size = 10638806, upload-time = "2026-04-20T19:28:29.651Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d4/0cefcbd2ad0f3d51762ccf58e652ec7da146eb6ae34f87228f6254bbb8be/ty-0.0.32-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e44ebe1bb4143a5628bc4db67ac0dfebe14594af671e4ee66f6f2e983da56501", size = 10726106, upload-time = "2026-04-20T19:29:06.3Z" }, + { url = "https://files.pythonhosted.org/packages/32/ad/2c8a97f91f06311f4367400f7d13534bbda2522c73c99a3e4c0757dff9b8/ty-0.0.32-py3-none-musllinux_1_2_i686.whl", hash = "sha256:06f17ada3e069cba6148342ef88e9929156beca8473e8d4f101b68f66c75643e", size = 10872951, upload-time = "2026-04-20T19:28:34.077Z" }, + { url = "https://files.pythonhosted.org/packages/ba/68/42293f9248106dd51875120971a5cc6ea315c2c4dcfb8e59aa063aa0af26/ty-0.0.32-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e96e60fa556cec04f15d7ea62d2ceee5982bd389233e961ab9fd42304e278175", size = 11363334, upload-time = "2026-04-20T19:28:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/df/92/be9abf4d3e589ad5023e2ea965b93e204ec856420d46adf73c5c36c04678/ty-0.0.32-py3-none-win32.whl", hash = "sha256:2ff2ebb4986b24aebcf1444db7db5ca41b36086040e95eea9f8fb851c11e805c", size = 10260689, upload-time = "2026-04-20T19:28:56.541Z" }, + { url = "https://files.pythonhosted.org/packages/14/61/dc86acea899349d2579cb8419aecedd83dc504d7d6a10df65eef546c8300/ty-0.0.32-py3-none-win_amd64.whl", hash = "sha256:ba7284a4a954b598c1b31500352b3ec1f89bff533825592b5958848226fdc7ee", size = 11255371, upload-time = "2026-04-20T19:28:39.917Z" }, + { url = "https://files.pythonhosted.org/packages/43/01/beffec56d71ca25b343ede63adb076456b5b3e211f1c066452a44cd120b3/ty-0.0.32-py3-none-win_arm64.whl", hash = "sha256:7e10aadbdbda989a7d567ee6a37f8b98d4d542e31e3b190a2879fd581f75d658", size = 10658087, upload-time = "2026-04-20T19:28:59.286Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +]