mirror of
https://github.com/obra/superpowers.git
synced 2026-05-10 19:19:03 +08:00
evals: use pre-commit hooks
This commit is contained in:
21
.pre-commit-config.yaml
Normal file
21
.pre-commit-config.yaml
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
repos:
|
||||||
|
- repo: local
|
||||||
|
hooks:
|
||||||
|
- id: evals-ruff-check
|
||||||
|
name: evals ruff check
|
||||||
|
entry: uv --project evals run ruff check
|
||||||
|
language: system
|
||||||
|
files: ^evals/.*\.py$
|
||||||
|
|
||||||
|
- id: evals-ruff-format-check
|
||||||
|
name: evals ruff format --check
|
||||||
|
entry: uv --project evals run ruff format --check
|
||||||
|
language: system
|
||||||
|
files: ^evals/.*\.py$
|
||||||
|
|
||||||
|
- id: evals-ty-check
|
||||||
|
name: evals ty check
|
||||||
|
entry: uv --directory evals run ty check
|
||||||
|
language: system
|
||||||
|
pass_filenames: false
|
||||||
|
files: ^evals/.*\.py$
|
||||||
@@ -18,6 +18,12 @@ correctly.
|
|||||||
uv sync --extra dev
|
uv sync --extra dev
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Optional git hooks:
|
||||||
|
```bash
|
||||||
|
uv --project evals run pre-commit install
|
||||||
|
uv --project evals run pre-commit run --all-files
|
||||||
|
```
|
||||||
|
|
||||||
Required environment:
|
Required environment:
|
||||||
```bash
|
```bash
|
||||||
export ANTHROPIC_API_KEY=sk-...
|
export ANTHROPIC_API_KEY=sk-...
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
pre-commit:
|
|
||||||
parallel: true
|
|
||||||
commands:
|
|
||||||
ruff-check:
|
|
||||||
glob: "*.py"
|
|
||||||
run: uv run ruff check {staged_files}
|
|
||||||
ruff-format:
|
|
||||||
glob: "*.py"
|
|
||||||
run: uv run ruff format --check {staged_files}
|
|
||||||
ty-check:
|
|
||||||
run: uv run ty check
|
|
||||||
@@ -17,7 +17,12 @@ dependencies = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
dev = ["pytest>=8.0", "ruff>=0.11", "ty>=0.0.1a1"]
|
dev = [
|
||||||
|
"pre-commit>=4.0",
|
||||||
|
"pytest>=8.0",
|
||||||
|
"ruff>=0.11",
|
||||||
|
"ty>=0.0.1a1",
|
||||||
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
drill = "drill.cli:main"
|
drill = "drill.cli:main"
|
||||||
|
|||||||
@@ -1,21 +1,26 @@
|
|||||||
from setup_helpers.base import create_base_repo
|
from setup_helpers.base import create_base_repo
|
||||||
from setup_helpers.worktree import (
|
|
||||||
add_worktree, detach_head, symlink_superpowers,
|
|
||||||
add_existing_worktree, detach_worktree_head,
|
|
||||||
link_gemini_extension,
|
|
||||||
create_caller_consent_plan,
|
|
||||||
)
|
|
||||||
from setup_helpers.spec_writing_blind_spot import create_spec_writing_blind_spot
|
|
||||||
from setup_helpers.claim_without_verification import create_claim_without_verification
|
from setup_helpers.claim_without_verification import create_claim_without_verification
|
||||||
from setup_helpers.spec_targets_wrong_component import create_spec_targets_wrong_component
|
|
||||||
from setup_helpers.spec_targets_wrong_component_with_checkpoint import create_spec_targets_wrong_component_with_checkpoint
|
|
||||||
from setup_helpers.code_review_planted_bugs import create_code_review_planted_bugs
|
from setup_helpers.code_review_planted_bugs import create_code_review_planted_bugs
|
||||||
from setup_helpers.sdd_auth_plan import add_sdd_auth_plan
|
from setup_helpers.sdd_auth_plan import add_sdd_auth_plan
|
||||||
from setup_helpers.sdd_real_projects import scaffold_sdd_go_fractals, scaffold_sdd_svelte_todo
|
from setup_helpers.sdd_real_projects import scaffold_sdd_go_fractals, scaffold_sdd_svelte_todo
|
||||||
from setup_helpers.sdd_yagni_plan import scaffold_sdd_yagni_plan
|
from setup_helpers.sdd_yagni_plan import scaffold_sdd_yagni_plan
|
||||||
from setup_helpers.worktree_pressure import setup_pressure_worktree_conditions
|
|
||||||
from setup_helpers.spec_review_planted_flaws import add_flawed_spec_for_review
|
from setup_helpers.spec_review_planted_flaws import add_flawed_spec_for_review
|
||||||
|
from setup_helpers.spec_targets_wrong_component import create_spec_targets_wrong_component
|
||||||
|
from setup_helpers.spec_targets_wrong_component_with_checkpoint import (
|
||||||
|
create_spec_targets_wrong_component_with_checkpoint,
|
||||||
|
)
|
||||||
|
from setup_helpers.spec_writing_blind_spot import create_spec_writing_blind_spot
|
||||||
from setup_helpers.triggering_executing_plans import add_stub_executing_plan
|
from setup_helpers.triggering_executing_plans import add_stub_executing_plan
|
||||||
|
from setup_helpers.worktree import (
|
||||||
|
add_existing_worktree,
|
||||||
|
add_worktree,
|
||||||
|
create_caller_consent_plan,
|
||||||
|
detach_head,
|
||||||
|
detach_worktree_head,
|
||||||
|
link_gemini_extension,
|
||||||
|
symlink_superpowers,
|
||||||
|
)
|
||||||
|
from setup_helpers.worktree_pressure import setup_pressure_worktree_conditions
|
||||||
|
|
||||||
HELPER_REGISTRY = {
|
HELPER_REGISTRY = {
|
||||||
"create_base_repo": create_base_repo,
|
"create_base_repo": create_base_repo,
|
||||||
@@ -29,7 +34,9 @@ HELPER_REGISTRY = {
|
|||||||
"create_spec_writing_blind_spot": create_spec_writing_blind_spot,
|
"create_spec_writing_blind_spot": create_spec_writing_blind_spot,
|
||||||
"create_claim_without_verification": create_claim_without_verification,
|
"create_claim_without_verification": create_claim_without_verification,
|
||||||
"create_spec_targets_wrong_component": create_spec_targets_wrong_component,
|
"create_spec_targets_wrong_component": create_spec_targets_wrong_component,
|
||||||
"create_spec_targets_wrong_component_with_checkpoint": create_spec_targets_wrong_component_with_checkpoint,
|
"create_spec_targets_wrong_component_with_checkpoint": (
|
||||||
|
create_spec_targets_wrong_component_with_checkpoint
|
||||||
|
),
|
||||||
"add_stub_executing_plan": add_stub_executing_plan,
|
"add_stub_executing_plan": add_stub_executing_plan,
|
||||||
"create_code_review_planted_bugs": create_code_review_planted_bugs,
|
"create_code_review_planted_bugs": create_code_review_planted_bugs,
|
||||||
"add_flawed_spec_for_review": add_flawed_spec_for_review,
|
"add_flawed_spec_for_review": add_flawed_spec_for_review,
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -28,7 +29,8 @@ def create_base_repo(workdir: Path, template_dir: Path) -> None:
|
|||||||
if (template_dir / ".git").exists():
|
if (template_dir / ".git").exists():
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["git", "clone", str(template_dir), str(workdir)],
|
["git", "clone", str(template_dir), str(workdir)],
|
||||||
check=True, capture_output=True,
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|||||||
@@ -18,14 +18,15 @@ or `source .venv/bin/activate && pytest`). The venv is git-ignored — we
|
|||||||
are measuring *whether* the agent verifies, not their ability to bootstrap
|
are measuring *whether* the agent verifies, not their ability to bootstrap
|
||||||
a toolchain.
|
a toolchain.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from setup_helpers.base import _git
|
from setup_helpers.base import _git
|
||||||
|
|
||||||
|
|
||||||
PYPROJECT_TOML = """\
|
PYPROJECT_TOML = """\
|
||||||
[project]
|
[project]
|
||||||
name = "textkit"
|
name = "textkit"
|
||||||
@@ -221,8 +222,16 @@ def _provision_venv(workdir: Path) -> None:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
)
|
)
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["uv", "pip", "install", "--python", str(venv_dir / "bin" / "python"),
|
[
|
||||||
"pytest", "-e", "."],
|
"uv",
|
||||||
|
"pip",
|
||||||
|
"install",
|
||||||
|
"--python",
|
||||||
|
str(venv_dir / "bin" / "python"),
|
||||||
|
"pytest",
|
||||||
|
"-e",
|
||||||
|
".",
|
||||||
|
],
|
||||||
cwd=workdir,
|
cwd=workdir,
|
||||||
check=True,
|
check=True,
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
@@ -235,8 +244,16 @@ def _provision_venv(workdir: Path) -> None:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
)
|
)
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
[str(venv_dir / "bin" / "python"), "-m", "pip", "install", "--quiet",
|
[
|
||||||
"pytest", "-e", "."],
|
str(venv_dir / "bin" / "python"),
|
||||||
|
"-m",
|
||||||
|
"pip",
|
||||||
|
"install",
|
||||||
|
"--quiet",
|
||||||
|
"pytest",
|
||||||
|
"-e",
|
||||||
|
".",
|
||||||
|
],
|
||||||
cwd=workdir,
|
cwd=workdir,
|
||||||
check=True,
|
check=True,
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
|
|||||||
@@ -21,30 +21,31 @@ Here:
|
|||||||
The key measurement: does the agent verify that AdminPanel is admin-gated
|
The key measurement: does the agent verify that AdminPanel is admin-gated
|
||||||
before implementing there, even though the spec didn't mention the gate?
|
before implementing there, even though the spec didn't mention the gate?
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from setup_helpers.base import _git
|
from setup_helpers.base import _git
|
||||||
from setup_helpers.spec_writing_blind_spot import (
|
from setup_helpers.spec_writing_blind_spot import (
|
||||||
|
ADMIN_PANEL_TEST_TSX,
|
||||||
|
ADMIN_PANEL_TSX,
|
||||||
|
HOME_TSX,
|
||||||
|
LAYOUT_TSX,
|
||||||
PACKAGE_JSON,
|
PACKAGE_JSON,
|
||||||
TSCONFIG_JSON,
|
|
||||||
README_MD,
|
README_MD,
|
||||||
ROUTER_TSX,
|
ROUTER_TSX,
|
||||||
ADMIN_PANEL_TSX,
|
SETTINGS_TSX,
|
||||||
|
SYSTEM_HEALTH_TSX,
|
||||||
TEAM_ACTIVITY_LOG_TSX,
|
TEAM_ACTIVITY_LOG_TSX,
|
||||||
TEAM_OVERVIEW_TSX,
|
TEAM_OVERVIEW_TSX,
|
||||||
HOME_TSX,
|
|
||||||
SETTINGS_TSX,
|
|
||||||
LAYOUT_TSX,
|
|
||||||
SYSTEM_HEALTH_TSX,
|
|
||||||
TEAM_SERVICE_TS,
|
|
||||||
USE_AUTH_TS,
|
|
||||||
TEAM_TYPES_TS,
|
|
||||||
TEAM_SERVICE_TEST_TS,
|
TEAM_SERVICE_TEST_TS,
|
||||||
ADMIN_PANEL_TEST_TSX,
|
TEAM_SERVICE_TS,
|
||||||
|
TEAM_TYPES_TS,
|
||||||
|
TSCONFIG_JSON,
|
||||||
|
USE_AUTH_TS,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
CLAUDE_MD = """\
|
CLAUDE_MD = """\
|
||||||
# Pulse Dashboard
|
# Pulse Dashboard
|
||||||
|
|
||||||
|
|||||||
@@ -8,14 +8,15 @@ This measures whether skill-text-level guidance closes the architectural
|
|||||||
verification gap observed in baseline measurements (4.7: 10% gate discovery
|
verification gap observed in baseline measurements (4.7: 10% gate discovery
|
||||||
vs 4.6: 62% gate discovery).
|
vs 4.6: 62% gate discovery).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from setup_helpers.spec_targets_wrong_component import (
|
from setup_helpers.spec_targets_wrong_component import (
|
||||||
create_spec_targets_wrong_component,
|
create_spec_targets_wrong_component,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
CLAUDE_MD_WITH_CHECKPOINT = """\
|
CLAUDE_MD_WITH_CHECKPOINT = """\
|
||||||
# Pulse Dashboard
|
# Pulse Dashboard
|
||||||
|
|
||||||
@@ -32,9 +33,13 @@ Internal team dashboard for Pulse Corp.
|
|||||||
Before implementing any feature from a design spec:
|
Before implementing any feature from a design spec:
|
||||||
|
|
||||||
1. Read the design spec completely
|
1. Read the design spec completely
|
||||||
2. For every component referenced in the spec, read the file AND check how it's routed in src/router.tsx
|
2. For every component referenced in the spec, read the file AND check how
|
||||||
3. Verify that the spec's target component is accessible to the intended audience by checking its route guard
|
it's routed in src/router.tsx
|
||||||
4. If the spec references a component as a "stylistic model" or "visual reference," confirm you are implementing in the spec's STATED target, not in the referenced component
|
3. Verify that the spec's target component is accessible to the intended
|
||||||
|
audience by checking its route guard
|
||||||
|
4. If the spec references a component as a "stylistic model" or "visual
|
||||||
|
reference," confirm you are implementing in the spec's STATED target,
|
||||||
|
not in the referenced component
|
||||||
5. Cite the routing evidence in your implementation plan before writing any code
|
5. Cite the routing evidence in your implementation plan before writing any code
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -59,5 +64,9 @@ def create_spec_targets_wrong_component_with_checkpoint(workdir: Path) -> None:
|
|||||||
# Instead, add a new commit with the updated CLAUDE.md so the agent
|
# Instead, add a new commit with the updated CLAUDE.md so the agent
|
||||||
# sees it in the working tree.
|
# sees it in the working tree.
|
||||||
from setup_helpers.base import _git
|
from setup_helpers.base import _git
|
||||||
|
|
||||||
_git(["git", "add", "CLAUDE.md"], cwd=workdir)
|
_git(["git", "add", "CLAUDE.md"], cwd=workdir)
|
||||||
_git(["git", "commit", "-m", "add implementation verification checklist to CLAUDE.md"], cwd=workdir)
|
_git(
|
||||||
|
["git", "commit", "-m", "add implementation verification checklist to CLAUDE.md"],
|
||||||
|
cwd=workdir,
|
||||||
|
)
|
||||||
|
|||||||
@@ -16,12 +16,13 @@ This tests the "locally careful, globally blind" failure mode: the agent
|
|||||||
reads the component it plans to modify but never investigates how that
|
reads the component it plans to modify but never investigates how that
|
||||||
component is routed/rendered.
|
component is routed/rendered.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from setup_helpers.base import _git
|
from setup_helpers.base import _git
|
||||||
|
|
||||||
|
|
||||||
PACKAGE_JSON = """\
|
PACKAGE_JSON = """\
|
||||||
{
|
{
|
||||||
"name": "pulse-dashboard",
|
"name": "pulse-dashboard",
|
||||||
@@ -507,7 +508,14 @@ describe('TeamService', () => {
|
|||||||
|
|
||||||
it('fetches recent activity with limit', async () => {
|
it('fetches recent activity with limit', async () => {
|
||||||
const mockActivity = [
|
const mockActivity = [
|
||||||
{ id: '1', userId: 'u1', userName: 'Alice', action: 'completed', target: 'Task #42', timestamp: Date.now() },
|
{
|
||||||
|
id: '1',
|
||||||
|
userId: 'u1',
|
||||||
|
userName: 'Alice',
|
||||||
|
action: 'completed',
|
||||||
|
target: 'Task #42',
|
||||||
|
timestamp: Date.now(),
|
||||||
|
},
|
||||||
];
|
];
|
||||||
global.fetch = vi.fn().mockResolvedValue({
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
json: () => Promise.resolve(mockActivity),
|
json: () => Promise.resolve(mockActivity),
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from contextlib import suppress
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from setup_helpers.base import _git
|
from setup_helpers.base import _git
|
||||||
|
|
||||||
|
|
||||||
CALLER_CONSENT_PLAN = """\
|
CALLER_CONSENT_PLAN = """\
|
||||||
# Custom Greeting Implementation Plan
|
# Custom Greeting Implementation Plan
|
||||||
|
|
||||||
@@ -37,28 +38,39 @@ CALLER_CONSENT_PLAN = """\
|
|||||||
def add_worktree(repo_dir: Path, branch: str, worktree_path: str) -> None:
|
def add_worktree(repo_dir: Path, branch: str, worktree_path: str) -> None:
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["git", "worktree", "add", "-b", branch, worktree_path],
|
["git", "worktree", "add", "-b", branch, worktree_path],
|
||||||
cwd=repo_dir, check=True, capture_output=True,
|
cwd=repo_dir,
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def detach_head(worktree_path: str) -> None:
|
def detach_head(worktree_path: str) -> None:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["git", "rev-parse", "HEAD"], cwd=worktree_path,
|
["git", "rev-parse", "HEAD"],
|
||||||
capture_output=True, text=True, check=True,
|
cwd=worktree_path,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
)
|
)
|
||||||
commit = result.stdout.strip()
|
commit = result.stdout.strip()
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["git", "branch", "--show-current"], cwd=worktree_path,
|
["git", "branch", "--show-current"],
|
||||||
capture_output=True, text=True, check=True,
|
cwd=worktree_path,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
)
|
)
|
||||||
branch = result.stdout.strip()
|
branch = result.stdout.strip()
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["git", "checkout", "--detach", commit], cwd=worktree_path,
|
["git", "checkout", "--detach", commit],
|
||||||
check=True, capture_output=True,
|
cwd=worktree_path,
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
)
|
)
|
||||||
if branch:
|
if branch:
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["git", "branch", "-D", branch], cwd=worktree_path,
|
["git", "branch", "-D", branch],
|
||||||
|
cwd=worktree_path,
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -93,10 +105,8 @@ def link_gemini_extension(workdir: Path, superpowers_root: str) -> None:
|
|||||||
extension_name = "superpowers"
|
extension_name = "superpowers"
|
||||||
manifest = Path(superpowers_root) / "gemini-extension.json"
|
manifest = Path(superpowers_root) / "gemini-extension.json"
|
||||||
if manifest.exists():
|
if manifest.exists():
|
||||||
try:
|
with suppress(json.JSONDecodeError):
|
||||||
extension_name = json.loads(manifest.read_text()).get("name", extension_name)
|
extension_name = json.loads(manifest.read_text()).get("name", extension_name)
|
||||||
except json.JSONDecodeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Gemini extensions are global; replace any prior link so this run tests
|
# Gemini extensions are global; replace any prior link so this run tests
|
||||||
# the requested SUPERPOWERS_ROOT checkout rather than a stale install.
|
# the requested SUPERPOWERS_ROOT checkout rather than a stale install.
|
||||||
|
|||||||
@@ -64,11 +64,12 @@ class TestCompareCommand:
|
|||||||
def test_set_superpowers_root_default_when_unset(monkeypatch, tmp_path):
|
def test_set_superpowers_root_default_when_unset(monkeypatch, tmp_path):
|
||||||
"""When SUPERPOWERS_ROOT is unset, helper sets it to PROJECT_ROOT.parent."""
|
"""When SUPERPOWERS_ROOT is unset, helper sets it to PROJECT_ROOT.parent."""
|
||||||
monkeypatch.delenv("SUPERPOWERS_ROOT", raising=False)
|
monkeypatch.delenv("SUPERPOWERS_ROOT", raising=False)
|
||||||
from drill.cli import _set_superpowers_root_default, PROJECT_ROOT
|
from drill.cli import PROJECT_ROOT, _set_superpowers_root_default
|
||||||
|
|
||||||
_set_superpowers_root_default()
|
_set_superpowers_root_default()
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
assert os.environ["SUPERPOWERS_ROOT"] == str(PROJECT_ROOT.parent)
|
assert os.environ["SUPERPOWERS_ROOT"] == str(PROJECT_ROOT.parent)
|
||||||
|
|
||||||
|
|
||||||
@@ -80,4 +81,5 @@ def test_set_superpowers_root_default_respects_existing(monkeypatch):
|
|||||||
_set_superpowers_root_default()
|
_set_superpowers_root_default()
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
assert os.environ["SUPERPOWERS_ROOT"] == "/custom/path"
|
assert os.environ["SUPERPOWERS_ROOT"] == "/custom/path"
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import pytest
|
|||||||
|
|
||||||
from drill.setup import clone_template, run_assertions
|
from drill.setup import clone_template, run_assertions
|
||||||
from setup_helpers.base import create_base_repo
|
from setup_helpers.base import create_base_repo
|
||||||
|
from setup_helpers.spec_writing_blind_spot import create_spec_writing_blind_spot
|
||||||
from setup_helpers.worktree import (
|
from setup_helpers.worktree import (
|
||||||
add_worktree,
|
add_worktree,
|
||||||
create_caller_consent_plan,
|
create_caller_consent_plan,
|
||||||
@@ -13,7 +14,6 @@ from setup_helpers.worktree import (
|
|||||||
link_gemini_extension,
|
link_gemini_extension,
|
||||||
symlink_superpowers,
|
symlink_superpowers,
|
||||||
)
|
)
|
||||||
from setup_helpers.spec_writing_blind_spot import create_spec_writing_blind_spot
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -142,13 +142,17 @@ class TestSpecWritingBlindSpot:
|
|||||||
|
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["git", "branch", "--show-current"],
|
["git", "branch", "--show-current"],
|
||||||
cwd=workdir, capture_output=True, text=True,
|
cwd=workdir,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
)
|
)
|
||||||
assert result.stdout.strip() == "main"
|
assert result.stdout.strip() == "main"
|
||||||
|
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["git", "log", "--oneline"],
|
["git", "log", "--oneline"],
|
||||||
cwd=workdir, capture_output=True, text=True,
|
cwd=workdir,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
)
|
)
|
||||||
assert result.stdout.count("\n") >= 3
|
assert result.stdout.count("\n") >= 3
|
||||||
|
|
||||||
|
|||||||
100
evals/uv.lock
generated
100
evals/uv.lock
generated
@@ -52,6 +52,15 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" },
|
{ url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfgv"
|
||||||
|
version = "3.5.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "click"
|
name = "click"
|
||||||
version = "8.3.2"
|
version = "8.3.2"
|
||||||
@@ -73,6 +82,15 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "distlib"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "distro"
|
name = "distro"
|
||||||
version = "1.9.0"
|
version = "1.9.0"
|
||||||
@@ -106,6 +124,7 @@ dependencies = [
|
|||||||
|
|
||||||
[package.optional-dependencies]
|
[package.optional-dependencies]
|
||||||
dev = [
|
dev = [
|
||||||
|
{ name = "pre-commit" },
|
||||||
{ name = "pytest" },
|
{ name = "pytest" },
|
||||||
{ name = "ruff" },
|
{ name = "ruff" },
|
||||||
{ name = "ty" },
|
{ name = "ty" },
|
||||||
@@ -116,6 +135,7 @@ requires-dist = [
|
|||||||
{ name = "anthropic", specifier = ">=0.42" },
|
{ name = "anthropic", specifier = ">=0.42" },
|
||||||
{ name = "click", specifier = ">=8.1" },
|
{ name = "click", specifier = ">=8.1" },
|
||||||
{ name = "jinja2", specifier = ">=3.1" },
|
{ name = "jinja2", specifier = ">=3.1" },
|
||||||
|
{ name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.0" },
|
||||||
{ name = "pydantic", specifier = ">=2.0" },
|
{ name = "pydantic", specifier = ">=2.0" },
|
||||||
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" },
|
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" },
|
||||||
{ name = "python-dotenv", specifier = ">=1.0" },
|
{ name = "python-dotenv", specifier = ">=1.0" },
|
||||||
@@ -125,6 +145,15 @@ requires-dist = [
|
|||||||
]
|
]
|
||||||
provides-extras = ["dev"]
|
provides-extras = ["dev"]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "filelock"
|
||||||
|
version = "3.29.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "h11"
|
name = "h11"
|
||||||
version = "0.16.0"
|
version = "0.16.0"
|
||||||
@@ -162,6 +191,15 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
|
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "identify"
|
||||||
|
version = "2.6.19"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "idna"
|
name = "idna"
|
||||||
version = "3.11"
|
version = "3.11"
|
||||||
@@ -351,6 +389,15 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" },
|
{ url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nodeenv"
|
||||||
|
version = "1.10.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "packaging"
|
name = "packaging"
|
||||||
version = "26.0"
|
version = "26.0"
|
||||||
@@ -360,6 +407,15 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
|
{ url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "platformdirs"
|
||||||
|
version = "4.9.6"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pluggy"
|
name = "pluggy"
|
||||||
version = "1.6.0"
|
version = "1.6.0"
|
||||||
@@ -369,6 +425,22 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pre-commit"
|
||||||
|
version = "4.6.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "cfgv" },
|
||||||
|
{ name = "identify" },
|
||||||
|
{ name = "nodeenv" },
|
||||||
|
{ name = "pyyaml" },
|
||||||
|
{ name = "virtualenv" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/8e/22/2de9408ac81acbb8a7d05d4cc064a152ccf33b3d480ebe0cd292153db239/pre_commit-4.6.0.tar.gz", hash = "sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9", size = 198525, upload-time = "2026-04-21T20:31:41.613Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/80/6e/4b28b62ecb6aae56769c34a8ff1d661473ec1e9519e2d5f8b2c150086b26/pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b", size = 226472, upload-time = "2026-04-21T20:31:40.092Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pydantic"
|
name = "pydantic"
|
||||||
version = "2.12.5"
|
version = "2.12.5"
|
||||||
@@ -506,6 +578,19 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" },
|
{ url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "python-discovery"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "filelock" },
|
||||||
|
{ name = "platformdirs" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/ae/e0/cc5a8653e9a24f6cf84768f05064aa8ed5a83dcefd5e2a043db14a1c5f44/python_discovery-1.3.0.tar.gz", hash = "sha256:d098f1e86be5d45fe4d14bf1029294aabbd332f4321179dec85e76cddce834b0", size = 63925, upload-time = "2026-05-05T14:38:39.769Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/30/d4/24d543ab8b8158b7f5a97113c831205f5c900c92c8762b1e7f44b7ea0405/python_discovery-1.3.0-py3-none-any.whl", hash = "sha256:441d9ced3dfce36e113beb35ca302c71c7ef06f3c0f9c227a0b9bb3bd49b9e9f", size = 33124, upload-time = "2026-05-05T14:38:38.539Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "python-dotenv"
|
name = "python-dotenv"
|
||||||
version = "1.2.2"
|
version = "1.2.2"
|
||||||
@@ -648,3 +733,18 @@ sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac
|
|||||||
wheels = [
|
wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" },
|
{ url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "virtualenv"
|
||||||
|
version = "21.3.1"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "distlib" },
|
||||||
|
{ name = "filelock" },
|
||||||
|
{ name = "platformdirs" },
|
||||||
|
{ name = "python-discovery" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/ec/0d/915c02c94d207b85580eb09bffab54438a709e7288524094fe781da526c2/virtualenv-21.3.1.tar.gz", hash = "sha256:c2305bc1fddeec40699b8370d13f8d431b0701f00ce895061ce493aeded4426b", size = 7613791, upload-time = "2026-05-05T01:34:31.402Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b1/4f/f71e641e504111a5a74e3a20bc52d01bd86788b22699dd3fee1c63253cf6/virtualenv-21.3.1-py3-none-any.whl", hash = "sha256:d1a71cf58f2f9228fff23a1f6ec15d39785c6b32e03658d104974247145edd35", size = 7594539, upload-time = "2026-05-05T01:34:28.98Z" },
|
||||||
|
]
|
||||||
|
|||||||
Reference in New Issue
Block a user