mirror of
https://github.com/obra/superpowers.git
synced 2026-05-10 19:19:03 +08:00
Lift drill into evals/ at 013fcb8b7dbefd6d3fa4653493e5d2ec8e7f985b
rsync of obra/drill@013fcb8b7d into superpowers/evals/, excluding .git/, .venv/, results/, .env/, __pycache__/, *.egg-info/, .private-journal/. The drill repo is unaffected by this commit; archival is a separate manual step after this PR merges. Source SHA recorded at evals/.drill-source-sha for divergence detection.
This commit is contained in:
committed by
Drew Ritter
parent
2e46e9590d
commit
3b412a3836
94
evals/tests/test_e2e.py
Normal file
94
evals/tests/test_e2e.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""End-to-end smoke test using a mock 'bash' backend."""
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from drill.engine import Engine, ScenarioConfig
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_scenario(tmp_path):
|
||||
scenario = tmp_path / "test-scenario.yaml"
|
||||
scenario.write_text("""
|
||||
scenario: e2e-smoke-test
|
||||
description: "Smoke test"
|
||||
user_posture: naive
|
||||
setup:
|
||||
helpers:
|
||||
- create_base_repo
|
||||
assertions:
|
||||
- "git rev-parse --is-inside-work-tree"
|
||||
turns:
|
||||
- intent: "List files in the current directory"
|
||||
limits:
|
||||
max_turns: 3
|
||||
turn_timeout: 10
|
||||
verify:
|
||||
criteria:
|
||||
- "Agent listed the files"
|
||||
observe: true
|
||||
""")
|
||||
return scenario
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_backend(tmp_path):
|
||||
backend_dir = tmp_path / "backends"
|
||||
backend_dir.mkdir()
|
||||
(backend_dir / "mock.yaml").write_text("""
|
||||
name: mock
|
||||
cli: bash
|
||||
args: []
|
||||
required_env: []
|
||||
hooks:
|
||||
pre_run: []
|
||||
post_run: []
|
||||
shutdown: "exit"
|
||||
idle:
|
||||
quiescence_seconds: 1
|
||||
ready_pattern: "\\\\$"
|
||||
startup_timeout: 5
|
||||
terminal:
|
||||
cols: 80
|
||||
rows: 24
|
||||
session_logs:
|
||||
pattern: ""
|
||||
""")
|
||||
return backend_dir
|
||||
|
||||
|
||||
class TestE2ESmoke:
|
||||
def test_scenario_config_loads(self, mock_scenario):
|
||||
config = ScenarioConfig.from_yaml(mock_scenario)
|
||||
assert config.scenario == "e2e-smoke-test"
|
||||
|
||||
def test_engine_setup_works(self, mock_scenario, mock_backend):
|
||||
fixtures_dir = Path(__file__).parent.parent / "fixtures"
|
||||
engine = Engine(
|
||||
scenario_path=mock_scenario,
|
||||
backend_name="mock",
|
||||
backends_dir=mock_backend,
|
||||
fixtures_dir=fixtures_dir,
|
||||
results_dir=Path("/tmp/drill-test-results"),
|
||||
)
|
||||
workdir = Path("/tmp/drill-e2e-smoke")
|
||||
if workdir.exists():
|
||||
shutil.rmtree(workdir)
|
||||
engine._setup(workdir)
|
||||
assert (workdir / "package.json").exists()
|
||||
assert (workdir / "src" / "index.js").exists()
|
||||
# Verify git state
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "branch", "--show-current"], cwd=workdir, capture_output=True, text=True
|
||||
)
|
||||
assert result.stdout.strip() == "main"
|
||||
result = subprocess.run(
|
||||
["git", "log", "--oneline"], cwd=workdir, capture_output=True, text=True
|
||||
)
|
||||
assert "initial commit" in result.stdout
|
||||
# Cleanup
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
Reference in New Issue
Block a user