mirror of
https://github.com/obra/superpowers.git
synced 2026-05-09 18:49:04 +08:00
3c046f579e
rsync of obra/drill@013fcb8b7d into superpowers/evals/, excluding .git/, .venv/, results/, .env/, __pycache__/, *.egg-info/, .private-journal/. The drill repo is unaffected by this commit; archival is a separate manual step after this PR merges. Source SHA recorded at evals/.drill-source-sha for divergence detection.
95 lines
2.3 KiB
Python
95 lines
2.3 KiB
Python
"""End-to-end smoke test using a mock 'bash' backend."""
|
|
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from drill.engine import Engine, ScenarioConfig
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_scenario(tmp_path):
|
|
scenario = tmp_path / "test-scenario.yaml"
|
|
scenario.write_text("""
|
|
scenario: e2e-smoke-test
|
|
description: "Smoke test"
|
|
user_posture: naive
|
|
setup:
|
|
helpers:
|
|
- create_base_repo
|
|
assertions:
|
|
- "git rev-parse --is-inside-work-tree"
|
|
turns:
|
|
- intent: "List files in the current directory"
|
|
limits:
|
|
max_turns: 3
|
|
turn_timeout: 10
|
|
verify:
|
|
criteria:
|
|
- "Agent listed the files"
|
|
observe: true
|
|
""")
|
|
return scenario
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_backend(tmp_path):
|
|
backend_dir = tmp_path / "backends"
|
|
backend_dir.mkdir()
|
|
(backend_dir / "mock.yaml").write_text("""
|
|
name: mock
|
|
cli: bash
|
|
args: []
|
|
required_env: []
|
|
hooks:
|
|
pre_run: []
|
|
post_run: []
|
|
shutdown: "exit"
|
|
idle:
|
|
quiescence_seconds: 1
|
|
ready_pattern: "\\\\$"
|
|
startup_timeout: 5
|
|
terminal:
|
|
cols: 80
|
|
rows: 24
|
|
session_logs:
|
|
pattern: ""
|
|
""")
|
|
return backend_dir
|
|
|
|
|
|
class TestE2ESmoke:
|
|
def test_scenario_config_loads(self, mock_scenario):
|
|
config = ScenarioConfig.from_yaml(mock_scenario)
|
|
assert config.scenario == "e2e-smoke-test"
|
|
|
|
def test_engine_setup_works(self, mock_scenario, mock_backend):
|
|
fixtures_dir = Path(__file__).parent.parent / "fixtures"
|
|
engine = Engine(
|
|
scenario_path=mock_scenario,
|
|
backend_name="mock",
|
|
backends_dir=mock_backend,
|
|
fixtures_dir=fixtures_dir,
|
|
results_dir=Path("/tmp/drill-test-results"),
|
|
)
|
|
workdir = Path("/tmp/drill-e2e-smoke")
|
|
if workdir.exists():
|
|
shutil.rmtree(workdir)
|
|
engine._setup(workdir)
|
|
assert (workdir / "package.json").exists()
|
|
assert (workdir / "src" / "index.js").exists()
|
|
# Verify git state
|
|
import subprocess
|
|
|
|
result = subprocess.run(
|
|
["git", "branch", "--show-current"], cwd=workdir, capture_output=True, text=True
|
|
)
|
|
assert result.stdout.strip() == "main"
|
|
result = subprocess.run(
|
|
["git", "log", "--oneline"], cwd=workdir, capture_output=True, text=True
|
|
)
|
|
assert "initial commit" in result.stdout
|
|
# Cleanup
|
|
shutil.rmtree(workdir, ignore_errors=True)
|