superpowers/evals/tests/test_sweep.py

"""Tests for Sweep orchestrator."""

from __future__ import annotations

import json
from dataclasses import asdict
from pathlib import Path
from unittest.mock import patch

from drill.engine import Engine, RunResult
from drill.sweep import RunGroup, RunStatus, Sweep, write_run_group


class TestRunStatus:
    def test_pass_status(self) -> None:
        rs = RunStatus(index=0, status="pass", duration=10.5)
        assert rs.error is None
        assert rs.status == "pass"

    def test_error_status(self) -> None:
        rs = RunStatus(index=2, status="error", duration=1.2, error="tmux crashed")
        assert rs.error == "tmux crashed"

    def test_serializes_to_dict(self) -> None:
        rs = RunStatus(index=0, status="pass", duration=10.5)
        d = asdict(rs)
        assert d["index"] == 0
        assert d["status"] == "pass"
        assert d["duration"] == 10.5
        assert d["error"] is None


class TestRunGroup:
    def test_creates_with_defaults(self) -> None:
        rg = RunGroup(
            scenario="test",
            backend="claude",
            n=3,
            timestamp="2026-04-20T14-30-00",
            sweep_id="abc12345",
            runs=[],
        )
        assert rg.partial is False

    def test_partial_flag(self) -> None:
        rg = RunGroup(
            scenario="test",
            backend="claude",
            n=3,
            timestamp="2026-04-20T14-30-00",
            sweep_id="abc12345",
            runs=[RunStatus(index=0, status="pass", duration=10.0)],
            partial=True,
        )
        assert rg.partial is True
        assert len(rg.runs) == 1


class TestWriteRunGroup:
    def test_writes_json(self, tmp_path: Path) -> None:
        rg = RunGroup(
            scenario="test-scenario",
            backend="claude",
            n=2,
            timestamp="2026-04-20T14-30-00",
            sweep_id="abc12345",
            runs=[
                RunStatus(index=0, status="pass", duration=100.0),
                RunStatus(index=1, status="fail", duration=95.0),
            ],
        )
        write_run_group(rg, tmp_path)
        path = tmp_path / "run-group.json"
        assert path.exists()
        data = json.loads(path.read_text())
        assert data["scenario"] == "test-scenario"
        assert data["sweep_id"] == "abc12345"
        assert data["partial"] is False
        assert len(data["runs"]) == 2
        assert data["runs"][0]["status"] == "pass"
        assert data["runs"][1]["status"] == "fail"

    def test_writes_partial(self, tmp_path: Path) -> None:
        rg = RunGroup(
            scenario="test",
            backend="claude",
            n=5,
            timestamp="2026-04-20T14-30-00",
            sweep_id="abc12345",
            runs=[RunStatus(index=0, status="pass", duration=100.0)],
            partial=True,
        )
        write_run_group(rg, tmp_path)
        data = json.loads((tmp_path / "run-group.json").read_text())
        assert data["partial"] is True
        assert len(data["runs"]) == 1

    def test_omits_null_errors(self, tmp_path: Path) -> None:
        rg = RunGroup(
            scenario="test",
            backend="claude",
            n=1,
            timestamp="2026-04-20T14-30-00",
            sweep_id="abc12345",
            runs=[RunStatus(index=0, status="pass", duration=50.0)],
        )
        write_run_group(rg, tmp_path)
        data = json.loads((tmp_path / "run-group.json").read_text())
        run_data = data["runs"][0]
        assert "error" not in run_data


class TestSweepIntegration:
    def test_full_sweep_writes_run_group(self, tmp_path: Path) -> None:
        """Test that Sweep creates run dirs and writes run-group.json."""
        scenario_file = tmp_path / "scenarios" / "test.yaml"
        scenario_file.parent.mkdir(parents=True)
        scenario_file.write_text(
            "scenario: test-scenario\n"
            "description: test\n"
            "user_posture: naive\n"
            "setup: {}\n"
            "turns:\n  - intent: do the thing\n"
            "limits:\n  max_turns: 5\n"
            "verify:\n  criteria:\n    - thing was done\n"
        )

        backends_dir = tmp_path / "backends"
        backends_dir.mkdir()
        (backends_dir / "mock-backend.yaml").write_text(
            "name: mock-backend\n"
            "cli: echo\n"
            "args: []\n"
            "required_env: []\n"
            "hooks:\n  pre_run: []\n  post_run: []\n"
            "shutdown: /exit\n"
            "idle:\n  quiescence_seconds: 1\n  ready_pattern: '.'\n"
            "startup_timeout: 5\n"
            "terminal:\n  cols: 80\n  rows: 24\n"
            "session_logs: {}\n"
        )

        results_dir = tmp_path / "results"
        fixtures_dir = tmp_path / "fixtures"
        fixtures_dir.mkdir()

        fake_verdict = json.dumps(
            {
                "criteria": [
                    {
                        "criterion": "thing was done",
                        "verdict": "pass",
                        "evidence": "yes",
                        "rationale": "it was done",
                    }
                ],
                "observations": [],
                "summary": "ok",
            }
        )

        fake_result = RunResult(
            scenario="test-scenario",
            backend="mock-backend",
            timestamp="2026-04-20T14-30-00",
            session_log="log",
            filesystem_json='{"files": []}',
            tool_calls_jsonl='{"tool": "Bash"}',
            verdict_json=fake_verdict,
            meta={"actor_turns": 3},
        )

        sweep = Sweep(
            scenario_path=scenario_file,
            backend_names=["mock-backend"],
            backends_dir=backends_dir,
            fixtures_dir=fixtures_dir,
            results_dir=results_dir,
            n=3,
            sweep_id="test1234",
        )

        with patch.object(Engine, "run", return_value=fake_result):
            groups = sweep.run_all()

        assert len(groups) == 1
        group = groups[0]
        assert group.scenario == "test-scenario"
        assert len(group.runs) == 3
        assert all(r.status == "pass" for r in group.runs)
        assert group.partial is False

        # Verify run-group.json was written
        scenario_results = results_dir / "test-scenario" / "mock-backend"
        assert scenario_results.exists()
        group_dirs = list(scenario_results.iterdir())
        assert len(group_dirs) == 1
        rg_path = group_dirs[0] / "run-group.json"
        assert rg_path.exists()
        rg_data = json.loads(rg_path.read_text())
        assert rg_data["sweep_id"] == "test1234"
        assert len(rg_data["runs"]) == 3