from __future__ import annotations import json import errno import os import socket import subprocess import tempfile import unittest from pathlib import Path from typing import Any from unittest.mock import patch from .runtime import ( BrowserRuntimeError, GenerationConflict, NativeRuntimeManager, RuntimeCleanupPending, RUNTIME_CLEANUP_SENTINEL, UnitStatus, ) class FakeUnits: def __init__(self) -> None: self.units: dict[str, UnitStatus] = {} self.commands: list[tuple[str, list[str]]] = [] self.next_pid = 1000 self.fail_stop: set[str] = set() def start(self, unit: str, command: list[str], **kwargs: Any) -> UnitStatus: del kwargs self.commands.append((unit, list(command))) self.next_pid += 1 status = UnitStatus(True, "active", self.next_pid, str(self.next_pid), "") self.units[unit] = status return status def status(self, unit: str) -> UnitStatus: return self.units.get(unit, UnitStatus(False, "inactive", 0, "", "")) def stop(self, unit: str, timeout: float) -> None: del timeout if unit in self.fail_stop: raise OSError("stop failed") self.units[unit] = UnitStatus(False, "inactive", 0, "", "") def reset(self, unit: str) -> None: del unit class SystemdUnitManagerTests(unittest.TestCase): def test_start_status_stop_and_reset(self) -> None: calls: list[list[str]] = [] stop_calls = 0 def runner(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: nonlocal stop_calls del kwargs calls.append(args) if args[0] == "systemctl" and args[2] == "show": return subprocess.CompletedProcess( args, 0, "ActiveState=active\nSubState=running\nMainPID=42\n" "ExecMainStartTimestampMonotonic=1\nExecMainStatus=0\n", "", ) if args[0] == "systemctl" and args[2] == "stop": stop_calls += 1 return subprocess.CompletedProcess(args, 1 if stop_calls == 1 else 0, "", "") return subprocess.CompletedProcess(args, 0, "", "") from .runtime import SystemdUnitManager with tempfile.TemporaryDirectory() as directory: manager = SystemdUnitManager("systemd-run", "systemctl", runner) status = manager.start( "creatorhub-test.service", ["/bin/true"], environment={"B": "2", "A": "1"}, working_directory=Path(directory), stdout_path=Path(directory) / "runtime.log", limits={"MemoryMax": "1G"}, ) self.assertEqual(status.pid, 42) self.assertTrue(any("--setenv" in item for item in calls[0])) self.assertEqual(manager.status("creatorhub-test.service").state, "running") manager.stop("creatorhub-test.service", 1) manager.reset("creatorhub-test.service") self.assertTrue(any(args[2] == "kill" for args in calls)) self.assertTrue(any(args[2] == "reset-failed" for args in calls)) def test_status_and_start_failures_are_explicit(self) -> None: from .runtime import BrowserRuntimeError, SystemdUnitManager with self.assertRaises(BrowserRuntimeError): SystemdUnitManager(runner=lambda *args, **kwargs: (_ for _ in ()).throw(OSError("down"))).status("x") def failed_start(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: del kwargs return subprocess.CompletedProcess(args, 1, "", "rejected") with tempfile.TemporaryDirectory() as directory, self.assertRaises( BrowserRuntimeError ): SystemdUnitManager("systemd-run", "systemctl", failed_start).start( "x", ["/bin/true"], environment={}, working_directory=Path(directory), stdout_path=Path(directory) / "x.log", limits={}, ) def missing_status(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: del kwargs if args[2] == "show": return subprocess.CompletedProcess(args, 1, "", "missing") return subprocess.CompletedProcess(args, 0, "", "") manager = SystemdUnitManager("systemd-run", "systemctl", missing_status) self.assertFalse(manager.status("missing").active) self.assertEqual(manager.status("missing").state, "not-found") manager.reset("missing") def test_stop_and_reset_failures_are_not_hidden(self) -> None: from .runtime import BrowserRuntimeError, SystemdUnitManager def kill_fails(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: del kwargs if args[2] == "stop": return subprocess.CompletedProcess(args, 1, "", "failed") if args[2] == "kill": return subprocess.CompletedProcess(args, 1, "", "failed") return subprocess.CompletedProcess(args, 0, "", "") with self.assertRaises(BrowserRuntimeError): SystemdUnitManager("systemd-run", "systemctl", kill_fails).stop("x", 1) def reset_fails(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: del kwargs return subprocess.CompletedProcess(args, 2, "", "failed") with self.assertRaises(BrowserRuntimeError): SystemdUnitManager("systemd-run", "systemctl", reset_fails).reset("x") def test_allocator_existing_and_file_lock_edges(self) -> None: from .runtime import ( BrowserRuntimeError, DisplayAllocator, FileLock, PortAllocator, ) with tempfile.TemporaryDirectory() as directory: lock_dir = Path(directory) lock = FileLock(lock_dir / "resource.lock") self.assertTrue(lock.acquire()) self.assertTrue(lock.acquire()) lock.release() lock.release() ports = PortAllocator(lock_dir) lease = ports.reserve_existing("cdp", 19999) with self.assertRaises(BrowserRuntimeError): ports.reserve_existing("cdp", 19999) lease.lock.release() with self.assertRaises(BrowserRuntimeError): ports.reserve_existing("cdp", 0) displays = DisplayAllocator(lock_dir) display = displays.reserve_existing(100) with self.assertRaises(BrowserRuntimeError): displays.reserve_existing(100) display.lock.release() with self.assertRaises(BrowserRuntimeError): displays.reserve_existing(0) class DisplayAllocationTests(unittest.TestCase): def test_dead_x_server_files_do_not_exhaust_display_pool(self) -> None: from .runtime import DisplayAllocator with tempfile.TemporaryDirectory() as directory: root = Path(directory) sockets = root / '.X11-unix' sockets.mkdir() stale = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) stale.bind(str(sockets / 'X100')) stale.close() (root / '.X100-lock').write_text('99999999\n') with patch('browser_gateway.runtime.X11_SOCKET_DIR', sockets), patch('browser_gateway.runtime.os.kill', side_effect=ProcessLookupError(errno.ESRCH, 'gone')): allocator = DisplayAllocator(root / 'leases') lease = allocator.reserve(100, 100, lambda value: False) self.assertEqual(lease.value, 100) lease.lock.release() def test_invalid_pid_lock_is_reported_without_leaking_reservation(self) -> None: from .runtime import DisplayAllocator with tempfile.TemporaryDirectory() as directory: root = Path(directory) sockets = root / '.X11-unix' sockets.mkdir() pid_lock = root / '.X100-lock' with patch('browser_gateway.runtime.X11_SOCKET_DIR', sockets): allocator = DisplayAllocator(root / 'leases') for value in ('bad', '0', '-1'): with self.subTest(value=value): pid_lock.write_text(value) with self.assertRaisesRegex(BrowserRuntimeError, 'invalid X11 PID lock'): allocator.reserve(100, 100, lambda value: False) pid_lock.unlink() lease = allocator.reserve(100, 100, lambda value: False) self.assertEqual(lease.value, 100) lease.lock.release() def test_live_x_server_pid_is_not_reused(self) -> None: from .runtime import DisplayAllocator with tempfile.TemporaryDirectory() as directory: root = Path(directory) sockets = root / '.X11-unix' sockets.mkdir() (root / '.X100-lock').write_text(str(os.getpid())) with patch('browser_gateway.runtime.X11_SOCKET_DIR', sockets): with self.assertRaisesRegex(BrowserRuntimeError, 'no free Xvfb display'): DisplayAllocator(root / 'leases').reserve(100, 100, lambda value: False) def test_listening_x_socket_without_pid_lock_is_not_reused(self) -> None: from .runtime import DisplayAllocator with tempfile.TemporaryDirectory() as directory: root = Path(directory) sockets = root / '.X11-unix' sockets.mkdir() with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server: server.bind(str(sockets / 'X100')) server.listen() with patch('browser_gateway.runtime.X11_SOCKET_DIR', sockets): with self.assertRaisesRegex(BrowserRuntimeError, 'no free Xvfb display'): DisplayAllocator(root / 'leases').reserve(100, 100, lambda value: False) class NativeRuntimeManagerTests(unittest.TestCase): def setUp(self) -> None: self.temp = tempfile.TemporaryDirectory() root = Path(self.temp.name) self.state = root / "state" self.profiles = root / "profiles" self.units = FakeUnits() self.manager = NativeRuntimeManager( state_dir=self.state, profile_root=self.profiles, node_id="node-a", browser_path="/bin/true", unit_manager=self.units, min_free_bytes=0, display_start=1000, display_end=1010, ) self.manager._wait_for_display = lambda record: None self.manager._wait_for_cdp = lambda record: None def test_runtime_lock_executes_server_as_main_process_for_graceful_stop(self) -> None: command = self.manager._locked_command(self.state / 'test.lock', ['/bin/true']) self.assertEqual(command[1], '--no-fork') def tearDown(self) -> None: self.manager.close() self.temp.cleanup() def payload(self, **extra: Any) -> dict[str, Any]: value: dict[str, Any] = { "alias": "account-a", "name": "Account A", "profile_id": "account-a-id", "cmd": ["--fingerprint=1000", "about:blank"], "binding_version": 1, "network_exit_id": "", "network_exit": {}, "stopped": False, } value.update(extra) return value def test_create_allocates_native_runtime_and_persists_generation(self) -> None: result = self.manager.create(self.payload()) self.assertEqual(result["state"], "running") self.assertEqual(result["node_id"], "node-a") self.assertRegex(result["runtime_id"], r"^[a-f0-9]{64}$") self.assertTrue(result["endpoint"].startswith("http://127.0.0.1:")) runtime_file = next(self.state.glob("runtimes/*/runtime.json")) stored = json.loads(runtime_file.read_text()) self.assertEqual(stored["runtime_id"], result["runtime_id"]) browser_command = self.units.commands[-1][1] self.assertNotIn("--no-sandbox", browser_command) self.assertNotIn("--", browser_command) self.assertIn("--disable-gpu", browser_command) self.assertIn("--disable-gpu-compositing", browser_command) self.assertIn("--password-store=basic", browser_command) self.assertIn("--remote-debugging-address=127.0.0.1", browser_command) self.assertIn("--user-data-dir=" + stored["profile_dir"], browser_command) def test_insufficient_disk_is_rejected_before_process_side_effects(self) -> None: limited = NativeRuntimeManager( state_dir=self.state / "disk-state", profile_root=self.profiles / "disk-profiles", node_id="node-a", browser_path="/bin/true", unit_manager=self.units, min_free_bytes=10**20, ) with self.assertRaises(BrowserRuntimeError) as caught: limited.create(self.payload(alias="disk-account")) self.assertIn("free disk space", str(caught.exception)) self.assertEqual(self.units.commands, []) limited.close() def test_external_display_reuses_existing_xvfb_without_starting_or_stopping_it(self) -> None: external = NativeRuntimeManager( state_dir=self.state / "external-state", profile_root=self.profiles / "external-profiles", node_id="node-a", browser_path="/bin/true", unit_manager=self.units, external_display=99, min_free_bytes=0, ) external._display_available = lambda display: display == 99 external._wait_for_cdp = lambda record: None result = external.create(self.payload(alias="external-account")) self.assertEqual(result["display"], 99) self.assertEqual(result["display_mode"], "external") self.assertEqual(len(self.units.commands), 1) self.assertIn("browser", self.units.commands[0][0]) generation = { "binding_version": 1, "runtime_id": result["runtime_id"], "network_id": result["network_id"], } external.change_state("external-account", "stop", generation) self.assertEqual(external.list_public()[0]["state"], "stopped") external.close() def test_external_display_rejects_a_second_active_runtime(self) -> None: external = NativeRuntimeManager( state_dir=self.state / "external-state-2", profile_root=self.profiles / "external-profiles-2", node_id="node-a", browser_path="/bin/true", unit_manager=self.units, external_display=99, min_free_bytes=0, ) external._display_available = lambda display: display == 99 external._wait_for_cdp = lambda record: None external.create(self.payload(alias="external-account")) with self.assertRaises(BrowserRuntimeError) as caught: external.create(self.payload(alias="external-account-2", profile_id="other")) self.assertIn("already in use", str(caught.exception)) external.close() def test_generation_mismatch_cannot_stop_or_remove_new_runtime(self) -> None: result = self.manager.create(self.payload()) wrong = { "binding_version": 1, "runtime_id": "0" * 64, "network_id": result["network_id"], } with self.assertRaises(GenerationConflict): self.manager.change_state("account-a", "stop", wrong) with self.assertRaises(GenerationConflict): self.manager.remove("account-a", wrong) self.assertEqual(self.manager.list_public()[0]["state"], "running") def test_stop_and_repeated_remove_keep_profile(self) -> None: result = self.manager.create(self.payload()) generation = { "binding_version": 1, "runtime_id": result["runtime_id"], "network_id": result["network_id"], } self.manager.change_state("account-a", "stop", generation) self.assertEqual(self.manager.list_public()[0]["state"], "stopped") self.manager.remove("account-a", generation) self.manager.remove("account-a", generation) self.assertEqual(self.manager.list_public()[0]["state"], "released") profile_dirs = list(self.profiles.iterdir()) self.assertEqual(len(profile_dirs), 1) self.assertTrue(profile_dirs[0].is_dir()) def test_stopped_runtime_does_not_reserve_profile_for_another_alias(self) -> None: stopped = self.manager.create(self.payload(stopped=True)) running = self.manager.create(self.payload(alias="account-b")) self.assertEqual(running["state"], "running") self.assertEqual(self.manager.list_public()[0]["state"], "stopped") with self.assertRaisesRegex(BrowserRuntimeError, "Profile is already in use"): self.manager.change_state("account-a", "start", { "binding_version": 1, "runtime_id": stopped["runtime_id"], "network_id": stopped["network_id"], }) def test_profile_conflict_does_not_leave_an_orphaned_runtime(self) -> None: self.manager.create(self.payload()) with self.assertRaisesRegex(BrowserRuntimeError, "Profile is already in use"): self.manager.create(self.payload(alias="account-b")) self.assertEqual([item["alias"] for item in self.manager.list_public()], ["account-a"]) def test_same_profile_is_exclusive_across_manager_instances(self) -> None: first = self.manager.create(self.payload()) del first other = NativeRuntimeManager( state_dir=self.state, profile_root=self.profiles, node_id="node-a", browser_path="/bin/true", unit_manager=self.units, min_free_bytes=0, ) other._wait_for_display = lambda record: None other._wait_for_cdp = lambda record: None with self.assertRaises(BrowserRuntimeError): other.create(self.payload(alias="account-b")) other.close() def test_cleanup_failure_is_visible_and_retryable(self) -> None: result = self.manager.create(self.payload()) generation = { "binding_version": 1, "runtime_id": result["runtime_id"], "network_id": result["network_id"], } self.units.fail_stop.update( { f"creatorhub-{result['alias']}-{result['runtime_id'][:16]}-browser.service", } ) with self.assertRaises(RuntimeCleanupPending): self.manager.remove("account-a", generation) public = self.manager.list_public()[0] self.assertEqual(public["cleanup_state"], "pending") self.assertTrue(public["cleanup_error"]) self.units.fail_stop.clear() self.manager.retry_cleanup("account-a", generation) self.assertEqual(self.manager.list_public()[0]["state"], "released") def test_reserved_browser_flags_are_rejected_before_side_effects(self) -> None: for flag in ("--no-sandbox", "--password-store=gnome"): with self.subTest(flag=flag), self.assertRaises(BrowserRuntimeError): self.manager.create(self.payload(cmd=[flag, "about:blank"])) self.assertEqual(list(self.state.glob("runtimes/*/runtime.json")), []) def test_stopped_runtime_can_be_created_without_processes(self) -> None: result = self.manager.create(self.payload(stopped=True)) self.assertEqual(result["state"], "stopped") self.assertEqual(self.units.commands, []) def test_listing_during_start_cannot_overwrite_runtime_readiness(self) -> None: from threading import Event, Thread waiting, finish = Event(), Event() outcomes: list[object] = [] def wait(record: object) -> None: waiting.set() if not finish.wait(2): raise AssertionError("startup test did not release CDP wait") def create() -> None: try: outcomes.append(self.manager.create(self.payload())) except Exception as exc: outcomes.append(exc) self.manager._wait_for_cdp = wait thread = Thread(target=create) thread.start() try: self.assertTrue(waiting.wait(1)) runtime_file = next(self.state.glob("runtimes/*/runtime.json")) before = runtime_file.read_text() stale_starting = self.manager._records() public = self.manager.list_public()[0] self.assertEqual(public["state"], "starting") self.assertFalse(public["ready"]) self.assertEqual(runtime_file.read_text(), before) finally: finish.set() thread.join(2) self.assertFalse(thread.is_alive()) self.assertEqual(len(outcomes), 1) self.assertIsInstance(outcomes[0], dict) result = outcomes[0] self.manager._ready = lambda record: True with patch.object(self.manager, "_records", return_value=stale_starting): public = self.manager.list_public()[0] self.assertEqual(public["state"], "running") self.assertTrue(public["ready"]) verified = self.manager.require_generation("account-a", result) self.assertEqual(verified.state, "running") self.assertEqual(verified.runtime_id, result["runtime_id"]) def test_listing_reloads_metadata_after_lifecycle_transition(self) -> None: result = self.manager.create(self.payload()) stale = self.manager._records() self.manager.change_state("account-a", "stop", result) with patch.object(self.manager, "_records", return_value=stale): public = self.manager.list_public()[0] self.assertEqual(public["state"], "stopped") self.assertEqual(public["cleanup_state"], "cleaned") self.assertEqual(self.manager._records()[0].state, "stopped") def test_verified_live_degraded_runtime_recovers_without_restart(self) -> None: result = self.manager.create(self.payload()) record = self.manager._records()[0] record.state = "degraded" self.manager._write(record) self.manager._ready = lambda record: True commands_before = list(self.units.commands) verified = self.manager.require_generation("account-a", result) self.assertEqual(verified.state, "running") self.assertEqual(verified.runtime_id, result["runtime_id"]) self.assertEqual(self.units.commands, commands_before) self.assertTrue(all(status.active for status in self.units.units.values())) def test_degraded_runtime_with_pending_cleanup_is_not_recovered(self) -> None: result = self.manager.create(self.payload()) record = self.manager._records()[0] record.state = "degraded" record.cleanup_state = "pending" self.manager._write(record) self.manager._ready = lambda record: True with self.assertRaises(BrowserRuntimeError) as caught: self.manager.require_generation("account-a", result) self.assertEqual(caught.exception.status, 503) self.assertEqual(self.manager._records()[0].state, "degraded") self.assertTrue(all(status.active for status in self.units.units.values())) def test_listing_respects_cross_process_alias_lock(self) -> None: from .runtime import FileLock self.manager.create(self.payload()) runtime_file = next(self.state.glob("runtimes/*/runtime.json")) before = runtime_file.read_text() external = FileLock(self.state / "locks" / "alias-account-a.lock") self.assertTrue(external.acquire()) try: with patch.object(self.manager, "_refresh", side_effect=AssertionError("busy alias refreshed")): public = self.manager.list_public()[0] self.assertFalse(public["ready"]) self.assertEqual(runtime_file.read_text(), before) finally: external.release() def test_timeout_marks_runtime_cleanup_pending(self) -> None: self.manager.ready_timeout = 0.001 self.manager._wait_for_display = NativeRuntimeManager._wait_for_display.__get__(self.manager) self.manager._wait_for_cdp = NativeRuntimeManager._wait_for_cdp.__get__(self.manager) with self.assertRaises(BrowserRuntimeError) as caught: self.manager.create(self.payload()) self.assertIn("Xvfb", str(caught.exception)) public = self.manager.list_public()[0] self.assertEqual(public["cleanup_state"], "pending") self.assertEqual(public["state"], "failed") def test_cancel_during_start_is_visible_and_does_not_start_browser(self) -> None: started = __import__("threading").Event() original_wait = self.manager._wait_for_display def wait(record: object) -> None: started.set() while True: self.manager._check_cancel(record) # type: ignore[arg-type] __import__("time").sleep(0.001) self.manager._wait_for_display = wait result: list[object] = [] def create() -> None: try: result.append(self.manager.create(self.payload())) except Exception as exc: # the assertion below checks the typed outcome result.append(exc) thread = __import__("threading").Thread(target=create) thread.start() self.assertTrue(started.wait(1)) runtime_file = next(self.state.glob("runtimes/*/runtime.json")) stored = json.loads(runtime_file.read_text()) self.manager.cancel( "account-a", { "binding_version": stored["binding_version"], "runtime_id": stored["runtime_id"], "network_id": stored["network_id"], }, ) thread.join(1) self.assertFalse(thread.is_alive()) self.assertTrue(result and isinstance(result[0], BrowserRuntimeError)) self.assertEqual(self.manager.list_public()[0]["cleanup_state"], "pending") self.manager._wait_for_display = original_wait def test_restart_recovers_only_matching_active_units(self) -> None: result = self.manager.create(self.payload()) self.manager.close() recovered = NativeRuntimeManager( state_dir=self.state, profile_root=self.profiles, node_id="node-a", browser_path="/bin/true", unit_manager=self.units, min_free_bytes=0, ) self.assertEqual(recovered.list_public()[0]["runtime_id"], result["runtime_id"]) self.assertEqual(recovered.list_public()[0]["node_id"], "node-a") recovered.close() def test_old_released_generation_cannot_touch_new_runtime(self) -> None: old = self.manager.create(self.payload(stopped=True)) old_generation = { "binding_version": 1, "runtime_id": old["runtime_id"], "network_id": old["network_id"], } self.manager.remove("account-a", old_generation) new = self.manager.create(self.payload(stopped=True)) self.assertNotEqual(old["runtime_id"], new["runtime_id"]) self.manager.remove("account-a", old_generation) self.assertEqual(self.manager.list_public()[-1]["runtime_id"], new["runtime_id"]) self.assertEqual(self.manager.list_public()[-1]["state"], "stopped") def test_sentinel_remove_with_only_released_records_is_noop(self) -> None: """pause 后(只剩 released 记录)再删除:sentinel 代不代表实物,不应 409。""" self.manager.create(self.payload()) record = next(r for r in self.manager._records() if r.alias == "account-a") self.manager._stop_locked(record, released=True) self.assertEqual(record.state, "released") # 控制面在 DB runtime 已清空后发 sentinel 代删除。 self.manager.remove("account-a", { "binding_version": 1, "runtime_id": RUNTIME_CLEANUP_SENTINEL, "network_id": "", }) def test_empty_or_sentinel_runtime_id_purges_released_profile(self) -> None: """账号删除(purge_profile)在 runtime 已释放后到达:空/哨兵代放行且清掉档案。""" self.manager.create(self.payload()) record = next(r for r in self.manager._records() if r.alias == "account-a") self.manager._stop_locked(record, released=True) self.manager.remove("account-a", { "binding_version": 1, "runtime_id": "", "network_id": "", }, purge_profile=True) self.assertFalse((self.profiles / "account-a").exists()) if __name__ == "__main__": unittest.main()