diff --git a/benchmark/edgebench/checkpoint_controller.py b/benchmark/edgebench/checkpoint_controller.py index a842f1882f..15dddec24b 100644 --- a/benchmark/edgebench/checkpoint_controller.py +++ b/benchmark/edgebench/checkpoint_controller.py @@ -24,6 +24,7 @@ from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener from loopx.control_plane.effect_runtime import effect_runtime_result +from benchmark.runtime.docker_checkpoint import _physical, _remaining from benchmark.runtime.scheduler import read_phase_deadline from benchmark.runtime.worker import PHASE_WORK_RESERVE_SECONDS @@ -33,13 +34,6 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return None # Never forward controller credentials to another origin. -def _physical(path: Path) -> Path: - path = Path(path) - if not path.is_absolute() or path.resolve(strict=True) != path: - raise ValueError("Controller paths must be normalized, physical and absolute") - return path - - def _controller(path: Path) -> dict: path = _physical(path) if path.stat().st_mode & 0o077: @@ -183,13 +177,6 @@ def _activation_result(result: dict, body: dict) -> None: raise ValueError("Native activation result differs from the original request") -def _remaining(deadline: float) -> float: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError("Checkpoint capture window closed") - return remaining - - def capture(controller_file: Path, *, token: str, capture_id: str, destination: Path, timeout: float = 30, phase_deadline_file: Path | None = None) -> dict: """Capture one native-owned point without exporting controller configuration.""" diff --git a/benchmark/edgebench/checkpoint_supervisor.py b/benchmark/edgebench/checkpoint_supervisor.py index 9b820d3f65..06e30d473a 100644 --- a/benchmark/edgebench/checkpoint_supervisor.py +++ b/benchmark/edgebench/checkpoint_supervisor.py @@ -12,12 +12,13 @@ from pathlib import Path from loopx.control_plane.effect_runtime import EffectRuntimeRemoteError +from benchmark.runtime.docker_checkpoint import ( + _archive, _guest_path, capture_failure_fields, qualify_export_storage, validate_worker_request, +) from benchmark.runtime.layout import _ACTIVE_PHASE_DEADLINE, _PHASE_DEADLINE from .checkpoint_controller import _controller -from .checkpoint_worker import ( - _archive, _guest_path, _validate_request, capture_failure_fields, capture_worker, qualify_export_storage, -) +from .checkpoint_worker import capture_worker class CheckpointSupervisor: @@ -36,8 +37,10 @@ def __init__(self, controller_file: Path, destination: Path, retention_policy: P from .terminal_capture import FailurePreservation if terminal_receipt is None or terminal_receipt.exists(): raise ValueError("Terminal preservation requires a fresh private receipt") - _validate_request({"container_id": "0" * 64, "components": terminal_components, - "phase_deadline_path": phase_deadline_path}) + validate_worker_request({ + "container_id": "0" * 64, "components": terminal_components, + "phase_deadline_path": phase_deadline_path, + }) self.preservation = FailurePreservation(self, terminal_components, terminal_receipt) self.token = None self.stop = threading.Event() diff --git a/benchmark/edgebench/checkpoint_worker.py b/benchmark/edgebench/checkpoint_worker.py index 92d6a4ea79..4d13623275 100644 --- a/benchmark/edgebench/checkpoint_worker.py +++ b/benchmark/edgebench/checkpoint_worker.py @@ -1,428 +1,42 @@ -"""Explicit controller-owned Docker writer barrier for inert checkpoint export. - -This provider adapts Docker pause/archive; the existing native and typed artifact -owners still own evaluator state and sealing. No code runs inside a frozen worker. -It does not capture process memory or qualify a native conversation cursor. -""" +"""Bind the shared Docker checkpoint transport to the native EdgeBench judge.""" from __future__ import annotations -from contextlib import closing -from copy import copy -import hashlib import json import math -import os -import re -import tarfile -import tempfile +from pathlib import Path import time -from pathlib import Path, PurePosixPath - -from loopx.control_plane.effect_runtime import effect_runtime_result -from loopx.file_lock import exclusive_file_lock -from benchmark.runtime.checkpoint import _participant_id -from benchmark.runtime.checkpoint_handoff import controller_completion_deadline - -from .checkpoint_controller import _controller, _physical, _remaining, _phase_budget, capture - - -_COMPONENT_ID = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{0,63}") - - -def capture_failure_fields(error: Exception) -> dict[str, str]: - """Keep a bounded diagnostic for a declared component's failed Docker read.""" - fields = {"error_kind": type(error).__name__} - component_id = getattr(error, "checkpoint_component_id", None) - if isinstance(component_id, str) and _COMPONENT_ID.fullmatch(component_id): - from docker.errors import NotFound - - if isinstance(error, NotFound): - fields["missing_component_id"] = component_id - return fields - - -def qualify_export_storage(destination: Path) -> None: - """Check filename fidelity before registration, without reserving a run. - - Linux source trees can contain case-only or Unicode-normalization-only - siblings. A shared host volume may silently merge these during extraction - or sealing, even when the controller itself runs on Linux. - """ - if not destination.is_absolute() or destination.resolve() != destination: - raise ValueError("Checkpoint export destination must be normalized, physical and absolute") - if destination.exists(): - raise ValueError("Checkpoint export destination must be fresh; preserve the existing bank") - parent = destination.parent - while not parent.exists(): - parent = parent.parent - _physical(parent) - names = ("Candidate", "candidate", "caf\u00e9", "cafe\u0301") - with tempfile.TemporaryDirectory(prefix=".checkpoint-storage-", dir=parent) as directory: - root = Path(directory) - try: - for index, name in enumerate(names): - with (root / name).open("xb") as stream: - stream.write(bytes([index])) - except FileExistsError as error: - raise ValueError("Checkpoint export storage must preserve case and Unicode filename distinctions; " - "select a compatible volume with --checkpoint-export-root") from error - if (set(entry.name for entry in root.iterdir()) != set(names) or - any((root / name).read_bytes() != bytes([index]) for index, name in enumerate(names))): - raise ValueError("Checkpoint export storage changed filename identities or bytes") - - -def _request(path: Path) -> dict: - return _validate_request(json.loads(_physical(path).read_bytes())) - - -def _validate_request(value: dict) -> dict: - if (not isinstance(value, dict) or - set(value) not in ({"container_id", "components", "phase_deadline_path"}, - {"container_id", "components", "phase_deadline_path", "handoff"}) or - not re.fullmatch(r"[0-9a-f]{64}", value.get("container_id", "")) or - not isinstance(value["components"], list) or not value["components"]): - raise ValueError("Worker request requires a full container ID, explicit components and phase_deadline_path") - ids = set() - paths = [] - for entry in value["components"]: - if (not isinstance(entry, dict) or set(entry) != {"id", "source_path"} or - not isinstance(entry["id"], str) or - not _COMPONENT_ID.fullmatch(entry["id"]) or entry["id"] in ids): - raise ValueError("Worker components require unique IDs") - ids.add(entry["id"]) - paths.append(_guest_path(entry["source_path"])) - paths.append(_guest_path(value["phase_deadline_path"])) - if "handoff" in value: - handoff = value["handoff"] - if (not isinstance(handoff, dict) or set(handoff) != {"source_path", "sha256"} or - not re.fullmatch(r"[0-9a-f]{64}", handoff.get("sha256", ""))): - raise ValueError("Worker handoff requires the exact private request digest") - paths.append(_guest_path(handoff["source_path"])) - if any(a == b or a in b.parents or b in a.parents - for index, a in enumerate(paths) for b in paths[index + 1:]): - raise ValueError("Worker components and phase clock must not overlap") - return value - - -def _guest_path(value: str) -> PurePosixPath: - if not isinstance(value, str): - raise ValueError("Worker paths must be normalized absolute paths below root") - path = PurePosixPath(value) - if not path.is_absolute() or str(path) != value or value == "/" or ".." in path.parts: - raise ValueError("Worker paths must be normalized absolute paths below root") - return path - -class _ArchiveReader: - """Check the shared clock during tar headers and large-file copy reads.""" - def __init__(self, raw, deadline): - self.raw, self.deadline = raw, deadline +from benchmark.runtime.docker_checkpoint import capture_docker_worker, validate_worker_request - def __getattr__(self, name): - return getattr(self.raw, name) - - def read(self, *args): - _remaining(self.deadline) - block = self.raw.read(*args) - _remaining(self.deadline) - return block - - -def _archive(container, remote: str, destination: Path, *, deadline: float | None = None) -> Path: - """Read through the daemon, rejecting symlinks/devices and archive traversal. - - Docker's archive reader works while paused, unlike exec-based file readers. - Keep only the declared object, not parent filesystem or process credentials. - """ - from docker.utils import decode_json_header - - def check(): - if deadline is not None: - _remaining(deadline) - - check() - # The SDK's get_archive hides its Response behind a lazy generator. Closing - # that generator alone need not close an unread HTTP body. Own the response - # so every early exit releases the daemon reader before writer cleanup. - api = container.client.api - response = api._get(api._url("/containers/{0}/archive", container.id), - params={"path": remote}, stream=True, headers={"Accept-Encoding": "identity"}, - **({"timeout": _remaining(deadline)} if deadline is not None else {})) - with closing(response): - check() - api._raise_for_status(response) - stat = decode_json_header(response.headers["x-docker-container-path-stat"]) - if stat.get("linkTarget"): - raise ValueError("Worker component must not be a symlink") - destination.mkdir(mode=0o700) - # The explicitly provisioned export volume owns both the raw transfer - # and extracted files; a full ambient /tmp must not defeat that choice. - with tempfile.TemporaryFile(dir=destination) as raw: - # _stream_raw_result is Docker's long-running-process reader: it - # disables socket timeouts. Archive IO must keep the request's read - # timeout as well as check the shared absolute clock on progress. - for chunk in response.iter_content(64 * 1024, decode_unicode=False): - check() - raw.write(chunk) - check() - raw.seek(0) - with tarfile.open(fileobj=_ArchiveReader(raw, deadline) if deadline is not None else raw) as archive: - members = [] - root = PurePosixPath(remote).name - names = {} - for member in archive: - check() - path = PurePosixPath(member.name) - if (path.is_absolute() or ".." in path.parts or not path.parts or - path.parts[0] != root or str(path) in names or - not (member.isfile() or member.isdir() or member.islnk())): - raise ValueError("Worker archive contains undeclared, linked or special members") - names[str(path)] = member - members.append(member) - - # The typed artifact owner observes regular files, not inode - # sharing. Docker encodes a second name for the same inode as a - # tar hardlink (common in build outputs). Materialize its bytes - # from an inventoried regular member; never resolve a link on - # the host filesystem or permit a link chain/symlink escape. - extracted = [] - for member in members: - check() - if member.islnk(): - target_path = PurePosixPath(member.linkname) - target = names.get(str(target_path)) - if (target_path.is_absolute() or ".." in target_path.parts or - target is None or not target.isfile()): - raise ValueError("Worker archive contains undeclared, linked or special members") - regular = copy(target) - regular.name, regular.mode = member.name, member.mode - extracted.append(regular) - else: - extracted.append(member) - - def filtered(member, path): - check() - result = tarfile.data_filter(member, path) - check() - return result - - archive.extractall(destination, members=extracted, filter=filtered) - # The data filter intentionally drops directory modes. This is a - # private raw transport witness, so restore only ordinary permission - # bits after safe extraction; never restore owners or special bits. - # Otherwise a private 0700 authority directory becomes 0755 and a - # matching frozen source is falsely rejected as having changed. - for member in sorted(members, key=lambda entry: len(PurePosixPath(entry.name).parts), reverse=True): - check() - (destination / member.name).chmod(member.mode & 0o777) - check() - return _physical(destination / root) - - -def _archive_component(container, remote: str, destination: Path, *, deadline: float | None = None) -> Path: - """Transport a standalone SQLite participant's committed storage too. - - A paused process can still have committed data only in its WAL. Reading - the main file alone silently exports an older, internally valid database. - The typed artifact owner's SQLite backup still decides the committed - snapshot; this Docker adapter only transports its physical input while - the same writer barrier is held. A directory archive already includes it. - """ - from docker.errors import NotFound - - copied = _archive(container, remote, destination, deadline=deadline) - if not copied.is_file(): - return copied - with copied.open("rb") as source: - if source.read(16) != b"SQLite format 3\0": - return copied - # SHM is a reader index, rebuilt locally. WAL and rollback journals can - # affect committed state and must not be dropped or treated as optional - # when transport, permissions or link validation fails. - for suffix in ("-wal", "-journal"): - with tempfile.TemporaryDirectory(prefix=".sqlite-transport-", dir=destination) as stage: - try: - companion = _archive(container, remote + suffix, Path(stage) / "member", deadline=deadline) - except NotFound: - continue - if not companion.is_file(): - raise ValueError("SQLite companion must be a physical regular file") - companion.rename(copied.with_name(copied.name + suffix)) - return copied - - -def _worker_lock(client, container_id: str) -> Path: - # Controller banks and endpoint aliases may differ for the same daemon. - # Coordinate captures by this user on this host, not by artifact location. - daemon_id = client.info().get("ID") - if not isinstance(daemon_id, str) or not daemon_id: - raise ValueError("Docker daemon identity is required for worker capture") - root = Path.home() / ".cache" / "loopx" / "checkpoint-workers" - root.mkdir(parents=True, exist_ok=True, mode=0o700) - _physical(root) - info = root.stat() - if info.st_uid != os.getuid() or info.st_mode & 0o077: - raise ValueError("Worker capture lock directory must be private and owned by the current user") - daemon_key = hashlib.sha256(daemon_id.encode()).hexdigest() - return root / f"{daemon_key}-{container_id}" +from .checkpoint_controller import _controller, _physical, _remaining, capture def capture_worker(controller_file: Path, *, token: str, capture_id: str, destination: Path, request_file: Path, timeout: float = 30, seal_within_phase: bool = False, terminal_scheduler_phases: bool = False) -> dict: - """Freeze one explicitly owned worker and seal its files with native state. - - A fresh identity is required. Failed or uncertain attempts never recapture a - later worker into an earlier evaluator point. The original worker is resumed - before publishing the sealed result; an unsuccessful unpause is an error. - The supervisor can explicitly seal the private exported files after release - within their original Host phase. The prepared deadline still bounds every - live-source/native capture step; ordinary callers retain the total timeout. - """ - import docker - + """Freeze one EdgeBench worker and bind the native evaluator and phase.""" if not math.isfinite(timeout) or timeout <= 0: raise ValueError("Worker capture timeout must be finite and positive") deadline = time.monotonic() + timeout - request = _request(request_file) + request = validate_worker_request(json.loads(_physical(request_file).read_bytes())) if seal_within_phase and "handoff" not in request: raise ValueError("Post-release sealing requires an exact prepared handoff") if terminal_scheduler_phases and "handoff" in request: raise ValueError("Terminal phase discovery cannot change a prepared handoff") controller = _controller(controller_file) - destination = Path(destination) - _physical(destination.parent) - if (destination.resolve() != destination or destination.exists() or destination.is_symlink() or - not re.fullmatch(r"[0-9a-f]{32}", capture_id)): - raise ValueError("Worker capture requires a fresh physical destination and 32-hex identity") - if (controller["bank_root"] / capture_id).exists(): - raise ValueError("Existing native point cannot acquire later worker files") - qualify_export_storage(destination) - attempts = controller["bank_root"] / ".loopx-worker-attempts" - attempts.mkdir(exist_ok=True, mode=0o700) - _physical(attempts) - (attempts / capture_id).mkdir(mode=0o700) # First attempt, including failures, wins. - client = docker.from_env(timeout=_remaining(deadline)) - try: - with exclusive_file_lock(_worker_lock(client, request["container_id"]), timeout_seconds=_remaining(deadline), - operation="checkpoint_worker_capture"): - container = client.containers.get(request["container_id"]) - container.reload() - state = container.attrs["State"] - if container.id != request["container_id"] or not state["Running"] or state["Paused"]: - raise ValueError("Worker must be the exact running, unpaused container owned by this controller") - with tempfile.TemporaryDirectory(prefix=".worker-capture-", dir=destination.parent) as directory: - stage = Path(directory) - (stage / "workers").mkdir(mode=0o700) - components = [] - prepared = None - pause_may_have_committed = True - try: - try: - container.pause() - except docker.errors.APIError as exc: - # Docker's conflict response rejects this pause. The - # observed pause belongs to another actor; leave it alone. - if exc.response is not None and exc.response.status_code == 409: - pause_may_have_committed = False - raise - container.reload() - if not container.attrs["State"]["Paused"]: - raise RuntimeError("Worker writer barrier was not established") - if terminal_scheduler_phases: - from .terminal_capture import _scheduler_components_at_barrier - request = _validate_request({**request, "components": _scheduler_components_at_barrier( - container, request["components"], stage, deadline=deadline)}) - if "handoff" in request: - held = _archive(container, request["handoff"]["source_path"], stage / "held", deadline=deadline) - raw = held.read_bytes() - if hashlib.sha256(raw).hexdigest() != request["handoff"]["sha256"]: - raise ValueError("Worker handoff changed before its writer barrier") - prepared = json.loads(raw) - if (set(prepared) != {"capture_id", "components", "point_metadata", "source_sha256", "source_witness", "deadline_epoch"} or - prepared["capture_id"] != capture_id or prepared["components"] != request["components"] or - not isinstance(prepared["point_metadata"], dict) or not isinstance(prepared["source_witness"], dict) or - type(prepared["deadline_epoch"]) not in (int, float) or - not math.isfinite(prepared["deadline_epoch"]) or time.time() >= prepared["deadline_epoch"]): - raise ValueError("Worker handoff is expired or differs from the original capture") - if prepared is not None: - deadline = min(deadline, time.monotonic() + prepared["deadline_epoch"] - time.time()) - for entry in request["components"]: - client.api.timeout = _remaining(deadline) - try: - copied = _archive_component( - container, entry["source_path"], stage / "workers" / entry["id"], deadline=deadline) - except docker.errors.NotFound as error: - # Keep the provider exception for direct capture callers. - error.checkpoint_component_id = entry["id"] - raise - components.append({"id": entry["id"], "source_path": str(copied)}) - if prepared is not None: - observed = effect_runtime_result("runtime.checkpoint_artifacts", { - "action": "probe", "components": components, - }, timeout=_remaining(deadline), retry_safe=False, large_local_snapshot=True) - if observed["source_sha256"] != prepared["source_sha256"]: - changed = [] - for key, rows in observed["source_witness"].items(): - before = {row["path"]: row for row in prepared["source_witness"].get(key, [])} - after = {row["path"]: row for row in rows} - changed.extend(key + ":" + path for path in sorted(before.keys() | after.keys()) - if before.get(path) != after.get(path)) - raise ValueError("Worker source changed after native/research preparation: " + ", ".join(changed)) - if time.time() >= prepared["deadline_epoch"]: - raise TimeoutError("Worker handoff expired before evaluator capture") - phase = _archive(container, request["phase_deadline_path"], stage / "phase", deadline=deadline) - native = capture(controller_file, token=token, capture_id=capture_id, - destination=stage / "native", timeout=_remaining(deadline), phase_deadline_file=phase) - container.reload() - if not container.attrs["State"]["Running"] or not container.attrs["State"]["Paused"]: - raise RuntimeError("Worker writer barrier changed during capture") - _remaining(deadline) # Late native/barrier completion cannot gain sealing time. - finally: - # An uncertain/lost reply keeps this original attempt's - # cleanup obligation. Its identity cannot be retried later. - if pause_may_have_committed: - # Cleanup retains a bounded transport allowance even if - # the capture window is exhausted; never strand our writer. - client.api.timeout = timeout - container.reload() - if container.attrs["State"]["Paused"]: - container.unpause() - container.reload() - if container.attrs["State"]["Paused"] or not container.attrs["State"]["Running"]: - raise RuntimeError("Worker was not restored to running; inspect the original container before retry") - if seal_within_phase: - # Only immutable controller-owned exports remain. Do not - # spend the held-decision window hashing them after source - # release, or grant a new trial budget/cooldown. Reuse the - # original phase owner and its captured physical clock. - remaining_phase = _phase_budget(phase.read_bytes(), { - "captured_at_epoch": time.time()})["remaining_seconds"] - completion = controller_completion_deadline( - prepared["deadline_epoch"], time.time() + remaining_phase) - deadline = time.monotonic() + completion - time.time() - sealing_epoch_ms = (time.time() + _remaining(deadline)) * 1000 - native_component = _participant_id(components, "native_capsule") - components.append({"id": native_component, "source_path": str(stage / "native")}) - metadata = {**(prepared["point_metadata"] if prepared is not None else {}), "worker_file_snapshot": { - "container_id": container.id, "image_id": container.attrs["Image"], - "components": request["components"], "phase_deadline_path": request["phase_deadline_path"], - "native_capsule_sha256": native["sha256"], - "native_capsule_component_id": native_component, - "writer_barrier": "docker_pause", "process_memory_captured": False, - "native_cursor_qualified": False, - **({"handoff_sha256": request["handoff"]["sha256"], - "prepared_source_verified": True} if prepared is not None else {}), - }} - if prepared is not None: - metadata["writer_barrier"] = "docker_pause" - probe = effect_runtime_result("runtime.checkpoint_artifacts", {"action": "probe", "components": components, - "deadline_epoch_ms": sealing_epoch_ms}, - timeout=_remaining(deadline), retry_safe=False, large_local_snapshot=True) - return effect_runtime_result("runtime.checkpoint_artifacts", { - "action": "capture", "components": components, "destination": str(destination), - "expected_source_sha256": probe["source_sha256"], "point_metadata": metadata, - "deadline_epoch_ms": sealing_epoch_ms, - }, timeout=_remaining(deadline), retry_safe=False, large_local_snapshot=True) - finally: - client.close() + resolve_components = None + if terminal_scheduler_phases: + from .terminal_capture import _scheduler_components_at_barrier + + resolve_components = _scheduler_components_at_barrier + + def capture_native(target, phase, remaining): + return capture(controller_file, token=token, capture_id=capture_id, + destination=target, timeout=remaining, phase_deadline_file=phase) + + return capture_docker_worker( + capture_id=capture_id, destination=destination, request=request, + native_bank=controller["bank_root"], capture_native=capture_native, + timeout=_remaining(deadline), seal_within_phase=seal_within_phase, + components_at_barrier=resolve_components, + ) diff --git a/benchmark/edgebench/cli_source_activation.py b/benchmark/edgebench/cli_source_activation.py index 9c39d15aad..f208208efa 100644 --- a/benchmark/edgebench/cli_source_activation.py +++ b/benchmark/edgebench/cli_source_activation.py @@ -20,7 +20,7 @@ from loopx.control_plane.effect_runtime import effect_runtime_result from loopx.file_lock import exclusive_file_lock, write_json_atomic -from .checkpoint_worker import _worker_lock +from benchmark.runtime.docker_checkpoint import _worker_lock from .cli_source_probe import _archive_response, source_goal_control diff --git a/benchmark/edgebench/cli_source_probe.py b/benchmark/edgebench/cli_source_probe.py index 13a4b335a2..3b701e542b 100644 --- a/benchmark/edgebench/cli_source_probe.py +++ b/benchmark/edgebench/cli_source_probe.py @@ -16,7 +16,7 @@ import tarfile import tempfile -from .checkpoint_worker import _guest_path +from benchmark.runtime.docker_checkpoint import _guest_path def _canonical(value): diff --git a/benchmark/edgebench/stopped_recovery.py b/benchmark/edgebench/stopped_recovery.py index a1c85fd0ad..0d93197f06 100644 --- a/benchmark/edgebench/stopped_recovery.py +++ b/benchmark/edgebench/stopped_recovery.py @@ -20,7 +20,7 @@ import time from .checkpoint_controller import _remaining -from .checkpoint_worker import _archive_component, _guest_path +from benchmark.runtime.docker_checkpoint import _archive_component, _guest_path def file_inventory(root: Path) -> list[dict]: diff --git a/benchmark/edgebench/terminal_capture.py b/benchmark/edgebench/terminal_capture.py index e50e4944e9..d8116a2890 100644 --- a/benchmark/edgebench/terminal_capture.py +++ b/benchmark/edgebench/terminal_capture.py @@ -41,8 +41,8 @@ def terminal_components(declaration: dict, workspace: str) -> list[dict[str, str )) # Let the shared transport reject aliases/overlaps before launch, rather # than silently dropping an explicit external research participant. - from benchmark.edgebench.checkpoint_worker import _validate_request - _validate_request({"container_id": "0" * 64, "components": components, + from benchmark.runtime.docker_checkpoint import validate_worker_request + validate_worker_request({"container_id": "0" * 64, "components": components, "phase_deadline_path": _PHASE_DEADLINE}) return components @@ -54,7 +54,7 @@ def _scheduler_components_at_barrier(container, components, stage, *, deadline): import docker from benchmark.runtime.layout import _ACTIVE_PHASE_DEADLINE from benchmark.runtime.scheduler import phase_state_name, read_phase_deadline - from .checkpoint_worker import _archive + from benchmark.runtime.docker_checkpoint import _archive root = PurePosixPath(_CONTROL, "scheduler-state.json") if {"id": "terminal_scheduler", "source_path": str(root)} not in components: @@ -150,7 +150,7 @@ def finish(self, execution): self.receipt.update(capture_succeeded=True, capture=point) except Exception as error: # No remote message, output, account information or token is retained. - from .checkpoint_worker import capture_failure_fields + from benchmark.runtime.docker_checkpoint import capture_failure_fields self.receipt.update(capture_failure_fields(error)) finally: diff --git a/benchmark/runtime/RUNTIME.md b/benchmark/runtime/RUNTIME.md index 91ae2b658d..57ad31922b 100644 --- a/benchmark/runtime/RUNTIME.md +++ b/benchmark/runtime/RUNTIME.md @@ -625,6 +625,18 @@ send deadline and remain unknown even when the upstream has received some bytes. ## Results, timeout and recovery +Docker-based controllers can use +`benchmark.runtime.docker_checkpoint.capture_docker_worker` to verify a +prepared capture request, pause the exact source container, and seal its +files and evaluator capsule. Supply the native capture callback and its +bank from the benchmark adapter. The callback runs under the writer barrier +and returns the digest of the capsule it wrote at the assigned destination. +EdgeBench supplies its SForge evaluator and source phase deadline through +this entry point. An operator with a deadline-free solving phase can pass +a null phase path and explicitly allow an open phase. The worker capture +and sealing windows still bound the handoff. + + ### Live session observation Frequent capacity and liveness polls may use diff --git a/benchmark/runtime/docker_checkpoint.py b/benchmark/runtime/docker_checkpoint.py new file mode 100644 index 0000000000..b7a5b4fa78 --- /dev/null +++ b/benchmark/runtime/docker_checkpoint.py @@ -0,0 +1,439 @@ +"""Pause a Docker worker and seal its prepared files with native state. + +The benchmark supplies the native capture operation. Docker owns the writer +barrier, and the typed artifact owner verifies and seals the copied bytes. +""" +from __future__ import annotations + +from collections.abc import Callable +from contextlib import closing +from copy import copy +import hashlib +import json +import math +import os +import re +import tarfile +import tempfile +import time +from pathlib import Path, PurePosixPath + +from loopx.control_plane.effect_runtime import effect_runtime_result +from loopx.file_lock import exclusive_file_lock +from benchmark.runtime.checkpoint import _participant_id +from benchmark.runtime.checkpoint_handoff import controller_completion_deadline + +from .scheduler import read_phase_deadline + + +def _physical(path: Path) -> Path: + path = Path(path) + if not path.is_absolute() or path.resolve(strict=True) != path: + raise ValueError("Controller paths must be normalized, physical and absolute") + return path + + +def _remaining(deadline: float) -> float: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("Checkpoint capture window closed") + return remaining + + +_COMPONENT_ID = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{0,63}") + + +def capture_failure_fields(error: Exception) -> dict[str, str]: + """Keep a bounded diagnostic for a declared component's failed Docker read.""" + fields = {"error_kind": type(error).__name__} + component_id = getattr(error, "checkpoint_component_id", None) + if isinstance(component_id, str) and _COMPONENT_ID.fullmatch(component_id): + from docker.errors import NotFound + + if isinstance(error, NotFound): + fields["missing_component_id"] = component_id + return fields + + +def qualify_export_storage(destination: Path) -> None: + """Check filename fidelity before registration, without reserving a run. + + Linux source trees can contain case-only or Unicode-normalization-only + siblings. A shared host volume may silently merge these during extraction + or sealing, even when the controller itself runs on Linux. + """ + if not destination.is_absolute() or destination.resolve() != destination: + raise ValueError("Checkpoint export destination must be normalized, physical and absolute") + if destination.exists(): + raise ValueError("Checkpoint export destination must be fresh; preserve the existing bank") + parent = destination.parent + while not parent.exists(): + parent = parent.parent + _physical(parent) + names = ("Candidate", "candidate", "caf\u00e9", "cafe\u0301") + with tempfile.TemporaryDirectory(prefix=".checkpoint-storage-", dir=parent) as directory: + root = Path(directory) + try: + for index, name in enumerate(names): + with (root / name).open("xb") as stream: + stream.write(bytes([index])) + except FileExistsError as error: + raise ValueError("Checkpoint export storage must preserve case and Unicode filename distinctions; " + "select a compatible volume with --checkpoint-export-root") from error + if (set(entry.name for entry in root.iterdir()) != set(names) or + any((root / name).read_bytes() != bytes([index]) for index, name in enumerate(names))): + raise ValueError("Checkpoint export storage changed filename identities or bytes") + + +def validate_worker_request(value: dict, *, allow_open_phase: bool = False) -> dict: + if (not isinstance(value, dict) or + set(value) not in ({"container_id", "components", "phase_deadline_path"}, + {"container_id", "components", "phase_deadline_path", "handoff"}) or + not re.fullmatch(r"[0-9a-f]{64}", value.get("container_id", "")) or + not isinstance(value["components"], list) or not value["components"]): + raise ValueError("Worker request requires a full container ID, explicit components and phase_deadline_path") + ids = set() + paths = [] + for entry in value["components"]: + if (not isinstance(entry, dict) or set(entry) != {"id", "source_path"} or + not isinstance(entry["id"], str) or + not _COMPONENT_ID.fullmatch(entry["id"]) or entry["id"] in ids): + raise ValueError("Worker components require unique IDs") + ids.add(entry["id"]) + paths.append(_guest_path(entry["source_path"])) + if value["phase_deadline_path"] is not None or not allow_open_phase: + paths.append(_guest_path(value["phase_deadline_path"])) + if "handoff" in value: + handoff = value["handoff"] + if (not isinstance(handoff, dict) or set(handoff) != {"source_path", "sha256"} or + not re.fullmatch(r"[0-9a-f]{64}", handoff.get("sha256", ""))): + raise ValueError("Worker handoff requires the exact private request digest") + paths.append(_guest_path(handoff["source_path"])) + if any(a == b or a in b.parents or b in a.parents + for index, a in enumerate(paths) for b in paths[index + 1:]): + raise ValueError("Worker components and phase clock must not overlap") + return value + + +def _guest_path(value: str) -> PurePosixPath: + if not isinstance(value, str): + raise ValueError("Worker paths must be normalized absolute paths below root") + path = PurePosixPath(value) + if not path.is_absolute() or str(path) != value or value == "/" or ".." in path.parts: + raise ValueError("Worker paths must be normalized absolute paths below root") + return path + + +class _ArchiveReader: + """Check the shared clock during tar headers and large-file copy reads.""" + def __init__(self, raw, deadline): + self.raw, self.deadline = raw, deadline + + def __getattr__(self, name): + return getattr(self.raw, name) + + def read(self, *args): + _remaining(self.deadline) + block = self.raw.read(*args) + _remaining(self.deadline) + return block + + +def _archive(container, remote: str, destination: Path, *, deadline: float | None = None) -> Path: + """Read through the daemon, rejecting symlinks/devices and archive traversal. + + Docker's archive reader works while paused, unlike exec-based file readers. + Keep only the declared object, not parent filesystem or process credentials. + """ + from docker.utils import decode_json_header + + def check(): + if deadline is not None: + _remaining(deadline) + + check() + # The SDK's get_archive hides its Response behind a lazy generator. Closing + # that generator alone need not close an unread HTTP body. Own the response + # so every early exit releases the daemon reader before writer cleanup. + api = container.client.api + response = api._get(api._url("/containers/{0}/archive", container.id), + params={"path": remote}, stream=True, headers={"Accept-Encoding": "identity"}, + **({"timeout": _remaining(deadline)} if deadline is not None else {})) + with closing(response): + check() + api._raise_for_status(response) + stat = decode_json_header(response.headers["x-docker-container-path-stat"]) + if stat.get("linkTarget"): + raise ValueError("Worker component must not be a symlink") + destination.mkdir(mode=0o700) + # The explicitly provisioned export volume owns both the raw transfer + # and extracted files; a full ambient /tmp must not defeat that choice. + with tempfile.TemporaryFile(dir=destination) as raw: + # _stream_raw_result is Docker's long-running-process reader: it + # disables socket timeouts. Archive IO must keep the request's read + # timeout as well as check the shared absolute clock on progress. + for chunk in response.iter_content(64 * 1024, decode_unicode=False): + check() + raw.write(chunk) + check() + raw.seek(0) + with tarfile.open(fileobj=_ArchiveReader(raw, deadline) if deadline is not None else raw) as archive: + members = [] + root = PurePosixPath(remote).name + names = {} + for member in archive: + check() + path = PurePosixPath(member.name) + if (path.is_absolute() or ".." in path.parts or not path.parts or + path.parts[0] != root or str(path) in names or + not (member.isfile() or member.isdir() or member.islnk())): + raise ValueError("Worker archive contains undeclared, linked or special members") + names[str(path)] = member + members.append(member) + + # The typed artifact owner observes regular files, not inode + # sharing. Docker encodes a second name for the same inode as a + # tar hardlink (common in build outputs). Materialize its bytes + # from an inventoried regular member; never resolve a link on + # the host filesystem or permit a link chain/symlink escape. + extracted = [] + for member in members: + check() + if member.islnk(): + target_path = PurePosixPath(member.linkname) + target = names.get(str(target_path)) + if (target_path.is_absolute() or ".." in target_path.parts or + target is None or not target.isfile()): + raise ValueError("Worker archive contains undeclared, linked or special members") + regular = copy(target) + regular.name, regular.mode = member.name, member.mode + extracted.append(regular) + else: + extracted.append(member) + + def filtered(member, path): + check() + result = tarfile.data_filter(member, path) + check() + return result + + archive.extractall(destination, members=extracted, filter=filtered) + # The data filter intentionally drops directory modes. This is a + # private raw transport witness, so restore only ordinary permission + # bits after safe extraction; never restore owners or special bits. + # Otherwise a private 0700 authority directory becomes 0755 and a + # matching frozen source is falsely rejected as having changed. + for member in sorted(members, key=lambda entry: len(PurePosixPath(entry.name).parts), reverse=True): + check() + (destination / member.name).chmod(member.mode & 0o777) + check() + return _physical(destination / root) + + +def _archive_component(container, remote: str, destination: Path, *, deadline: float | None = None) -> Path: + """Transport a standalone SQLite participant's committed storage too. + + A paused process can still have committed data only in its WAL. Reading + the main file alone silently exports an older, internally valid database. + The typed artifact owner's SQLite backup still decides the committed + snapshot; this Docker adapter only transports its physical input while + the same writer barrier is held. A directory archive already includes it. + """ + from docker.errors import NotFound + + copied = _archive(container, remote, destination, deadline=deadline) + if not copied.is_file(): + return copied + with copied.open("rb") as source: + if source.read(16) != b"SQLite format 3\0": + return copied + # SHM is a reader index, rebuilt locally. WAL and rollback journals can + # affect committed state and must not be dropped or treated as optional + # when transport, permissions or link validation fails. + for suffix in ("-wal", "-journal"): + with tempfile.TemporaryDirectory(prefix=".sqlite-transport-", dir=destination) as stage: + try: + companion = _archive(container, remote + suffix, Path(stage) / "member", deadline=deadline) + except NotFound: + continue + if not companion.is_file(): + raise ValueError("SQLite companion must be a physical regular file") + companion.rename(copied.with_name(copied.name + suffix)) + return copied + + +def _worker_lock(client, container_id: str) -> Path: + # Controller banks and endpoint aliases may differ for the same daemon. + # Coordinate captures by this user on this host, not by artifact location. + daemon_id = client.info().get("ID") + if not isinstance(daemon_id, str) or not daemon_id: + raise ValueError("Docker daemon identity is required for worker capture") + root = Path.home() / ".cache" / "loopx" / "checkpoint-workers" + root.mkdir(parents=True, exist_ok=True, mode=0o700) + _physical(root) + info = root.stat() + if info.st_uid != os.getuid() or info.st_mode & 0o077: + raise ValueError("Worker capture lock directory must be private and owned by the current user") + daemon_key = hashlib.sha256(daemon_id.encode()).hexdigest() + return root / f"{daemon_key}-{container_id}" + + +def capture_docker_worker( + *, capture_id: str, destination: Path, request: dict, native_bank: Path, + capture_native: Callable[[Path, Path | None, float], dict], + timeout: float = 30, seal_within_phase: bool = False, allow_open_phase: bool = False, + components_at_barrier: Callable | None = None, docker_host: str | None = None, +) -> dict: + """Freeze the selected worker, check its handoff and seal one fresh point. + + The callback runs while the exact worker is paused. It must materialize its + native capsule at the given path and return its verified artifact digest. + Capture is bounded even when the source has no solver deadline. + """ + import docker + + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("Worker capture timeout must be finite and positive") + deadline = time.monotonic() + timeout + request = validate_worker_request(request, allow_open_phase=allow_open_phase) + if seal_within_phase and "handoff" not in request: + raise ValueError("Post-release sealing requires an exact prepared handoff") + if components_at_barrier is not None and "handoff" in request: + raise ValueError("Terminal phase discovery cannot change a prepared handoff") + native_bank = _physical(native_bank) + destination = Path(destination) + _physical(destination.parent) + if (destination.resolve() != destination or destination.exists() or destination.is_symlink() or + not re.fullmatch(r"[0-9a-f]{32}", capture_id)): + raise ValueError("Worker capture requires a fresh physical destination and 32-hex identity") + if (native_bank / capture_id).exists(): + raise ValueError("Existing native point cannot acquire later worker files") + qualify_export_storage(destination) + attempts = native_bank / ".loopx-worker-attempts" + attempts.mkdir(exist_ok=True, mode=0o700) + _physical(attempts) + (attempts / capture_id).mkdir(mode=0o700) # First attempt, including failures, wins. + client = ( + docker.from_env(timeout=_remaining(deadline)) if docker_host is None + else docker.DockerClient(base_url=docker_host, timeout=_remaining(deadline)) + ) + try: + with exclusive_file_lock(_worker_lock(client, request["container_id"]), timeout_seconds=_remaining(deadline), + operation="checkpoint_worker_capture"): + container = client.containers.get(request["container_id"]) + container.reload() + state = container.attrs["State"] + if container.id != request["container_id"] or not state["Running"] or state["Paused"]: + raise ValueError("Worker must be the exact running, unpaused container owned by this controller") + with tempfile.TemporaryDirectory(prefix=".worker-capture-", dir=destination.parent) as directory: + stage = Path(directory) + (stage / "workers").mkdir(mode=0o700) + components = [] + prepared = None + pause_may_have_committed = True + try: + try: + container.pause() + except docker.errors.APIError as exc: + # Docker's conflict response rejects this pause. The + # observed pause belongs to another actor; leave it alone. + if exc.response is not None and exc.response.status_code == 409: + pause_may_have_committed = False + raise + container.reload() + if not container.attrs["State"]["Paused"]: + raise RuntimeError("Worker writer barrier was not established") + if components_at_barrier is not None: + request = validate_worker_request({ + **request, "components": components_at_barrier( + container, request["components"], stage, deadline=deadline), + }, allow_open_phase=allow_open_phase) + if "handoff" in request: + held = _archive(container, request["handoff"]["source_path"], stage / "held", deadline=deadline) + raw = held.read_bytes() + if hashlib.sha256(raw).hexdigest() != request["handoff"]["sha256"]: + raise ValueError("Worker handoff changed before its writer barrier") + prepared = json.loads(raw) + if (set(prepared) != {"capture_id", "components", "point_metadata", "source_sha256", "source_witness", "deadline_epoch"} or + prepared["capture_id"] != capture_id or prepared["components"] != request["components"] or + not isinstance(prepared["point_metadata"], dict) or not isinstance(prepared["source_witness"], dict) or + type(prepared["deadline_epoch"]) not in (int, float) or + not math.isfinite(prepared["deadline_epoch"]) or time.time() >= prepared["deadline_epoch"]): + raise ValueError("Worker handoff is expired or differs from the original capture") + if prepared is not None: + deadline = min(deadline, time.monotonic() + prepared["deadline_epoch"] - time.time()) + for entry in request["components"]: + client.api.timeout = _remaining(deadline) + try: + copied = _archive_component( + container, entry["source_path"], stage / "workers" / entry["id"], deadline=deadline) + except docker.errors.NotFound as error: + # Keep the provider exception for direct capture callers. + error.checkpoint_component_id = entry["id"] + raise + components.append({"id": entry["id"], "source_path": str(copied)}) + if prepared is not None: + observed = effect_runtime_result("runtime.checkpoint_artifacts", { + "action": "probe", "components": components, + }, timeout=_remaining(deadline), retry_safe=False, large_local_snapshot=True) + if observed["source_sha256"] != prepared["source_sha256"]: + changed = [] + for key, rows in observed["source_witness"].items(): + before = {row["path"]: row for row in prepared["source_witness"].get(key, [])} + after = {row["path"]: row for row in rows} + changed.extend(key + ":" + path for path in sorted(before.keys() | after.keys()) + if before.get(path) != after.get(path)) + raise ValueError("Worker source changed after native/research preparation: " + ", ".join(changed)) + if time.time() >= prepared["deadline_epoch"]: + raise TimeoutError("Worker handoff expired before evaluator capture") + phase_path = request["phase_deadline_path"] + phase = _archive(container, phase_path, stage / "phase", deadline=deadline) if phase_path else None + native = capture_native(stage / "native", phase, _remaining(deadline)) + container.reload() + if not container.attrs["State"]["Running"] or not container.attrs["State"]["Paused"]: + raise RuntimeError("Worker writer barrier changed during capture") + _remaining(deadline) # Late native/barrier completion cannot gain sealing time. + finally: + # An uncertain/lost reply keeps this original attempt's + # cleanup obligation. Its identity cannot be retried later. + if pause_may_have_committed: + # Cleanup retains a bounded transport allowance even if + # the capture window is exhausted; never strand our writer. + client.api.timeout = timeout + container.reload() + if container.attrs["State"]["Paused"]: + container.unpause() + container.reload() + if container.attrs["State"]["Paused"] or not container.attrs["State"]["Running"]: + raise RuntimeError("Worker was not restored to running; inspect the original container before retry") + if seal_within_phase: + # The released source is no longer read. The same capture + # request owns the sealing window and any original phase. + phase_end = read_phase_deadline(phase.read_bytes()) if phase is not None else None + completion = controller_completion_deadline(prepared["deadline_epoch"], phase_end) + deadline = time.monotonic() + completion - time.time() + sealing_epoch_ms = (time.time() + _remaining(deadline)) * 1000 + native_component = _participant_id(components, "native_capsule") + components.append({"id": native_component, "source_path": str(stage / "native")}) + metadata = {**(prepared["point_metadata"] if prepared is not None else {}), "worker_file_snapshot": { + "container_id": container.id, "image_id": container.attrs["Image"], + "components": request["components"], "phase_deadline_path": request["phase_deadline_path"], + "native_capsule_sha256": native["sha256"], + "native_capsule_component_id": native_component, + "writer_barrier": "docker_pause", "process_memory_captured": False, + "native_cursor_qualified": False, + **({"handoff_sha256": request["handoff"]["sha256"], + "prepared_source_verified": True} if prepared is not None else {}), + }} + if prepared is not None: + metadata["writer_barrier"] = "docker_pause" + probe = effect_runtime_result("runtime.checkpoint_artifacts", {"action": "probe", "components": components, + "deadline_epoch_ms": sealing_epoch_ms}, + timeout=_remaining(deadline), retry_safe=False, large_local_snapshot=True) + return effect_runtime_result("runtime.checkpoint_artifacts", { + "action": "capture", "components": components, "destination": str(destination), + "expected_source_sha256": probe["source_sha256"], "point_metadata": metadata, + "deadline_epoch_ms": sealing_epoch_ms, + }, timeout=_remaining(deadline), retry_safe=False, large_local_snapshot=True) + finally: + client.close() diff --git a/benchmark/tests/qualify_checkpoint_joint.py b/benchmark/tests/qualify_checkpoint_joint.py index c52d1e87ce..2978e86b3d 100644 --- a/benchmark/tests/qualify_checkpoint_joint.py +++ b/benchmark/tests/qualify_checkpoint_joint.py @@ -67,7 +67,7 @@ def require_one_pass(report): def _stage_source_configuration(container, native, destination): - from benchmark.edgebench.checkpoint_worker import _archive + from benchmark.runtime.docker_checkpoint import _archive # The source guest can be root while its controller is an ordinary user. # Read only the bound config through the existing daemon archive seam; @@ -233,7 +233,7 @@ def guest(command, volumes, env, log, *, capture_supervisor=None): # The real source writer may be root. Its private pytest # directory is intentionally unreadable to this controller; # stage only the declared result through the daemon seam. - from benchmark.edgebench.checkpoint_worker import _archive + from benchmark.runtime.docker_checkpoint import _archive report = _archive(container, "/qualification/context-delivery-source.json", output / "source-context-result.json") report.chmod(0o600) diff --git a/benchmark/tests/test_checkpoint_archive_transport.py b/benchmark/tests/test_checkpoint_archive_transport.py index 0e5b41ec86..839ddf8af6 100644 --- a/benchmark/tests/test_checkpoint_archive_transport.py +++ b/benchmark/tests/test_checkpoint_archive_transport.py @@ -11,7 +11,7 @@ import pytest -from benchmark.edgebench import checkpoint_worker as worker +from benchmark.runtime import docker_checkpoint as worker class ArchiveStream: diff --git a/benchmark/tests/test_checkpoint_diagnostics.py b/benchmark/tests/test_checkpoint_diagnostics.py index b76259e08b..e686891d7a 100644 --- a/benchmark/tests/test_checkpoint_diagnostics.py +++ b/benchmark/tests/test_checkpoint_diagnostics.py @@ -17,7 +17,8 @@ import pytest from benchmark.edgebench import checkpoint_supervisor as supervisor_module -from benchmark.edgebench import checkpoint_worker as worker_module +from benchmark.edgebench import checkpoint_worker as edgebench_worker +from benchmark.runtime import docker_checkpoint as worker_module from loopx.control_plane.effect_runtime import effect_runtime_result, restart_effect_runtime @@ -130,7 +131,7 @@ def source_archive(*args, **kwargs): def native_capture(*args, **kwargs): pytest.fail("A failed source probe must not reach native capture") - monkeypatch.setattr(worker_module, "capture", native_capture) + monkeypatch.setattr(edgebench_worker, "capture", native_capture) malformed_root = tmp_path / "synthetic-private-path" malformed_root.mkdir() if fault == "directory": diff --git a/benchmark/tests/test_checkpoint_worker.py b/benchmark/tests/test_checkpoint_worker.py index 99e717a51b..560aad0f00 100644 --- a/benchmark/tests/test_checkpoint_worker.py +++ b/benchmark/tests/test_checkpoint_worker.py @@ -104,7 +104,7 @@ def test_ordinary_controller_stages_root_private_native_configuration(writer, tm def test_archive_keeps_private_directory_modes_without_special_bits(writer, tmp_path): - from benchmark.edgebench.checkpoint_worker import _archive + from benchmark.runtime.docker_checkpoint import _archive container, _ = writer assert container.exec_run(["python", "-c", "from pathlib import Path; " "p=Path('/private-state'); p.mkdir(mode=0o700); " @@ -118,7 +118,7 @@ def test_archive_keeps_private_directory_modes_without_special_bits(writer, tmp_ "retention", "retention_default", "retention_off", "retention_pin", "retention_failure", "continued_clock"]) def test_supervisor_captures_only_the_prepared_worker_boundary(controllers, writer, tmp_path, monkeypatch, fault): from loopx.control_plane.effect_runtime import effect_runtime_result - from benchmark.edgebench.checkpoint_worker import _archive + from benchmark.runtime.docker_checkpoint import _archive from benchmark.edgebench.checkpoint_supervisor import CheckpointSupervisor source, _, first, _, files, grades = controllers @@ -156,11 +156,11 @@ def test_supervisor_captures_only_the_prepared_worker_boundary(controllers, writ from benchmark.edgebench import checkpoint_supervisor original = checkpoint_supervisor._archive reads = [] - def interrupted_archive(*args): + def interrupted_archive(*args, **kwargs): reads.append(True) if len(reads) == 1: raise tarfile.ReadError("request removed during Docker archive") - return original(*args) + return original(*args, **kwargs) monkeypatch.setattr(checkpoint_supervisor, "_archive", interrupted_archive) if fault == "late_native": from benchmark.edgebench import checkpoint_worker @@ -258,7 +258,7 @@ def with_old_point(*args, **kwargs): @pytest.mark.parametrize("seal_within_phase,expire_phase", [(True, False), (False, False), (True, True)]) def test_post_release_sealing_preserves_captured_bytes_and_original_phase( controllers, writer, tmp_path, monkeypatch, seal_within_phase, expire_phase): - from benchmark.edgebench import checkpoint_worker + from benchmark.runtime import docker_checkpoint as checkpoint_worker from loopx.control_plane.effect_runtime import effect_runtime_result source, _, first, _, files, grades = controllers @@ -339,7 +339,7 @@ def test_paused_sqlite_capture_retains_committed_wal_not_pending_transaction( def test_sqlite_companion_failure_cannot_publish_an_older_valid_database( controllers, writer, tmp_path, monkeypatch, fault): from docker.errors import APIError - from benchmark.edgebench import checkpoint_worker + from benchmark.runtime import docker_checkpoint as checkpoint_worker source, _, first, _, files, grades = controllers container, request = writer @@ -494,7 +494,7 @@ def test_full_export_volume_releases_real_archive_and_writer( controllers, writer, tmp_path, monkeypatch): import errno from contextlib import contextmanager - from benchmark.edgebench import checkpoint_worker as worker + from benchmark.runtime import docker_checkpoint as worker source, _, first, _, files, _ = controllers container, request = writer @@ -624,7 +624,7 @@ def unexpected_native_call(*args, **kwargs): def test_capture_budget_is_shared_and_exhaustion_releases_writer(controllers, writer, tmp_path, monkeypatch): - from benchmark.edgebench import checkpoint_worker + from benchmark.runtime import docker_checkpoint as checkpoint_worker source, _, first, _, files, grades = controllers container, request = writer @@ -689,7 +689,7 @@ def test_one_progressing_archive_expires_without_stranding_writer( controllers, writer, tmp_path, monkeypatch, stage): from contextlib import contextmanager import requests - from benchmark.edgebench import checkpoint_worker as worker + from benchmark.runtime import docker_checkpoint as worker source, _, first, _, files, grades = controllers container, request = writer diff --git a/benchmark/tests/test_docker_checkpoint.py b/benchmark/tests/test_docker_checkpoint.py new file mode 100644 index 0000000000..6d925008f1 --- /dev/null +++ b/benchmark/tests/test_docker_checkpoint.py @@ -0,0 +1,53 @@ +"""A real Docker capture with an evaluator that has no solver-phase clock.""" +import json +import os +from uuid import uuid4 + +import pytest + +from benchmark.runtime.docker_checkpoint import capture_docker_worker, validate_worker_request +from benchmark.tests import test_checkpoint_worker +from loopx.control_plane.effect_runtime import effect_runtime_result + + +writer = test_checkpoint_worker.writer + + +@pytest.mark.skipif(os.environ.get("LOOPX_EDGEBENCH_DOCKER_SMOKE") != "1", + reason="Requires explicit isolated Docker writer-barrier qualification") +def test_callback_seals_open_phase_with_shared_writer_barrier(writer, tmp_path): + source, path = writer + request = json.loads(path.read_bytes()) + request["phase_deadline_path"] = None + with pytest.raises(ValueError, match="Worker paths"): + validate_worker_request(request) + + opportunity = tmp_path / "train-calls" + opportunity.write_text("3") + bank = tmp_path / "bank" + bank.mkdir() + destination = tmp_path / "point" + + def capture_native(target, phase, remaining): + source.reload() + assert source.attrs["State"]["Paused"] and phase is None and 0 < remaining <= 30 + probe = effect_runtime_result("runtime.checkpoint_artifacts", { + "action": "probe", "components": [{"id": "train_calls", "source_path": str(opportunity)}], + }) + return effect_runtime_result("runtime.checkpoint_artifacts", { + "action": "capture", "destination": str(target), "expected_source_sha256": probe["source_sha256"], + "components": [{"id": "train_calls", "source_path": str(opportunity)}], "point_metadata": {}, + }) + + receipt = capture_docker_worker( + capture_id=uuid4().hex, destination=destination, native_bank=bank, + request=request, capture_native=capture_native, allow_open_phase=True, + ) + assert receipt["integrity_verified"] + point = json.loads((destination / "checkpoint.json").read_bytes()) + metadata = point["point_metadata"]["worker_file_snapshot"] + capsule = destination / "components" / metadata["native_capsule_component_id"] + assert metadata["phase_deadline_path"] is None + assert (capsule / "components/train_calls").read_text() == "3" + source.reload() + assert source.attrs["State"]["Running"] and not source.attrs["State"]["Paused"] diff --git a/benchmark/tests/test_scheduler_phase_resume.py b/benchmark/tests/test_scheduler_phase_resume.py index 6ff48ba5cb..43bec5653f 100644 --- a/benchmark/tests/test_scheduler_phase_resume.py +++ b/benchmark/tests/test_scheduler_phase_resume.py @@ -21,7 +21,7 @@ def test_scheduler_continuation_clock_reaches_both_host_capture_paths(tmp_path, import docker from benchmark.edgebench import checkpoint_supervisor as host from benchmark.edgebench.checkpoint_controller import _phase_budget - from benchmark.edgebench.checkpoint_worker import _archive + from benchmark.runtime.docker_checkpoint import _archive from benchmark.runtime.layout import _PHASE_DEADLINE client = docker.from_env() diff --git a/benchmark/tests/test_terminal_preservation.py b/benchmark/tests/test_terminal_preservation.py index acf4ae678b..16eb4d8e29 100644 --- a/benchmark/tests/test_terminal_preservation.py +++ b/benchmark/tests/test_terminal_preservation.py @@ -452,7 +452,7 @@ def archive(container, remote, destination, *, deadline): raise FileNotFoundError(remote) destination.write_bytes(files[remote]) return destination - monkeypatch.setattr("benchmark.edgebench.checkpoint_worker._archive", archive) + monkeypatch.setattr("benchmark.runtime.docker_checkpoint._archive", archive) components = terminal_components({"components": []}, "/data") if fault: with pytest.raises((ValueError, FileNotFoundError)):