diff --git a/benchmark/LHTB/.env.example b/benchmark/LHTB/.env.example index 75ddb9c516..211cc7bb51 100644 --- a/benchmark/LHTB/.env.example +++ b/benchmark/LHTB/.env.example @@ -11,7 +11,6 @@ AGENT_TIMEOUT_SEC=5400 # The scheduler must finish before Harbor's outer agent timeout. LOOPX_SCHEDULER_TIMEOUT_SEC=5080 -LOOPX_WAKE_TIMEOUT_SEC=4800 LOOPX_CODEX_TURN_TIMEOUT_SEC=4700 CODEX_BIN=/absolute/path/to/vendor/x86_64-unknown-linux-musl/bin/codex diff --git a/benchmark/LHTB/README.md b/benchmark/LHTB/README.md index 0e8408c601..8d8bb5f582 100644 --- a/benchmark/LHTB/README.md +++ b/benchmark/LHTB/README.md @@ -28,7 +28,7 @@ owning a second copy. It does not use the app-server heartbeat agent. ## Run Prerequisites are an LHTB checkout with its Harbor virtual environment, Docker, -the native Codex bundle, a portable Python 3.11+ tree, and Node 22.6+. Copy +the native Codex bundle, a portable Python 3.11+ tree, and Node 22.18.0+. Copy `.env.example` to `.env`, set `LHTB_ROOT`, the model gateway, and local runtime paths, then run from this directory. @@ -70,29 +70,30 @@ stage native Codex + current LoopX source/profile -> expose staged Node 22 through BASH_ENV for Codex login shells -> create trial-local registry/runtime/task document -> bootstrap one trial-local Goal - -> Harbor adapter marks the connection provider-prevalidated - -> skip the generic repo-intake onboarding Todo - -> register lhtb-codex-heartbeat + -> register benchmark-agent -> configure replan_after_completed_todos=3 and read it back -> add the current Harbor phase as a claimed advancement Todo -> start LoopX external_scheduler_worker.py -> quota should-run --runtime-profile generic_cli - -> when allowed, invoke wake_once.py + -> when allowed, invoke benchmark.runtime.worker -> create a unique TURN_ID -> heartbeat-prompt --thin --runtime-profile generic_cli -> require ok=true and a non-empty current task_body - -> create a new CODEX_HOME + -> use the isolated trial CODEX_HOME -> codex exec --json, with task_body on stdin -> save JSONL/receipt and any emitted session files - -> delete the temporary CODEX_HOME + -> retain sessions for the trial -> obey LoopX local_scheduler wait/stop hints -> Harbor runs interim/final verifier and owns trial termination ``` -There is no `codex exec resume` path. A new Codex model conversation starts on -each wake. Only the task workspace and the trial-local LoopX registry/runtime -persist across wakes. Each wake's `invocation.json` records -`fresh_codex_exec=true`, `resume=false`, and its unique Turn ID. +The default heartbeat mode starts a new model conversation on each wake. The +workspace, Codex home and trial-local LoopX state persist. This replaces the old +per-wake home and is a disclosed behavior change. Memory generation/injection +are disabled. The [shared runtime](../runtime/RUNTIME.md) also supports governed +Turn fresh/resume and native Goal modes; historical results retain their original +configuration and do not describe these new combinations. Each wake receipt +records the requested mode/context and its unique identity. Codex tool commands run through a login shell, which can replace the inherited `PATH`. The adapter supplies a trial-local `BASH_ENV` that prepends the staged @@ -105,11 +106,29 @@ trial-local goal document, and the selected P0 Todo points the model to that document. Registries are inside their own task containers, so no Goal, Todo, or scheduler state is shared between the 46 trials. -The Harbor adapter has already validated the project bridge and writes the -benchmark phase as an explicit P0 Todo. Bootstrap therefore uses -`--no-onboarding-scan --onboarding-connection-validation -provider-prevalidated`. This suppresses the unrelated generic repo-intake Todo; -it does not suppress successor Todos created while solving the LHTB task. +The adapter registers the project through the current public bootstrap CLI and +adds the benchmark phase as an explicit P0 Todo. Retired onboarding flags are +not replayed; bootstrap and Todo lifecycle follow the installed product version. + +## Shared execution configuration + +`LOOPX_EXECUTION_MODE` selects `plain`, `native-goal`, `heartbeat` (default), +`turn` or `loopx-goal`. `LOOPX_ITERATION_CONTEXT` defaults to `fresh`; only Turn +accepts `resume-if-available`. Turn also requires `LOOPX_VALIDATION_COMMAND_JSON`, +an argv array for the independently protected task validator. No generic +benchmark scoring or hidden-verifier feedback is introduced. + +`LOOPX_TASK_ENTRY=seeded-todo` preserves the generic phase Todo default. +`LOOPX_TASK_ENTRY=loopx-planned` invokes the product planning checkpoint before +heartbeat, Turn or LoopX Goal execution. `LOOPX_PLANNING_TIMEOUT_SEC` defaults +to 300 and consumes the existing phase budget. Both entry policies preserve +existing waits when new phases arrive. See the shared runtime for session and +planning-readback semantics. + +Model and effort defaults remain unchanged but may be selected explicitly. +`run.sh prepare` performs networking/Harbor preparation. `preflight` now checks +existing preparation without patching Harbor or creating a network. Smoke/full +runs still prepare the environment as part of the authorized launch. ## Replan cadence @@ -117,7 +136,7 @@ The runner applies and reads back: ```bash loopx configure-goal \ - --goal-id lhtb-heartbeat-goal \ + --goal-id benchmark-goal \ --execution-replan-after-todos 3 \ --execute ``` @@ -200,7 +219,7 @@ authoritative captured stream when it does not. - `configs/heartbeat-generic-cli.yaml`: immutable 46-task template. - `agents/codex_loopx_heartbeat.py`: Harbor lifecycle and LoopX Goal setup. - `../swe-marathon/agents/codex_offline.py`: shared native Codex staging. -- `runtime/wake_once.py`: unique Turn, thin heartbeat body, fresh Codex exec. +- `../runtime/worker.py`: unique Turn, thin heartbeat body, fresh Codex exec. - `scripts/preflight.py`: fail-closed parity and safety checks. - `harbor_patch/`: opt-in model-only Docker networking patch. - `verifier-images/`: the two task-declared separate verifier images. diff --git a/benchmark/LHTB/agents/codex_loopx_heartbeat.py b/benchmark/LHTB/agents/codex_loopx_heartbeat.py index b0c1b9b950..2a270ab9e4 100644 --- a/benchmark/LHTB/agents/codex_loopx_heartbeat.py +++ b/benchmark/LHTB/agents/codex_loopx_heartbeat.py @@ -1,579 +1,7 @@ -"""Harbor agent for LoopX generic_cli heartbeat + fresh Codex exec wakes.""" - -from __future__ import annotations - -import json -import os -import re -import shlex -import subprocess -import tempfile -from pathlib import Path -from typing import Iterable - -from harbor.agents.installed.base import with_prompt_template -from harbor.environments.base import BaseEnvironment -from harbor.models.agent.context import AgentContext -from harbor.models.trajectories import FinalMetrics, Trajectory -from harbor.utils.trajectory_utils import format_trajectory_json - -from codex_offline import CodexOffline - - -_ROOT = "/opt/loopx-lhtb" -_SRC = f"{_ROOT}/source" -_PYTHON = f"{_ROOT}/python" -_NODE = f"{_ROOT}/node" -_PROFILE = f"{_ROOT}/profile" -_PROFILE_HOME = f"{_PROFILE}/home" -_SHARED_CODEX_HOME = f"{_PROFILE}/codex-home" -_SHARED_SKILLS = f"{_SHARED_CODEX_HOME}/skills" -_CLI = f"{_PROFILE}/bin/loopx" -_CONTROL = f"{_ROOT}/control" -_REGISTRY = f"{_CONTROL}/registry.json" -_LOOPX_RUNTIME = f"{_ROOT}/state/runtime" -_SCHEDULER_STATE = f"{_CONTROL}/scheduler-state.json" -_TASK_DOC = f"{_CONTROL}/task.md" -_BASH_ENV = f"{_CONTROL}/bash-env" -_TURN_ROOT = f"{_ROOT}/turns" -_WAKE_SCRIPT = f"{_ROOT}/runtime/wake_once.py" -_WAKE_LOG_DIR = "/logs/agent/wakes" -_GOAL_ID = "lhtb-heartbeat-goal" -_AGENT_ID = "lhtb-codex-heartbeat" -_REPLAN_AFTER_TODOS = 3 - - -class LoopxHeartbeatCodex(CodexOffline): - """One independent LoopX control plane per Harbor trial.""" - - _phase_number = 0 +"""Compatibility entry for existing LHTB Harbor configs.""" +from benchmark.runtime.harbor import BenchmarkCodex +class LoopxHeartbeatCodex(BenchmarkCodex): @staticmethod def name() -> str: return "loopx-generic-cli-heartbeat-codex" - - def _container_id(self, environment: BaseEnvironment) -> str: - from harbor.environments.docker.docker import ( - _sanitize_docker_compose_project_name, - ) - - project = _sanitize_docker_compose_project_name(environment.session_id) - completed = subprocess.run( - [ - "docker", "ps", "-q", - "--filter", f"label=com.docker.compose.project={project}", - "--filter", "label=com.docker.compose.service=main", - ], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=60, - check=False, - ) - ids = completed.stdout.split() - if len(ids) != 1: - raise RuntimeError( - f"expected one main container for compose project {project}, got {ids}" - ) - return ids[0] - - @staticmethod - def _copy_tree(container_id: str, source: Path, destination: str) -> None: - if not source.is_dir(): - raise FileNotFoundError(f"required directory is missing: {source}") - completed = subprocess.run( - ["docker", "cp", f"{source}/.", f"{container_id}:{destination}"], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=1200, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError( - f"docker cp {source} failed: {(completed.stderr or completed.stdout)[-500:]}" - ) - - @staticmethod - def _copy_git_snapshot(container_id: str, source: Path, destination: str) -> None: - """Stage only files tracked by the pinned LoopX commit. - - The LoopX checkout also hosts benchmark runs. Copying the working tree - would expose prior trajectories and artifacts inside the agent container. - """ - archive = subprocess.Popen( - ["git", "-C", str(source), "archive", "--format=tar", "HEAD"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if archive.stdout is None: - archive.kill() - raise RuntimeError("could not open LoopX git archive stream") - extract = subprocess.Popen( - ["docker", "exec", "-i", container_id, "tar", "-x", "-C", destination], - stdin=archive.stdout, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=False, - ) - archive.stdout.close() - try: - extract_stdout, extract_stderr = extract.communicate(timeout=1200) - except subprocess.TimeoutExpired: - extract.kill() - archive.kill() - extract.communicate() - archive.communicate() - raise RuntimeError("timed out staging the LoopX git snapshot") - archive_stderr = archive.stderr.read() if archive.stderr is not None else b"" - archive_returncode = archive.wait(timeout=30) - if archive_returncode != 0 or extract.returncode != 0: - detail = archive_stderr or extract_stderr or extract_stdout - raise RuntimeError( - "failed to stage clean LoopX git snapshot: " - + detail.decode("utf-8", errors="replace")[-500:] - ) - - def _profile_env(self) -> dict[str, str]: - return { - "HOME": _PROFILE_HOME, - "CODEX_HOME": _SHARED_CODEX_HOME, - "PATH": f"{_NODE}/bin:{_PROFILE}/bin:/usr/local/bin:/usr/bin:/bin", - "LOOPX_PYTHON": f"{_PYTHON}/bin/python3", - "LOOPX_PROMOTE_DEFAULT": "1", - "LOOPX_INSTALL_CANARY": "0", - "LOOPX_BIN_DIR": f"{_PROFILE}/bin", - "LOOPX_RELEASES_DIR": f"{_PROFILE}/releases", - "LOOPX_RELEASE_ID": "lhtb-generic-cli-heartbeat", - "LOOPX_MAN_ROOT": f"{_PROFILE}/man", - "LOOPX_MAN_DIR": f"{_PROFILE}/man/man1", - "LOOPX_SHELL_PROFILE": f"{_PROFILE_HOME}/.profile", - "LOOPX_SKILLS_DIR": _SHARED_SKILLS, - "LOOPX_INSTALL_SLASH_COMMANDS": "0", - "LOOPX_INSTALL_OPENCODE": "0", - "LOOPX_INSTALL_CLAUDE": "0", - "LOOPX_SKILL_DEDUPE_OTHER_ROOT": "0", - # Codex tool calls use `bash -lc`, whose login profile may replace - # PATH. BASH_ENV restores the staged Node for LoopX subprocesses. - "BASH_ENV": _BASH_ENV, - } - - async def install(self, environment: BaseEnvironment) -> None: - await super().install(environment) - - loopx_src = Path(os.environ["LOOPX_SRC_DIR"]).resolve() - portable_python = Path(os.environ["LOOPX_PORTABLE_PYTHON"]).resolve() - node_root = Path(os.environ["LOOPX_NODE_DIR"]).resolve() - wake_source = Path(__file__).resolve().parent.parent / "runtime" / "wake_once.py" - expected_commit = os.environ.get("LOOPX_EXPECTED_COMMIT", "").strip() - - actual_commit = subprocess.run( - ["git", "-C", str(loopx_src), "rev-parse", "HEAD"], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=30, - check=False, - ).stdout.strip() - if expected_commit and actual_commit != expected_commit: - raise RuntimeError( - f"LoopX commit mismatch: expected {expected_commit}, got {actual_commit}" - ) - - await self.exec_as_root( - environment, - command=( - f"mkdir -p {_SRC} {_PYTHON} {_NODE} {_PROFILE_HOME} " - f"{_SHARED_CODEX_HOME} {_PROFILE}/bin {_PROFILE}/releases {_PROFILE}/man " - f"{_CONTROL} " - f"{_LOOPX_RUNTIME} {_TURN_ROOT} {os.path.dirname(_WAKE_SCRIPT)} " - f"{_WAKE_LOG_DIR}; chmod -R 0777 {_ROOT} {_WAKE_LOG_DIR}" - ), - timeout_sec=180, - ) - container_id = self._container_id(environment) - self._copy_git_snapshot(container_id, loopx_src, _SRC) - self._copy_tree(container_id, portable_python, _PYTHON) - self._copy_tree(container_id, node_root, _NODE) - await environment.upload_file(wake_source, _WAKE_SCRIPT) - await self.exec_as_root( - environment, - command=( - f"chmod 0755 {_WAKE_SCRIPT}; " - f"printf '%s\\n' 'export PATH={_NODE}/bin:$PATH' > {_BASH_ENV}; " - f"chmod 0644 {_BASH_ENV}; " - f"find {_SRC} -maxdepth 2 \\( -name '*.egg-info' -o " - f"-name '*.dist-info' \\) -exec rm -rf {{}} +; " - f"chmod -R a+rX {_SRC} {_PYTHON} {_NODE}; " - f"chmod -R a+rwX {_PROFILE} {_CONTROL} {_TURN_ROOT} {_WAKE_LOG_DIR}" - ), - timeout_sec=300, - ) - install = await self.exec_as_agent( - environment, - command=f"bash {_SRC}/scripts/install-local.sh", - env=self._profile_env(), - timeout_sec=1200, - ) - if "error" in (install.stderr or "").lower(): - self.logger.debug("LoopX installer stderr: %s", install.stderr[-1000:]) - - doctor = await self.exec_as_agent( - environment, - command=f"{_CLI} --format json doctor --agent-type codex-cli", - env=self._profile_env(), - timeout_sec=300, - ) - try: - doctor_payload = json.loads(doctor.stdout or "") - except json.JSONDecodeError as exc: - raise RuntimeError("LoopX doctor returned invalid JSON") from exc - if doctor_payload.get("ok") is not True: - raise RuntimeError(f"LoopX doctor failed: {doctor_payload}") - - receipt = { - "loopx_commit": actual_commit, - "runtime_profile": "generic_cli", - "codex_driver": "fresh_exec_per_wake", - "onboarding_connection_validation": "provider-prevalidated", - "login_shell_node_path": _BASH_ENV, - "scheduler_terminal_packet_compatibility": True, - "replan_after_completed_todos": _REPLAN_AFTER_TODOS, - } - await self.exec_as_agent( - environment, - command=( - f"printf %s {shlex.quote(json.dumps(receipt, sort_keys=True))} " - f"> /logs/agent/loopx-install.json" - ), - env=self._profile_env(), - ) - - async def _write_task_document( - self, environment: BaseEnvironment, instruction: str - ) -> None: - descriptor, name = tempfile.mkstemp(prefix="lhtb-loopx-task-", suffix=".md") - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - handle.write("# Current LHTB task\n\n") - handle.write(instruction.strip()) - handle.write("\n") - await environment.upload_file(Path(name), _TASK_DOC) - await self.exec_as_root( - environment, - command=f"chmod 0644 {_TASK_DOC}", - ) - finally: - Path(name).unlink(missing_ok=True) - - async def _loopx( - self, - environment: BaseEnvironment, - args: list[str], - *, - cwd: str, - require_ok: bool = True, - ) -> dict: - argv = [ - _CLI, - "--format", "json", - "--registry", _REGISTRY, - "--runtime-root", _LOOPX_RUNTIME, - *args, - ] - result = await self.exec_as_agent( - environment, - command=shlex.join(argv), - env=self._profile_env(), - cwd=cwd, - timeout_sec=300, - ) - text = (result.stdout or "").strip() - if text.startswith("```"): - text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip() - try: - payload = json.loads(text) - except json.JSONDecodeError as exc: - raise RuntimeError(f"LoopX command returned invalid JSON: {text[:300]}") from exc - if require_ok and payload.get("ok") is False: - raise RuntimeError(f"LoopX command failed: {payload.get('error')}") - return payload - - async def _registry_exists(self, environment: BaseEnvironment) -> bool: - result = await environment.exec(command=f"test -s {_REGISTRY}") - return result.return_code == 0 - - async def _prepare_phase( - self, environment: BaseEnvironment, instruction: str, *, cwd: str - ) -> None: - await self._write_task_document(environment, instruction) - if not await self._registry_exists(environment): - await self._loopx( - environment, - [ - "bootstrap", - "--project", ".", - "--goal-id", _GOAL_ID, - "--objective", - "Complete the current LHTB task through validated LoopX Todos.", - "--goal-doc", _TASK_DOC, - "--adapter-kind", "read_only_project_map_v0", - "--adapter-status", "connected-read-only", - "--write-scope", "**", - "--no-onboarding-scan", - "--onboarding-connection-validation", "provider-prevalidated", - "--begin-autonomous-advance", - "--codex-app-heartbeat", "no", - "--no-global-sync", - ], - cwd=cwd, - ) - await self._loopx( - environment, - [ - "configure-goal", - "--goal-id", _GOAL_ID, - "--registered-agent", _AGENT_ID, - "--execution-replan-after-todos", str(_REPLAN_AFTER_TODOS), - "--agent-work-mode", f"{_AGENT_ID}=active", - "--execute", - ], - cwd=cwd, - ) - else: - await self._loopx( - environment, - [ - "configure-goal", - "--goal-id", _GOAL_ID, - "--execution-replan-after-todos", str(_REPLAN_AFTER_TODOS), - "--clear-waiting-on", - "--agent-work-mode", f"{_AGENT_ID}=active", - "--execute", - ], - cwd=cwd, - ) - - todo_id = f"lhtb-task-phase-{self._phase_number:03d}" - await self._loopx( - environment, - [ - "todo", "add", - "--goal-id", _GOAL_ID, - "--role", "agent", - "--todo-id", todo_id, - "--text", - ( - f"[P0] Execute benchmark phase {self._phase_number}. Read the exact " - f"current task from {_TASK_DOC}; inspect the workspace, implement and " - "validate it, and create bounded successor Todos for remaining work." - ), - "--task-class", "advancement_task", - "--action-kind", "lhtb_benchmark_task", - "--claimed-by", _AGENT_ID, - "--status", "open", - "--execute", - ], - cwd=cwd, - ) - - cadence = await self._loopx( - environment, - ["configure-goal", "--goal-id", _GOAL_ID], - cwd=cwd, - ) - configured_state = cadence.get("after") or cadence.get("before") or {} - configured = configured_state.get("execution_profile", {}).get( - "replan_after_completed_todos" - ) - if configured != _REPLAN_AFTER_TODOS: - raise RuntimeError( - f"replan cadence readback mismatch: expected 3, got {configured!r}" - ) - - def _worker_env(self, *, cwd: str) -> dict[str, str]: - return { - **self._profile_env(), - "LOOPX_CLI": _CLI, - "LOOPX_REGISTRY": _REGISTRY, - "LOOPX_RUNTIME_ROOT": _LOOPX_RUNTIME, - "LOOPX_GOAL_ID": _GOAL_ID, - "LOOPX_AGENT_ID": _AGENT_ID, - "LOOPX_PROJECT": cwd, - "LOOPX_WAKE_LOG_DIR": _WAKE_LOG_DIR, - "LOOPX_TURN_ROOT": _TURN_ROOT, - "LOOPX_SHARED_SKILLS": _SHARED_SKILLS, - "LOOPX_CODEX_TURN_TIMEOUT_SEC": os.environ.get( - "LOOPX_CODEX_TURN_TIMEOUT_SEC", "4700" - ), - "CODEX_BIN": "/usr/local/bin/codex", - "MODEL_NAME": self.model_name or "", - "REASONING_EFFORT": str( - self._resolved_flags.get("reasoning_effort", "max") - ), - "OPENAI_BASE_URL": self._get_env("OPENAI_BASE_URL") or "", - "OPENAI_API_KEY": self._get_env("OPENAI_API_KEY") or "", - "CODEX_WIRE_API": self._get_env("CODEX_WIRE_API") or "responses", - } - - def _session_trajectories(self, roots: Iterable[Path]) -> list[Trajectory]: - parents: set[Path] = set() - for root in roots: - if root.is_dir(): - parents.update(path.parent for path in root.glob("sessions/**/*.jsonl")) - trajectories: list[Trajectory] = [] - for parent in sorted(parents): - try: - trajectory = self._convert_events_to_trajectory(parent) - except Exception: - self.logger.exception("failed to parse Codex session under %s", parent) - continue - if trajectory is not None: - trajectories.append(trajectory) - return trajectories - - @staticmethod - def _totals(trajectories: Iterable[Trajectory]) -> dict[str, int | float | None]: - prompt = completion = cached = 0 - costs: list[float] = [] - for trajectory in trajectories: - metrics = trajectory.final_metrics - if metrics is None: - continue - prompt += metrics.total_prompt_tokens or 0 - completion += metrics.total_completion_tokens or 0 - cached += metrics.total_cached_tokens or 0 - if metrics.total_cost_usd is not None: - costs.append(metrics.total_cost_usd) - return { - "prompt": prompt, - "completion": completion, - "cached": cached, - "cost": sum(costs) if costs else None, - } - - def _write_aggregate_trajectory(self) -> list[Trajectory]: - wake_root = self.logs_dir / "wakes" - wake_dirs = sorted(path for path in wake_root.iterdir() if path.is_dir()) if wake_root.is_dir() else [] - trajectories = self._session_trajectories(wake_dirs) - if not trajectories: - return [] - steps = [] - for trajectory in trajectories: - for step in trajectory.steps: - copied = step.model_copy(deep=True) - copied.step_id = len(steps) + 1 - steps.append(copied) - totals = self._totals(trajectories) - aggregate = Trajectory( - schema_version="ATIF-v1.5", - session_id=f"loopx-heartbeat-{self.logs_dir.parent.name}", - agent=trajectories[0].agent, - steps=steps, - final_metrics=FinalMetrics( - total_prompt_tokens=totals["prompt"] or None, - total_completion_tokens=totals["completion"] or None, - total_cached_tokens=totals["cached"] or None, - total_cost_usd=totals["cost"], - total_steps=len(steps), - extra={"heartbeat_wakes": len(trajectories)}, - ), - ) - (self.logs_dir / "trajectory.json").write_text( - format_trajectory_json(aggregate.to_json_dict()), encoding="utf-8" - ) - return trajectories - - def _populate_context(self, context: AgentContext, wake_dirs: list[Path]) -> None: - phase_trajectories = self._session_trajectories(wake_dirs) - totals = self._totals(phase_trajectories) - context.n_input_tokens = int(totals["prompt"] or 0) - context.n_output_tokens = int(totals["completion"] or 0) - context.n_cache_tokens = int(totals["cached"] or 0) - context.cost_usd = totals["cost"] - context.metadata = { - "loopx_runtime_profile": "generic_cli", - "codex_session_policy": "fresh_exec_per_wake", - "codex_resume_used": False, - "replan_after_completed_todos": _REPLAN_AFTER_TODOS, - "onboarding_connection_validation": "provider-prevalidated", - "login_shell_node_path": _BASH_ENV, - "scheduler_terminal_packet_compatibility": True, - "heartbeat_wakes": len(wake_dirs), - "benchmark_phase": self._phase_number, - } - self._write_aggregate_trajectory() - - def populate_context_post_run(self, context: AgentContext) -> None: - wake_root = self.logs_dir / "wakes" - wake_dirs = sorted(path for path in wake_root.iterdir() if path.is_dir()) if wake_root.is_dir() else [] - self._populate_context(context, wake_dirs) - - @with_prompt_template - async def run( - self, - instruction: str, - environment: BaseEnvironment, - context: AgentContext, - ) -> None: - if not self.model_name: - raise ValueError("model_name is required") - self._phase_number += 1 - pwd = await self.exec_as_agent(environment, command="pwd", timeout_sec=30) - cwd = (pwd.stdout or "").strip() - if not cwd.startswith("/"): - raise RuntimeError(f"could not resolve container working directory: {cwd!r}") - - wake_root = self.logs_dir / "wakes" - before = {path.name for path in wake_root.iterdir() if path.is_dir()} if wake_root.is_dir() else set() - try: - await self._prepare_phase(environment, instruction, cwd=cwd) - wake_command = shlex.join([f"{_PYTHON}/bin/python3", _WAKE_SCRIPT]) - worker_argv = [ - f"{_PYTHON}/bin/python3", - f"{_SRC}/scripts/external_scheduler_worker.py", - "--cli-bin", _CLI, - "--registry", _REGISTRY, - "--runtime-root", _LOOPX_RUNTIME, - "--runtime-profile", "generic_cli", - "--goal-id", _GOAL_ID, - "--agent-id", _AGENT_ID, - "--state-file", _SCHEDULER_STATE, - "--wake-cmd", wake_command, - "--wake-timeout-seconds", os.environ.get( - "LOOPX_WAKE_TIMEOUT_SEC", "4800" - ), - "--quota-timeout-seconds", "30", - "--error-backoff-seconds", "15", - ] - scheduler_timeout = int(os.environ.get("LOOPX_SCHEDULER_TIMEOUT_SEC", "5080")) - phase_log = f"/logs/agent/loopx-worker-phase-{self._phase_number:03d}.log" - shell = ( - "set +e; " - f"timeout --signal=TERM --kill-after=15 {scheduler_timeout}s " - f"{shlex.join(worker_argv)} >> {shlex.quote(phase_log)} 2>&1; " - "rc=$?; set -e; " - f"if [ \"$rc\" -eq 124 ]; then echo scheduler_timeout >> {shlex.quote(phase_log)}; exit 0; fi; " - "exit \"$rc\"" - ) - await self.exec_as_agent( - environment, - command=shell, - env=self._worker_env(cwd=cwd), - cwd=cwd, - timeout_sec=scheduler_timeout + 60, - ) - finally: - after_dirs = ( - sorted(path for path in wake_root.iterdir() if path.is_dir() and path.name not in before) - if wake_root.is_dir() - else [] - ) - self._populate_context(context, after_dirs) - - -def safe_trial_slug(value: str) -> str: - """Retained for receipts and tests that need a public-safe trial label.""" - - normalized = re.sub(r"[^A-Za-z0-9._-]", "-", value).strip("-._") - return normalized or "lhtb-trial" diff --git a/benchmark/LHTB/configs/heartbeat-generic-cli.yaml b/benchmark/LHTB/configs/heartbeat-generic-cli.yaml index 6f3743b78d..7a701590cc 100644 --- a/benchmark/LHTB/configs/heartbeat-generic-cli.yaml +++ b/benchmark/LHTB/configs/heartbeat-generic-cli.yaml @@ -17,10 +17,12 @@ environment: no_proxy: "localhost,127.0.0.1,game" agents: - - import_path: codex_loopx_heartbeat:LoopxHeartbeatCodex + - import_path: benchmark.runtime.harbor:BenchmarkCodex model_name: openai/gpt-5.6-sol override_timeout_sec: 5400 kwargs: + execution_mode: heartbeat + iteration_context: fresh reasoning_effort: max goals: "false" web_search: disabled diff --git a/benchmark/LHTB/run.sh b/benchmark/LHTB/run.sh index 99eed61db5..a1ca51be6b 100755 --- a/benchmark/LHTB/run.sh +++ b/benchmark/LHTB/run.sh @@ -18,9 +18,9 @@ LHTB_ROOT="$(cd "$LHTB_ROOT" && pwd)" MODE="${1:-preflight}" SMOKE_TASK="${2:-tabular-data-feature-covshift}" case "$MODE" in - preflight|smoke|full) ;; + prepare|preflight|smoke|full) ;; *) - echo "Usage: $0 {preflight|smoke [task-name]|full}" >&2 + echo "Usage: $0 {prepare|preflight|smoke [task-name]|full}" >&2 exit 2 ;; esac @@ -38,7 +38,6 @@ REASONING_EFFORT="${REASONING_EFFORT:-max}" CONCURRENCY="${CONCURRENCY:-4}" AGENT_TIMEOUT_SEC="${AGENT_TIMEOUT_SEC:-5400}" LOOPX_SCHEDULER_TIMEOUT_SEC="${LOOPX_SCHEDULER_TIMEOUT_SEC:-5080}" -LOOPX_WAKE_TIMEOUT_SEC="${LOOPX_WAKE_TIMEOUT_SEC:-4800}" LOOPX_CODEX_TURN_TIMEOUT_SEC="${LOOPX_CODEX_TURN_TIMEOUT_SEC:-4700}" LHTB_MAX_RETRIES="${LHTB_MAX_RETRIES:-2}" RUNNER_RESTARTS="${RUNNER_RESTARTS:-2}" @@ -46,7 +45,13 @@ LHTB_MODELONLY_NETWORK="${LHTB_MODELONLY_NETWORK:-lhtb-modelonly}" LHTB_MODELONLY_SUBNET="${LHTB_MODELONLY_SUBNET:-192.0.2.0/24}" LHTB_MODELONLY_GATEWAY="${LHTB_MODELONLY_GATEWAY:-192.0.2.1}" LOOPX_SRC_DIR="${LOOPX_SRC_DIR:-$LOOPX_ROOT}" -SHARED_CODEX_AGENT_DIR="$LOOPX_SRC_DIR/benchmark/swe-marathon/agents" +export PYTHONPATH="$LOOPX_SRC_DIR${PYTHONPATH:+:$PYTHONPATH}" +LOOPX_EXECUTION_MODE="${LOOPX_EXECUTION_MODE:-heartbeat}" +LOOPX_TASK_ENTRY="${LOOPX_TASK_ENTRY:-seeded-todo}" +LOOPX_PLANNING_TIMEOUT_SEC="${LOOPX_PLANNING_TIMEOUT_SEC:-300}" +LOOPX_ITERATION_CONTEXT="${LOOPX_ITERATION_CONTEXT:-fresh}" +LOOPX_VALIDATION_COMMAND_JSON="${LOOPX_VALIDATION_COMMAND_JSON:-[]}" +SHARED_CODEX_AGENT_DIR="$LOOPX_SRC_DIR/benchmark/runtime" [[ -f "$SHARED_CODEX_AGENT_DIR/codex_offline.py" ]] || \ die "Shared offline Codex adapter not found: $SHARED_CODEX_AGENT_DIR/codex_offline.py" LOOPX_EXPECTED_COMMIT="${LOOPX_EXPECTED_COMMIT:-$(git -C "$LOOPX_SRC_DIR" rev-parse HEAD)}" @@ -68,28 +73,25 @@ if [[ -z "${LOOPX_NODE_DIR:-}" ]]; then node_binary="$(readlink -f "$node_command" 2>/dev/null || true)" [[ -n "$node_binary" ]] && LOOPX_NODE_DIR="$(cd "$(dirname "$node_binary")/.." && pwd)" fi -[[ -n "${LOOPX_NODE_DIR:-}" && -x "$LOOPX_NODE_DIR/bin/node" ]] || die "Set LOOPX_NODE_DIR to a Node >=22.6 root" +[[ -n "${LOOPX_NODE_DIR:-}" && -x "$LOOPX_NODE_DIR/bin/node" ]] || die "Set LOOPX_NODE_DIR to a supported Node root" for value in "$CONCURRENCY" "$AGENT_TIMEOUT_SEC" "$LOOPX_SCHEDULER_TIMEOUT_SEC" \ - "$LOOPX_WAKE_TIMEOUT_SEC" "$LOOPX_CODEX_TURN_TIMEOUT_SEC" "$LHTB_MAX_RETRIES" "$RUNNER_RESTARTS"; do + "$LOOPX_CODEX_TURN_TIMEOUT_SEC" "$LHTB_MAX_RETRIES" "$RUNNER_RESTARTS"; do [[ "$value" =~ ^[0-9]+$ ]] || die "numeric configuration expected, got: $value" done -(( LOOPX_CODEX_TURN_TIMEOUT_SEC < LOOPX_WAKE_TIMEOUT_SEC )) || die "Codex turn timeout must be below wake timeout" -(( LOOPX_WAKE_TIMEOUT_SEC < LOOPX_SCHEDULER_TIMEOUT_SEC )) || die "wake timeout must be below scheduler timeout" +(( LOOPX_CODEX_TURN_TIMEOUT_SEC + 150 < LOOPX_SCHEDULER_TIMEOUT_SEC )) || die "scheduler timeout must exceed Codex timeout plus 150s cleanup allowance" (( LOOPX_SCHEDULER_TIMEOUT_SEC < AGENT_TIMEOUT_SEC )) || die "scheduler timeout must be below Harbor agent timeout" -[[ "$MODEL_NAME" == "openai/gpt-5.6-sol" ]] || die "this treatment is pinned to openai/gpt-5.6-sol" -[[ "$REASONING_EFFORT" == "max" ]] || die "this treatment is pinned to reasoning=max" gateway_host="$($VENV/bin/python -c 'from urllib.parse import urlsplit; import sys; print(urlsplit(sys.argv[1]).hostname or "")' "$OPENAI_BASE_URL")" [[ -n "$gateway_host" ]] || die "Invalid OPENAI_BASE_URL: $OPENAI_BASE_URL" echo "=== prepare Harbor model-only networking ===" HARBOR_DOCKER_DIR="$LHTB_ROOT/upstream/harbor/src/harbor/environments/docker" -if ! grep -q 'LHTB_MODELONLY_NET' "$HARBOR_DOCKER_DIR/docker.py" 2>/dev/null; then +if [[ "$MODE" != preflight ]] && ! grep -q 'LHTB_MODELONLY_NET' "$HARBOR_DOCKER_DIR/docker.py" 2>/dev/null; then LHTB_HARBOR_SRC="$LHTB_ROOT/upstream/harbor/src/harbor" \ "$VENV/bin/python" "$CODE_DIR/harbor_patch/prepare_harbor_modelonly.py" fi -if ! docker network inspect "$LHTB_MODELONLY_NETWORK" >/dev/null 2>&1; then +if [[ "$MODE" != preflight ]] && ! docker network inspect "$LHTB_MODELONLY_NETWORK" >/dev/null 2>&1; then docker network create --driver bridge --internal \ --subnet "$LHTB_MODELONLY_SUBNET" \ --gateway "$LHTB_MODELONLY_GATEWAY" \ @@ -101,6 +103,11 @@ network_gateway="$(docker network inspect "$LHTB_MODELONLY_NETWORK" --format '{{ [[ "$network_gateway" == "$LHTB_MODELONLY_GATEWAY" ]] || die "network gateway mismatch: $network_gateway" [[ "$gateway_host" == "$network_gateway" ]] || die "offline tasks can only reach $network_gateway; gateway uses $gateway_host" +if [[ "$MODE" == prepare ]]; then + echo "Harbor model-only networking prepared. Run preflight next." + exit 0 +fi + run_stamp="$(date +%Y%m%d-%H%M%S)" task_args=() expected_task_count=46 @@ -110,7 +117,7 @@ if [[ "$MODE" == smoke ]]; then expected_task_count=1 job_suffix="smoke-${SMOKE_TASK}" fi -job_name="lhtb-loopx-hb-gpt56sol-max-${job_suffix}-${run_stamp}" +job_name="lhtb-${LOOPX_EXECUTION_MODE}-${LOOPX_TASK_ENTRY}-${LOOPX_ITERATION_CONTEXT}-${job_suffix}-${run_stamp}" generated_config="$CODE_DIR/.generated/${job_name}.yaml" jobs_dir="$CODE_DIR/runs" @@ -123,17 +130,24 @@ jobs_dir="$CODE_DIR/runs" --model "$MODEL_NAME" \ --effort "$REASONING_EFFORT" \ --timeout "$AGENT_TIMEOUT_SEC" \ + --execution-mode "$LOOPX_EXECUTION_MODE" \ + --task-entry "$LOOPX_TASK_ENTRY" \ + --planning-timeout "$LOOPX_PLANNING_TIMEOUT_SEC" \ + --iteration-context "$LOOPX_ITERATION_CONTEXT" \ + --validation-command-json "$LOOPX_VALIDATION_COMMAND_JSON" \ + --turn-timeout "$LOOPX_CODEX_TURN_TIMEOUT_SEC" \ + --scheduler-timeout "$LOOPX_SCHEDULER_TIMEOUT_SEC" \ "${task_args[@]}" export OPENAI_BASE_URL OPENAI_API_KEY MODEL_NAME REASONING_EFFORT export CODEX_BIN CODEX_OFFLINE_DIR CODEX_WIRE_API="${CODEX_WIRE_API:-responses}" export LOOPX_SRC_DIR LOOPX_EXPECTED_COMMIT LOOPX_PORTABLE_PYTHON LOOPX_NODE_DIR -export LOOPX_SCHEDULER_TIMEOUT_SEC LOOPX_WAKE_TIMEOUT_SEC LOOPX_CODEX_TURN_TIMEOUT_SEC +export LOOPX_SCHEDULER_TIMEOUT_SEC LOOPX_CODEX_TURN_TIMEOUT_SEC export LHTB_MODELONLY_NETWORK CONCURRENCY AGENT_TIMEOUT_SEC export LHTB_MODELONLY_NET=1 HB_VERIFIER_FEEDBACK_MODE=binary export DOCKER_DEFAULT_PLATFORM="${DOCKER_DEFAULT_PLATFORM:-linux/amd64}" export LITELLM_LOCAL_MODEL_COST_MAP="${LITELLM_LOCAL_MODEL_COST_MAP:-True}" -export PYTHONPATH="$CODE_DIR/agents:$SHARED_CODEX_AGENT_DIR:$LOOPX_SRC_DIR${PYTHONPATH:+:$PYTHONPATH}" +export PYTHONPATH="$LOOPX_SRC_DIR${PYTHONPATH:+:$PYTHONPATH}" export NO_PROXY="127.0.0.1,localhost,$gateway_host,${NO_PROXY:-}" export no_proxy="$NO_PROXY" @@ -160,8 +174,7 @@ receipt="$CODE_DIR/reports/${job_name}.env" printf 'job_name=%s\nmode=%s\nmodel=%s\nreasoning_effort=%s\n' "$job_name" "$MODE" "$MODEL_NAME" "$REASONING_EFFORT" printf 'concurrency=%s\nagent_timeout_sec=%s\nscheduler_timeout_sec=%s\n' "$CONCURRENCY" "$AGENT_TIMEOUT_SEC" "$LOOPX_SCHEDULER_TIMEOUT_SEC" printf 'gateway=%s\nwire_api=%s\nweb_search=disabled\n' "$OPENAI_BASE_URL" "$CODEX_WIRE_API" - printf 'runtime_profile=generic_cli\ncodex_driver=fresh_exec_per_wake\ncodex_resume=false\n' - printf 'onboarding_connection_validation=provider-prevalidated\n' + printf 'execution_mode=%s\niteration_context=%s\ncodex_home_scope=trial\n' "$LOOPX_EXECUTION_MODE" "$LOOPX_ITERATION_CONTEXT" printf 'scheduler_terminal_packet_compatibility=true\n' printf 'replan_after_completed_todos=3\nverifier_policy=44_shared_2_separate\n' printf 'loopx_commit=%s\n' "$(git -C "$LOOPX_SRC_DIR" rev-parse HEAD)" diff --git a/benchmark/LHTB/runtime/wake_once.py b/benchmark/LHTB/runtime/wake_once.py deleted file mode 100755 index 881637f7f3..0000000000 --- a/benchmark/LHTB/runtime/wake_once.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python3 -"""Run one LoopX heartbeat as one fresh, non-resumed Codex exec session.""" - -from __future__ import annotations - -import json -import os -import shutil -import signal -import subprocess -import sys -import time -import uuid -from pathlib import Path -from typing import Any - - -GLOBAL_REGISTRY_TOKEN = "$HOME/.codex/loopx/registry.global.json" - - -def required_env(name: str) -> str: - value = os.environ.get(name, "").strip() - if not value: - raise RuntimeError(f"missing required environment variable: {name}") - return value - - -def parse_json_output(text: str, *, command: str) -> dict[str, Any]: - value = text.strip() - if value.startswith("```"): - value = value.split("\n", 1)[-1].rsplit("```", 1)[0].strip() - try: - payload = json.loads(value) - except json.JSONDecodeError as exc: - raise RuntimeError(f"{command} returned invalid JSON: {value[:300]}") from exc - if not isinstance(payload, dict): - raise RuntimeError(f"{command} returned a non-object JSON value") - return payload - - -def run_loopx(argv: list[str], *, cwd: Path, env: dict[str, str]) -> dict[str, Any]: - completed = subprocess.run( - argv, - cwd=cwd, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=120, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError( - f"LoopX exited {completed.returncode}: " - f"{(completed.stderr or completed.stdout)[-500:]}" - ) - return parse_json_output(completed.stdout, command="heartbeat-prompt") - - -def build_heartbeat_argv( - *, cli: str, registry: str, runtime_root: str, goal_id: str, - agent_id: str, turn_id: str, -) -> list[str]: - return [ - cli, - "--format", "json", - "--registry", registry, - "--runtime-root", runtime_root, - "heartbeat-prompt", - "--thin", - "--runtime-profile", "generic_cli", - "--goal-id", goal_id, - "--agent-id", agent_id, - "--turn-instance-id", turn_id, - "--cli-bin", cli, - "--available-capability", "shell", - "--available-capability", "filesystem_write", - ] - - -def build_codex_argv(*, codex_bin: str, model: str, effort: str, cwd: str) -> list[str]: - # Deliberately contains no `resume`: every heartbeat is a new Codex session. - return [ - codex_bin, - "exec", - "--dangerously-bypass-approvals-and-sandbox", - "--skip-git-repo-check", - "--cd", cwd, - "--model", model, - "--json", - "--enable", "unified_exec", - "-c", f'model_reasoning_effort="{effort}"', - "-c", "features.goals=false", - "-c", 'web_search="disabled"', - "-c", "model_providers.harbor.request_max_retries=8", - "-c", "model_providers.harbor.stream_max_retries=8", - "-c", "model_providers.harbor.stream_idle_timeout_ms=300000", - "-", - ] - - -def write_codex_home( - path: Path, *, base_url: str, api_key: str, wire_api: str, - workspace: str, shared_skills: Path, -) -> None: - path.mkdir(parents=True, exist_ok=False) - (path / "auth.json").write_text( - json.dumps({"OPENAI_API_KEY": api_key}) + "\n", encoding="utf-8" - ) - (path / "auth.json").chmod(0o600) - quoted_workspace = json.dumps(workspace) - config = "\n".join( - [ - 'web_search = "disabled"', - 'sandbox_mode = "danger-full-access"', - 'model_provider = "harbor"', - f"[projects.{quoted_workspace}]", - 'trust_level = "trusted"', - "[model_providers.harbor]", - 'name = "harbor"', - f"base_url = {json.dumps(base_url)}", - f"wire_api = {json.dumps(wire_api)}", - 'env_key = "OPENAI_API_KEY"', - "request_max_retries = 8", - "stream_max_retries = 8", - "stream_idle_timeout_ms = 300000", - "", - ] - ) - (path / "config.toml").write_text(config, encoding="utf-8") - if shared_skills.is_dir(): - (path / "skills").symlink_to(shared_skills, target_is_directory=True) - - -def terminate_process_group(process: subprocess.Popen[bytes]) -> None: - try: - os.killpg(process.pid, signal.SIGTERM) - process.wait(timeout=10) - except (ProcessLookupError, subprocess.TimeoutExpired): - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - - -def main() -> int: - cli = required_env("LOOPX_CLI") - registry = required_env("LOOPX_REGISTRY") - runtime_root = required_env("LOOPX_RUNTIME_ROOT") - goal_id = required_env("LOOPX_GOAL_ID") - agent_id = required_env("LOOPX_AGENT_ID") - workspace = required_env("LOOPX_PROJECT") - codex_bin = required_env("CODEX_BIN") - model = required_env("MODEL_NAME").split("/")[-1] - effort = required_env("REASONING_EFFORT") - base_url = required_env("OPENAI_BASE_URL") - api_key = required_env("OPENAI_API_KEY") - wire_api = os.environ.get("CODEX_WIRE_API", "responses").strip() or "responses" - output_root = Path(required_env("LOOPX_WAKE_LOG_DIR")) - turn_root = Path(required_env("LOOPX_TURN_ROOT")) - shared_skills = Path(required_env("LOOPX_SHARED_SKILLS")) - timeout_seconds = float(os.environ.get("LOOPX_CODEX_TURN_TIMEOUT_SEC", "4700")) - - turn_id = f"lhtb-{time.time_ns()}-{uuid.uuid4().hex[:16]}" - wake_dir = output_root / turn_id - wake_dir.mkdir(parents=True, exist_ok=False) - turn_dir = turn_root / turn_id - codex_home = turn_dir / "codex-home" - turn_dir.mkdir(parents=True, exist_ok=False) - - env = dict(os.environ) - env["LOOPX_TURN"] = turn_id - heartbeat_argv = build_heartbeat_argv( - cli=cli, - registry=registry, - runtime_root=runtime_root, - goal_id=goal_id, - agent_id=agent_id, - turn_id=turn_id, - ) - payload = run_loopx(heartbeat_argv, cwd=Path(workspace), env=env) - (wake_dir / "heartbeat.json").write_text( - json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" - ) - if payload.get("ok") is not True: - raise RuntimeError(f"heartbeat-prompt not ready: {payload.get('error')}") - if payload.get("turn_instance_id") != turn_id: - raise RuntimeError( - "heartbeat Turn identity mismatch: " - f"expected {turn_id!r}, got {payload.get('turn_instance_id')!r}" - ) - body = payload.get("task_body") - if not isinstance(body, str) or not body.strip(): - raise RuntimeError("heartbeat-prompt returned no task_body") - if "--runtime-profile generic_cli" not in body: - raise RuntimeError("heartbeat task_body is not bound to generic_cli") - body = body.replace(GLOBAL_REGISTRY_TOKEN, registry) - if GLOBAL_REGISTRY_TOKEN in body: - raise RuntimeError("heartbeat task_body retained the global registry token") - (wake_dir / "task-body.md").write_text(body, encoding="utf-8") - - write_codex_home( - codex_home, - base_url=base_url, - api_key=api_key, - wire_api=wire_api, - workspace=workspace, - shared_skills=shared_skills, - ) - codex_env = dict(env) - codex_env["CODEX_HOME"] = str(codex_home) - codex_argv = build_codex_argv( - codex_bin=codex_bin, - model=model, - effort=effort, - cwd=workspace, - ) - (wake_dir / "invocation.json").write_text( - json.dumps( - { - "turn_id": turn_id, - "runtime_profile": "generic_cli", - "model": model, - "reasoning_effort": effort, - "fresh_codex_exec": True, - "resume": False, - "web_search": "disabled", - "argv": codex_argv, - }, - ensure_ascii=False, - indent=2, - ) - + "\n", - encoding="utf-8", - ) - - timed_out = False - return_code = 1 - with (wake_dir / "codex-events.jsonl").open("wb") as stdout_handle, ( - wake_dir / "codex-stderr.log" - ).open("wb") as stderr_handle: - process = subprocess.Popen( - codex_argv, - cwd=workspace, - env=codex_env, - stdin=subprocess.PIPE, - stdout=stdout_handle, - stderr=stderr_handle, - start_new_session=True, - ) - try: - process.communicate(input=body.encode("utf-8"), timeout=timeout_seconds) - except subprocess.TimeoutExpired: - timed_out = True - terminate_process_group(process) - return_code = process.returncode if process.returncode is not None else 124 - - sessions = codex_home / "sessions" - if sessions.is_dir(): - shutil.copytree(sessions, wake_dir / "sessions", dirs_exist_ok=True) - receipt = { - "ok": return_code == 0 and not timed_out, - "turn_id": turn_id, - "codex_return_code": return_code, - "timed_out": timed_out, - "fresh_codex_exec": True, - "resume": False, - "task_body_sha256": __import__("hashlib").sha256(body.encode()).hexdigest(), - } - (wake_dir / "receipt.json").write_text( - json.dumps(receipt, indent=2) + "\n", encoding="utf-8" - ) - shutil.rmtree(turn_dir, ignore_errors=True) - print(json.dumps(receipt, separators=(",", ":")), flush=True) - return 0 if receipt["ok"] else (124 if timed_out else max(1, return_code)) - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except Exception as exc: - print( - json.dumps( - {"ok": False, "error": f"{type(exc).__name__}: {exc}"}, - separators=(",", ":"), - ), - file=sys.stderr, - ) - raise diff --git a/benchmark/LHTB/scripts/preflight.py b/benchmark/LHTB/scripts/preflight.py index 97d6b3ce01..cc3fd909d7 100755 --- a/benchmark/LHTB/scripts/preflight.py +++ b/benchmark/LHTB/scripts/preflight.py @@ -7,6 +7,8 @@ import importlib.util import json import os +import re +import tempfile import subprocess import sys import tomllib @@ -14,10 +16,15 @@ from urllib.parse import urlsplit import yaml +from benchmark.runtime.codex import Execution +from loopx.control_plane.effect_runtime import ( + MINIMUM_NODE_VERSION, + MINIMUM_NODE_VERSION_TEXT, +) EXPECTED_SEPARATE = {"langchain-version-migration", "nbody-accel-iterative"} -EXPECTED_AGENT = "codex_loopx_heartbeat:LoopxHeartbeatCodex" +EXPECTED_AGENT = "benchmark.runtime.harbor:BenchmarkCodex" def command(argv: list[str], timeout: int = 30) -> tuple[int, str]: @@ -82,9 +89,16 @@ def check(label: str, passed: bool, detail: str) -> None: if manifest.get("environment", {}).get("allow_internet") is False: offline += 1 verifier = manifest.get("verifier", {}) - if verifier.get("environment_mode") == "separate" or verifier.get("environment") is not None: + if ( + verifier.get("environment_mode") == "separate" + or verifier.get("environment") is not None + ): separate.add(task.name) - check("LHTB internet policy", offline == 22, f"offline={offline}, online={46 - offline}") + check( + "LHTB internet policy", + offline == 22, + f"offline={offline}, online={46 - offline}", + ) check( "LHTB verifier policy", separate == EXPECTED_SEPARATE, @@ -95,19 +109,46 @@ def check(label: str, passed: bool, detail: str) -> None: selected = config["datasets"][0]["task_names"] check( "selected task coverage", - len(selected) == args.expected_task_count and set(selected) <= {p.name for p in task_dirs}, + len(selected) == args.expected_task_count + and set(selected) <= {p.name for p in task_dirs}, f"{len(selected)}/{args.expected_task_count}", ) agent = config["agents"][0] kwargs = agent.get("kwargs", {}) - check("Harbor adapter", agent.get("import_path") == EXPECTED_AGENT, str(agent.get("import_path"))) - check("model", agent.get("model_name") == "openai/gpt-5.6-sol", str(agent.get("model_name"))) - check("reasoning", kwargs.get("reasoning_effort") == "max", str(kwargs.get("reasoning_effort"))) - check("native Goal disabled", kwargs.get("goals") == "false", str(kwargs.get("goals"))) - check("web search disabled", kwargs.get("web_search") == "disabled", str(kwargs.get("web_search"))) + check( + "Harbor adapter", + agent.get("import_path") == EXPECTED_AGENT, + str(agent.get("import_path")), + ) + check("model selected", bool(agent.get("model_name")), str(agent.get("model_name"))) + check( + "reasoning selected", + bool(kwargs.get("reasoning_effort")), + str(kwargs.get("reasoning_effort")), + ) + execution = Execution( + mode=kwargs.get("execution_mode", "heartbeat"), + task_entry=kwargs.get("task_entry", "seeded-todo"), + context=kwargs.get("iteration_context", "fresh"), + validation_command=kwargs.get("validation_command", []), + ) + check( + "native Goal setting", + kwargs.get("goals") == str(execution.native_goal).lower(), + str(kwargs.get("goals")), + ) + check( + "web search disabled", + kwargs.get("web_search") == "disabled", + str(kwargs.get("web_search")), + ) actual_commit = command(["git", "-C", str(args.loopx_src), "rev-parse", "HEAD"])[1] - check("LoopX commit", actual_commit == args.expected_commit, actual_commit or "unavailable") + check( + "LoopX commit", + actual_commit == args.expected_commit, + actual_commit or "unavailable", + ) check( "LoopX external scheduler", (args.loopx_src / "scripts" / "external_scheduler_worker.py").is_file(), @@ -155,90 +196,43 @@ def check(label: str, passed: bool, detail: str) -> None: terminal_compatible, terminal_detail, ) - shared_codex_adapter = ( - args.loopx_src / "benchmark" / "swe-marathon" / "agents" / "codex_offline.py" - ) + shared_codex_adapter = args.loopx_src / "benchmark" / "runtime" / "codex_offline.py" check( "shared offline Codex adapter", shared_codex_adapter.is_file(), str(shared_codex_adapter), ) - rc, help_text = command([str(args.loopx_src / "scripts" / "loopx"), "configure-goal", "--help"]) + rc, help_text = command( + [str(args.loopx_src / "scripts" / "loopx"), "configure-goal", "--help"] + ) check( "Todo replan cadence CLI", rc == 0 and "--execution-replan-after-todos {1,2,3,4,5}" in help_text, "supports threshold 1..5", ) - wake_path = args.config.resolve().parents[1] / "runtime" / "wake_once.py" try: - wake = load_module(wake_path, "lhtb_loopx_wake_once") - turn_id = "preflight-unique-turn" - heartbeat = wake.build_heartbeat_argv( - cli="/opt/loopx", - registry="/tmp/registry.json", - runtime_root="/tmp/runtime", - goal_id="goal", - agent_id="agent", - turn_id=turn_id, - ) - codex = wake.build_codex_argv( - codex_bin="codex", model="gpt-5.6-sol", effort="max", cwd="/app" - ) - check( - "generic_cli heartbeat contract", - "generic_cli" in heartbeat and "--turn-instance-id" in heartbeat and turn_id in heartbeat, - "runtime_profile=generic_cli + explicit TURN_ID", - ) - check( - "fresh Codex exec contract", - codex[:2] == ["codex", "exec"] and "resume" not in codex, - "codex exec; resume absent", - ) - check("Codex effort", 'model_reasoning_effort="max"' in codex, "max") - check("Codex web search", 'web_search="disabled"' in codex, "disabled") - check("Codex native Goal", "features.goals=false" in codex, "disabled") - except Exception as exc: - check("wake module", False, f"{type(exc).__name__}: {exc}") + from benchmark.runtime.harbor import BenchmarkCodex - agent_source = args.config.resolve().parents[1] / "agents" / "codex_loopx_heartbeat.py" - source_text = agent_source.read_text(encoding="utf-8") - check( - "replan threshold pinned", - "_REPLAN_AFTER_TODOS = 3" in source_text - and '"--execution-replan-after-todos", str(_REPLAN_AFTER_TODOS)' in source_text, - "3 with readback gate", - ) - check( - "benchmark-owned onboarding", - '"--no-onboarding-scan"' in source_text - and '"--onboarding-connection-validation", "provider-prevalidated"' in source_text - and '"--accept-onboarding-agent-todos"' not in source_text, - "provider-prevalidated; no unrelated repo-intake Todo", - ) - check( - "LoopX install profile directories", - "_PROFILE_HOME" in source_text - and "_SHARED_CODEX_HOME" in source_text - and "{_PROFILE}/releases" in source_text, - "HOME, CODEX_HOME, bin, releases and man are pre-created", - ) - check( - "clean LoopX source staging", - "_copy_git_snapshot" in source_text - and "self._copy_git_snapshot(container_id, loopx_src, _SRC)" in source_text, - "pinned git snapshot excludes local benchmark runs and artifacts", - ) - check( - "Codex login-shell Node bridge", - '"BASH_ENV": _BASH_ENV' in source_text - and "export PATH={_NODE}/bin:$PATH" in source_text, - "portable Node survives bash -lc", - ) + with tempfile.TemporaryDirectory(prefix="benchmark-preflight-") as directory: + candidate = BenchmarkCodex( + logs_dir=Path(directory), model_name=agent["model_name"], **kwargs + ) + check( + "shared runtime configuration", + True, + f"{candidate.execution.mode}/{candidate.execution.context}", + ) + except (ImportError, TypeError, ValueError) as exc: + check("shared runtime configuration", False, str(exc)) harbor = lhtb_root / ".venv" / "bin" / "harbor" check("Harbor", harbor.is_file() and os.access(harbor, os.X_OK), str(harbor)) - check("Codex binary", args.codex_bin.is_file() and os.access(args.codex_bin, os.X_OK), str(args.codex_bin)) + check( + "Codex binary", + args.codex_bin.is_file() and os.access(args.codex_bin, os.X_OK), + str(args.codex_bin), + ) check( "Codex code-mode sidecar", (args.codex_bin.parent / "codex-code-mode-host").is_file(), @@ -249,20 +243,45 @@ def check(label: str, passed: bool, detail: str) -> None: (args.portable_python / "bin" / "python3").is_file(), str(args.portable_python), ) - check("Node runtime", (args.node_dir / "bin" / "node").is_file(), str(args.node_dir)) + rc, node_version = command([str(args.node_dir / "bin" / "node"), "--version"]) + match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)", node_version) + check( + "Node runtime", + rc == 0 + and match is not None + and tuple(map(int, match.groups())) >= MINIMUM_NODE_VERSION, + f"{node_version}; requires >= {MINIMUM_NODE_VERSION_TEXT}", + ) - docker_source = lhtb_root / "upstream/harbor/src/harbor/environments/docker/docker.py" - patched = docker_source.is_file() and "LHTB_MODELONLY_NET" in docker_source.read_text(encoding="utf-8") + docker_source = ( + lhtb_root / "upstream/harbor/src/harbor/environments/docker/docker.py" + ) + patched = ( + docker_source.is_file() + and "LHTB_MODELONLY_NET" in docker_source.read_text(encoding="utf-8") + ) check("model-only Harbor patch", patched, str(docker_source)) rc, output = command(["docker", "info"]) check("Docker daemon", rc == 0, output.splitlines()[0] if output else "unavailable") - rc, output = command(["docker", "network", "inspect", args.network, "--format", "{{.Internal}}"]) - check("internal model network", rc == 0 and output == "true", f"{args.network}: {output or 'missing'}") + rc, output = command( + ["docker", "network", "inspect", args.network, "--format", "{{.Internal}}"] + ) + check( + "internal model network", + rc == 0 and output == "true", + f"{args.network}: {output or 'missing'}", + ) parsed = urlsplit(args.gateway) health = f"{parsed.scheme}://{parsed.netloc}/health" - rc, output = command(["curl", "-sS", "--noproxy", "*", "-m", "5", health], timeout=10) - check("model gateway", rc == 0 and bool(output), f"{health}: {(output or 'unreachable')[:120]}") + rc, output = command( + ["curl", "-sS", "--noproxy", "*", "-m", "5", health], timeout=10 + ) + check( + "model gateway", + rc == 0 and bool(output), + f"{health}: {(output or 'unreachable')[:120]}", + ) receipt = { "ok": not failures, @@ -270,7 +289,9 @@ def check(label: str, passed: bool, detail: str) -> None: "task_count": len(selected), "loopx_commit": actual_commit, "runtime_profile": "generic_cli", - "codex_driver": "fresh_exec_per_wake", + "execution_mode": execution.mode, + "iteration_context": execution.context, + "home_scope": "trial", "model": agent.get("model_name"), "reasoning_effort": kwargs.get("reasoning_effort"), "replan_after_completed_todos": 3, diff --git a/benchmark/LHTB/scripts/render_config.py b/benchmark/LHTB/scripts/render_config.py index 497989ed54..b27023536e 100755 --- a/benchmark/LHTB/scripts/render_config.py +++ b/benchmark/LHTB/scripts/render_config.py @@ -4,10 +4,12 @@ from __future__ import annotations import argparse +import json import re from pathlib import Path import yaml +from benchmark.runtime.codex import CONTEXTS, MODES, TASK_ENTRIES, Execution def main() -> int: @@ -21,6 +23,13 @@ def main() -> int: parser.add_argument("--effort", required=True) parser.add_argument("--timeout", type=int, required=True) parser.add_argument("--task", action="append", default=[]) + parser.add_argument("--execution-mode", choices=MODES, default="heartbeat") + parser.add_argument("--iteration-context", choices=CONTEXTS, default="fresh") + parser.add_argument("--task-entry", choices=TASK_ENTRIES, default="seeded-todo") + parser.add_argument("--planning-timeout", type=float, default=300) + parser.add_argument("--validation-command-json", default="[]") + parser.add_argument("--turn-timeout", type=float, default=4700) + parser.add_argument("--scheduler-timeout", type=int, default=5080) args = parser.parse_args() if not 1 <= args.concurrency <= 64: @@ -47,7 +56,23 @@ def main() -> int: agent["model_name"] = args.model agent["override_timeout_sec"] = args.timeout agent["kwargs"]["reasoning_effort"] = args.effort - agent["kwargs"]["goals"] = "false" + execution = Execution( + mode=args.execution_mode, + context=args.iteration_context, + timeout_seconds=args.turn_timeout, + validation_command=json.loads(args.validation_command_json), + task_entry=args.task_entry, + ) + agent["kwargs"].update( + execution_mode=execution.mode, + iteration_context=execution.context, + validation_command=list(execution.validation_command), + turn_timeout_sec=execution.timeout_seconds, + scheduler_timeout_sec=args.scheduler_timeout, + task_entry=execution.task_entry, + planning_timeout_sec=args.planning_timeout, + ) + agent["kwargs"]["goals"] = str(execution.native_goal).lower() agent["kwargs"]["web_search"] = "disabled" args.output.parent.mkdir(parents=True, exist_ok=True) diff --git a/benchmark/runtime/RUNTIME.md b/benchmark/runtime/RUNTIME.md new file mode 100644 index 0000000000..682d04942c --- /dev/null +++ b/benchmark/runtime/RUNTIME.md @@ -0,0 +1,187 @@ +# Shared Codex benchmark execution + +LHTB, SWE-Marathon and other Harbor tasks use +`benchmark.runtime.harbor:BenchmarkCodex`, with the repository root on +`PYTHONPATH`. This research runner is not another installed product package. +Native tasks, environment, phases, feedback, verifier and scores stay in Harbor. + +## Configure the native job + +Use this agent in the benchmark's existing job config, retaining its dataset +and environment settings: + +```yaml +agents: + - import_path: benchmark.runtime.harbor:BenchmarkCodex + model_name: openai/gpt-5.6-sol + override_timeout_sec: 5400 + kwargs: + execution_mode: heartbeat + task_entry: seeded-todo + iteration_context: fresh + reasoning_effort: max + codex_sandbox: danger-full-access + turn_timeout_sec: 4700 + scheduler_timeout_sec: 5080 + replan_after_todos: 3 +``` + +| Mode | Execution/continuation | LoopX skills and state | +| --- | --- | --- | +| `plain` | One Codex exec, native Goals disabled | Absent | +| `native-goal` | Installed native Goal transport; objective `Finish the task.` | Absent | +| `heartbeat` | Product thin heartbeat + external scheduler, fresh each wake | Present | +| `turn` | Public Turn CLI, typed result, independent validation, settlement | Present | +| `loopx-goal` | Product Goal body + installed native Goal transport | Present | + +Only `turn` accepts `iteration_context: resume-if-available`. Core session +compatibility determines whether it actually resumes, including after changing +Todo. Native Goal continuation stays with Codex; blocked Goals are not +automatically unblocked. Plain exec versus Goal app-server also changes +transport; it does not isolate the continuation effect alone. + +`turn` requires `validation_command`, an argv list for an independently +protected validator available inside the task environment. It receives the +normalized candidate result on stdin. Missing validation fails before execution. +The runner supplies no HEAD-moved/clean-worktree/exit-only substitute and never +calls hidden benchmark verification to provide intermediate feedback. Independent +validator protection remains the environment owner's responsibility. + +## Task entry and planning ablation + +`task_entry` is independent of the execution mode: + +- `seeded-todo` (the compatibility default) writes a generic execution Todo. + Follow-up phases update that Todo while it remains live and owned by this + agent; completed or deferred work gets a new Todo. Updates preserve blocked + state. The agent can still plan and replan during execution. +- `loopx-planned` runs the installed `$loopx` skill against the public + `loopx todo plan` checkpoint before execution. The checkpoint shares the + product's planner and continuation-aware Todo delta; it creates no planning + Todo and starts no host loop. Select it only for heartbeat, Turn or LoopX Goal. + +The model writes or reuses actual task Todos through the public CLI. The worker +reads the product packet again and checks the input digest, identity, Todo ids +and runnable/blocked state. A fabricated id, changed input, wrong owner, failed +planning process or missing result fails the entry; it never falls back to a +generic Todo. A blocked entry retains the referenced blockers and starts no +execution driver. Readback proves state and ownership, not semantic plan quality. + +Planning uses a separate fresh `codex exec` session with native Goals disabled +for that call. Its session is not inserted into core Turn session bindings or +resumed by the subsequent execution. This is a planning-contract ablation, not +an exact reproduction of same-conversation interactive `$loopx` startup. +The default `planning_timeout_sec` is 300; planning and preparation consume the +same `scheduler_timeout_sec` phase budget as execution. Planning sessions are +included in native session/token aggregation. No planning checkpoint is counted +as a completed advancement Todo or settled work Turn. + +Each phase keeps an immutable task document. New phases preserve Goal/Agent +identity and expose existing Todos to the planner; they do not clear waiting +state or force the agent active. An unresolved Turn must be recovered before +another phase can replace its task input. These wait/recovery rules apply to +both entry policies; they correct the earlier unconditional phase reset. +Every scheduler wake caps its host timeout against the remaining phase budget +before opening an execution. If only startup and settlement reserve remains, +it records a budget-exhausted no-op without creating a pending Turn. +The deadline uses the task environment's clock, including remote Harbor backends. + +To compare entry policies, hold the execution mode, session policy, model, +effort, tools, feedback and total budget fixed, and use separate trials: + +```yaml +kwargs: + execution_mode: heartbeat + task_entry: loopx-planned + planning_timeout_sec: 300 + iteration_context: fresh + turn_timeout_sec: 4700 + scheduler_timeout_sec: 5080 +``` + +## Install and isolate + +From the candidate worktree, provide: + +```sh +export LOOPX_SRC_DIR="$PWD" +export LOOPX_EXPECTED_COMMIT="$(git rev-parse HEAD)" +export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" +``` + +Also set `CODEX_OFFLINE_DIR` (Codex, code-mode sidecar, rg), +`LOOPX_PORTABLE_PYTHON` (Python >=3.11 distribution) and `LOOPX_NODE_DIR` +(Node >=22.18.0 distribution). Staging archives the verified commit SHA, never local run +artifacts. The host import must come from that checkout, whose tracked files +must match HEAD. Commit the candidate before real validation. Baselines stage only +the runner/native transport, without installing LoopX skills or initializing +its state. LoopX modes use the formal installer and doctor readback. + +Supply `OPENAI_BASE_URL`, `OPENAI_API_KEY` and optionally `CODEX_WIRE_API` +(default `responses`), or stage standard Codex authentication using +`CODEX_AUTH_JSON_PATH`. Credentials are excluded from session/log collection. + +Each trial owns one isolated Codex home. Config, provider and skills stay fixed; +fresh creates a new conversation and resume reuses a compatible conversation. +Workspace and LoopX state persist. Memory generation and injection are disabled +explicitly; confirm support in the pinned Codex version. Fresh does not make +historical files inaccessible or reset task work. Hold model, effort, tool, +feedback and time-budget settings fixed when comparing modes. + +`danger-full-access` explicitly delegates isolation to the task environment. +It does not certify that environment's security. Core Turn still defaults to +`read-only`; callers may select `workspace-write` or `read-only` consistently +across comparison arms. No wrapper silently replaces sandbox flags with bypass. +LoopX arms record the trial's task-workspace write authorization through +`configure-goal --boundary-authority-scope`. Turn carries that checkpointed +approval in its envelope; publishing and production actions keep their gates. + +Staging uses Harbor upload/exec methods, without Docker-label container discovery. +Backend-specific networking stays in the benchmark's native launcher. + +## Results, timeout and recovery + +Private per-wake receipts distinguish process success, Turn settlement and +native benchmark results. Sessions are collected once per native filename; +resume updates that copy instead of counting the old prefix in every wake. +Harbor converts each session independently and phase token counts use deltas. + +Timeout preserves partial task artifacts for native scoring. Goal receipts +retain the observed transaction on timeout. Cancellation reaps child processes. +Harbor deadlines must exceed scheduler deadlines, which must exceed host +timeouts plus validation and cleanup allowance. +The scheduler wake deadline is derived from the host timeout plus 150 seconds; +the retired LHTB `LOOPX_WAKE_TIMEOUT_SEC` setting is no longer used. + +Pending controlled Turns retain their identity across worker restarts. Core +`--resume-turn-key --retry-failed-turn` decides recovery eligibility and retry limits. The runner +does not delete homes or session bindings, edit registries directly, or +monkeypatch CLI internals. + +## Migration and qualification + +This is the native runtime bridge/adapter slice in the research program's +[engineering plan](../../docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.md#11-engineering-construction-plan). +Synthetic Harbor conformance does not qualify a matched study or a benchmark +score claim. Full LHTB and SWE-Marathon studies retain their own acceptance. + +Existing named LHTB/SWE agent imports remain thin compatibility entries; new +studies use the shared entry with explicit modes. Retired WEN controls fail +instead of silently changing their meaning. The old fixed-stage Turn and +automatic-unblock implementations are available in Git at +`8330a974cc2631ffd006d1fb7bd1627d2d690e85`. Historical results and withdrawal +notices retain their original provenance; no old scores are reassigned. + +LHTB now uses a **trial home instead of a per-wake home**. This is a disclosed +behavior change, not strict execution parity. Roll back by using the previous +revision and a new trial; do not rewrite active homes or historical receipts. + +```sh +uv run --extra test python -m pytest benchmark/tests/test_shared_codex_runtime.py \ + benchmark/tests/test_native_codex_goal.py tests/test_loopx_turn_codex_cli.py +``` + +Install the intended Harbor version for the adapter tests. Real qualification +also needs installed Codex, the native Harbor backend and independently checked +task output. Unit tests establish no score or model-uplift claim. Validate small +jobs through each benchmark's native configuration before launching a study. diff --git a/benchmark/runtime/__init__.py b/benchmark/runtime/__init__.py new file mode 100644 index 0000000000..82672617b8 --- /dev/null +++ b/benchmark/runtime/__init__.py @@ -0,0 +1 @@ +"""Shared research runner; not an installed LoopX product package.""" diff --git a/benchmark/runtime/codex.py b/benchmark/runtime/codex.py new file mode 100644 index 0000000000..4da2ff4fe7 --- /dev/null +++ b/benchmark/runtime/codex.py @@ -0,0 +1,142 @@ +"""Matched Codex settings and trial-local environment for Harbor runners.""" + +from __future__ import annotations + +import json +import math +import os +from dataclasses import dataclass +from pathlib import Path + + +MODES = ("plain", "native-goal", "heartbeat", "turn", "loopx-goal") +CONTEXTS = ("fresh", "resume-if-available") +TASK_ENTRIES = ("seeded-todo", "loopx-planned") +SANDBOXES = ("read-only", "workspace-write", "danger-full-access") + + +@dataclass(frozen=True) +class Execution: + mode: str = "heartbeat" + context: str = "fresh" + sandbox: str = "danger-full-access" + timeout_seconds: float = 4700 + validation_command: tuple[str, ...] = () + task_entry: str = "seeded-todo" + + def __post_init__(self) -> None: + if self.mode not in MODES or self.context not in CONTEXTS: + raise ValueError("unsupported execution mode or iteration context") + if self.task_entry not in TASK_ENTRIES: + raise ValueError("unsupported task entry") + if self.task_entry == "loopx-planned" and not self.uses_loopx: + raise ValueError("loopx-planned requires a LoopX execution mode") + if self.context != "fresh" and self.mode != "turn": + raise ValueError("resume-if-available currently requires mode=turn") + if self.sandbox not in SANDBOXES: + raise ValueError("unsupported Codex sandbox") + if not math.isfinite(self.timeout_seconds) or self.timeout_seconds <= 0: + raise ValueError("execution timeout must be finite and positive") + if not isinstance(self.validation_command, (list, tuple)): + raise ValueError("validation_command must be an argv list") + if any(not isinstance(arg, str) or not arg for arg in self.validation_command): + raise ValueError("validation_command must contain non-empty argv strings") + object.__setattr__(self, "validation_command", tuple(self.validation_command)) + if self.mode == "turn" and not self.validation_command: + raise ValueError("mode=turn requires an independent validation_command") + if self.mode != "turn" and self.validation_command: + raise ValueError("validation_command is only used by mode=turn") + + @property + def uses_loopx(self) -> bool: + return self.mode in {"heartbeat", "turn", "loopx-goal"} + + @property + def native_goal(self) -> bool: + return self.mode in {"native-goal", "loopx-goal"} + + +def prepare_codex_home( + home: Path, + *, + execution: Execution, + workspace: Path, + model: str, + effort: str, + base_url: str, + api_key: str, + wire_api: str, + skills: Path | None, +) -> None: + """Materialize fixed inputs once; never delete sessions between iterations. + + A different effective configuration requires a different trial. Secrets are + kept out of the settings identity and may rotate without resetting history. + """ + if not model or not effort: + raise ValueError("model and effort are required") + if base_url and not api_key: + raise ValueError("custom provider requires OPENAI_API_KEY") + if not base_url and not (home / "auth.json").is_file(): + raise ValueError("provide a gateway and API key, or stage CODEX_AUTH_JSON_PATH") + if wire_api not in {"responses", "chat"}: + raise ValueError("unsupported provider wire_api") + home.mkdir(parents=True, exist_ok=True) + settings = "\n".join( + [ + f"model = {json.dumps(model)}", + f"model_reasoning_effort = {json.dumps(effort)}", + 'approval_policy = "never"', + f"sandbox_mode = {json.dumps(execution.sandbox)}", + 'web_search = "disabled"', + f'model_provider = "{"harbor" if base_url else "openai"}"', + "[features]", + f"goals = {str(execution.native_goal).lower()}", + "unified_exec = true", + "[memories]", + "generate_memories = false", + "use_memories = false", + f"[projects.{json.dumps(str(workspace))}]", + 'trust_level = "trusted"', + ] + + ( + [ + "[model_providers.harbor]", + 'name = "harbor"', + f"base_url = {json.dumps(base_url)}", + f"wire_api = {json.dumps(wire_api)}", + 'env_key = "OPENAI_API_KEY"', + "request_max_retries = 8", + "stream_max_retries = 8", + "stream_idle_timeout_ms = 300000", + ] + if base_url + else [] + ) + + [""] + ) + config = home / "config.toml" + if config.exists() and config.read_text(encoding="utf-8") != settings: + raise ValueError("Codex settings changed within a trial; use a new trial") + if not config.exists(): + config.write_text(settings, encoding="utf-8") + config.chmod(0o444) + skills_link = home / "skills" + if execution.uses_loopx: + if skills is None or not skills.is_dir(): + raise ValueError("LoopX execution requires formally installed skills") + if not skills_link.exists(): + skills_link.symlink_to(skills.resolve(), target_is_directory=True) + if skills_link.resolve() != skills.resolve(): + raise ValueError("Codex skills changed within a trial") + elif skills_link.exists(): + raise ValueError("baseline Codex home must not contain LoopX skills") + # The provider reads OPENAI_API_KEY. No credential file is copied to logs. + + +def process_environment( + home: Path, *, base: dict[str, str] | None = None +) -> dict[str, str]: + env = dict(os.environ if base is None else base) + env["CODEX_HOME"] = str(home) + return env diff --git a/benchmark/runtime/codex_offline.py b/benchmark/runtime/codex_offline.py new file mode 100644 index 0000000000..84cd420098 --- /dev/null +++ b/benchmark/runtime/codex_offline.py @@ -0,0 +1,101 @@ +"""Offline Codex staging shared by Harbor benchmark adapters.""" + +import os +from pathlib import Path + +from harbor.agents.installed.codex import Codex +from harbor.environments.base import BaseEnvironment + +_DEFAULT_OFFLINE_DIR = str( + Path(__file__).resolve().parents[1] / "swe-marathon" / "codex" +) + +_STAGE_DIR = "/tmp/codex-offline" + +_RETRY_FLAGS = ( + "-c model_providers.harbor.name=harbor" + " -c model_providers.harbor.request_max_retries=8" + " -c model_providers.harbor.stream_max_retries=8" + " -c model_providers.harbor.stream_idle_timeout_ms=300000" +) + + +class CodexOffline(Codex): + def __init__(self, *args, goals="false", web_search="disabled", **kwargs): + if str(goals) not in {"true", "false"}: + raise ValueError("goals must be true or false") + if web_search not in {"disabled", "cached", "live"}: + raise ValueError("unsupported web_search setting") + self._goals = str(goals) + self._web_search = web_search + super().__init__(*args, **kwargs) + + @staticmethod + def name() -> str: + return "codex-offline" + + def version(self) -> str | None: + return self._version or "offline" + + def get_version_command(self) -> str | None: + return "/usr/local/bin/codex --version" + + def build_cli_flags(self) -> str: + flags = super().build_cli_flags() + return ( + f"{flags} -c features.goals={self._goals}" + f' -c web_search="{self._web_search}" {_RETRY_FLAGS}' + ).strip() + + async def install(self, environment: BaseEnvironment) -> None: + offline_dir = Path(os.environ.get("CODEX_OFFLINE_DIR", _DEFAULT_OFFLINE_DIR)) + codex_bin = (offline_dir / "codex").resolve() + rg_bin = (offline_dir / "rg").resolve() + sidecar_bin = (offline_dir / "codex-code-mode-host").resolve() + bwrap_bin = offline_dir / "codex-resources" / "bwrap" + if not codex_bin.is_file(): + raise FileNotFoundError( + f"离线 codex 二进制不存在: {codex_bin}。" + " 用 stage_codex_offline.sh 从宿主机的 @openai/codex 包里取出来。" + ) + if not sidecar_bin.is_file(): + raise FileNotFoundError( + f"codex-code-mode-host 不存在: {sidecar_bin}。" + " 重跑 stage_codex_offline.sh —— 旧版脚本只抠 codex 和 rg，" + " 缺 sidecar 会让容器里的工具面静默全废。" + ) + + await self.exec_as_root(environment, command=f"mkdir -p {_STAGE_DIR}") + + await environment.upload_file(codex_bin, f"{_STAGE_DIR}/codex") + await environment.upload_file(sidecar_bin, f"{_STAGE_DIR}/codex-code-mode-host") + if rg_bin.is_file(): + await environment.upload_file(rg_bin, f"{_STAGE_DIR}/rg") + if bwrap_bin.is_file(): + await environment.upload_file(bwrap_bin, f"{_STAGE_DIR}/bwrap") + + await self.exec_as_root( + environment, + command=( + "set -eu; " + f"install -m 0755 {_STAGE_DIR}/codex /usr/local/bin/codex; " + f"install -m 0755 {_STAGE_DIR}/codex-code-mode-host " + " /usr/local/bin/codex-code-mode-host; " + f"if [ -f {_STAGE_DIR}/rg ]; then " + f" install -m 0755 {_STAGE_DIR}/rg /usr/local/bin/rg; " + "fi; " + f"if [ -f {_STAGE_DIR}/bwrap ]; then " + f" install -m 0755 {_STAGE_DIR}/bwrap /usr/local/bin/bwrap; " + "fi; " + f"rm -rf {_STAGE_DIR}; " + "mkdir -p /logs/agent; " + "{ /usr/local/bin/codex --version; " + " md5sum /usr/local/bin/codex /usr/local/bin/codex-code-mode-host; " + "} > /logs/agent/codex_version.txt 2>&1; " + "cat /logs/agent/codex_version.txt" + ), + ) + + self.logger.info( + f"codex 离线安装完成（来源 {offline_dir}，含 code-mode sidecar）" + ) diff --git a/benchmark/runtime/harbor.py b/benchmark/runtime/harbor.py new file mode 100644 index 0000000000..b2c7270114 --- /dev/null +++ b/benchmark/runtime/harbor.py @@ -0,0 +1,680 @@ +"""Shared Harbor adapter. Native tasks, phases, feedback and scoring stay in Harbor.""" + +from __future__ import annotations + +import json +import os +import shlex +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Iterable + +from harbor.agents.installed.base import with_prompt_template +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from harbor.models.trajectories import FinalMetrics, Trajectory +from harbor.utils.trajectory_utils import format_trajectory_json + +from .codex_offline import CodexOffline +from .codex import Execution + + +_ROOT = "/opt/loopx-benchmark" +_SRC = f"{_ROOT}/source" +_PYTHON = f"{_ROOT}/python" +_NODE = f"{_ROOT}/node" +_PROFILE = f"{_ROOT}/profile" +_PROFILE_HOME = f"{_PROFILE}/home" +_SHARED_CODEX_HOME = f"{_PROFILE}/codex-home" +_SHARED_SKILLS = f"{_SHARED_CODEX_HOME}/skills" +_CLI = f"{_PROFILE}/bin/loopx" +_CONTROL = f"{_ROOT}/control" +_REGISTRY = f"{_CONTROL}/registry.json" +_LOOPX_RUNTIME = f"{_ROOT}/state/runtime" +_SCHEDULER_STATE = f"{_CONTROL}/scheduler-state.json" +_TASK_DOC = f"{_CONTROL}/task.md" +_BASH_ENV = f"{_CONTROL}/bash-env" +_CODEX_HOME = f"{_ROOT}/codex-home" +_WORKER_MODULE = "benchmark.runtime.worker" +_WAKE_LOG_DIR = "/logs/agent/wakes" +_GOAL_ID = "benchmark-goal" +_AGENT_ID = "benchmark-agent" + + +class BenchmarkCodex(CodexOffline): + """One independent LoopX control plane per Harbor trial.""" + + def __init__( + self, + *args, + execution_mode="heartbeat", + iteration_context="fresh", + codex_sandbox="danger-full-access", + validation_command=None, + turn_timeout_sec=4700, + scheduler_timeout_sec=5080, + replan_after_todos=3, + task_entry="seeded-todo", + planning_timeout_sec=300, + **kwargs, + ): + if isinstance(validation_command, str): + raise ValueError("validation_command must be an argv list, not shell text") + self.execution = Execution( + execution_mode, + iteration_context, + codex_sandbox, + float(turn_timeout_sec), + validation_command if validation_command is not None else (), + task_entry, + ) + self.planning_timeout = float(planning_timeout_sec) + if not 0 < self.planning_timeout < float("inf"): + raise ValueError("planning timeout must be finite and positive") + self.scheduler_timeout = int(scheduler_timeout_sec) + if self.scheduler_timeout <= self.execution.timeout_seconds + 150: + raise ValueError( + "scheduler timeout must exceed turn timeout plus cleanup allowance" + ) + self.replan_after_todos = int(replan_after_todos) + if not 1 <= self.replan_after_todos <= 5: + raise ValueError("replan_after_todos must be between 1 and 5") + self._phase_number = 0 + self._seeded_todo_id: str | None = None + super().__init__(*args, **kwargs) + + @staticmethod + def name() -> str: + return "benchmark-codex" + + async def _stage_source(self, environment: BaseEnvironment, source: Path) -> str: + if Path(__file__).resolve() != source / "benchmark/runtime/harbor.py": + raise RuntimeError("Harbor must import the adapter from LOOPX_SRC_DIR") + dirty = subprocess.run( + ["git", "-C", str(source), "diff", "HEAD", "--quiet"], + check=False, + ) + if dirty.returncode: + raise RuntimeError("Commit tracked source changes before staging a trial") + head = subprocess.run( + ["git", "-C", str(source), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + expected = os.environ.get("LOOPX_EXPECTED_COMMIT", head) + if head != expected: + raise RuntimeError("LoopX source does not match LOOPX_EXPECTED_COMMIT") + # Never upload the checkout, local experiment outputs or trajectories. + with tempfile.TemporaryDirectory(prefix="benchmark-source-") as directory: + archive = Path(directory) / "source.tar" + command = ["git", "-C", str(source), "archive", "--format=tar", head] + if not self.execution.uses_loopx: + command += [ + "benchmark/runtime", + "loopx/capabilities/benchmark_toolkit/native_codex_goal.py", + ] + with archive.open("wb") as output: + subprocess.run(command, stdout=output, check=True, timeout=120) + await environment.upload_file(archive, f"{_ROOT}/source.tar") + await self.exec_as_root( + environment, + command=f"tar -xf {_ROOT}/source.tar -C {_SRC} && rm {_ROOT}/source.tar", + timeout_sec=180, + ) + return head + + def _profile_env(self) -> dict[str, str]: + return { + "HOME": _PROFILE_HOME, + "CODEX_HOME": _SHARED_CODEX_HOME, + "PATH": f"{_NODE}/bin:{_PROFILE}/bin:/usr/local/bin:/usr/bin:/bin", + "LOOPX_PYTHON": f"{_PYTHON}/bin/python3", + "LOOPX_PROMOTE_DEFAULT": "1", + "LOOPX_INSTALL_CANARY": "0", + "LOOPX_BIN_DIR": f"{_PROFILE}/bin", + "LOOPX_RELEASES_DIR": f"{_PROFILE}/releases", + "LOOPX_RELEASE_ID": "benchmark-runtime", + "LOOPX_MAN_ROOT": f"{_PROFILE}/man", + "LOOPX_MAN_DIR": f"{_PROFILE}/man/man1", + "LOOPX_SHELL_PROFILE": f"{_PROFILE_HOME}/.profile", + "LOOPX_SKILLS_DIR": _SHARED_SKILLS, + "LOOPX_INSTALL_SLASH_COMMANDS": "0", + "LOOPX_INSTALL_OPENCODE": "0", + "LOOPX_INSTALL_CLAUDE": "0", + "LOOPX_SKILL_DEDUPE_OTHER_ROOT": "0", + # Codex tool calls use `bash -lc`, whose login profile may replace + # PATH. BASH_ENV restores the staged Node for LoopX subprocesses. + "BASH_ENV": _BASH_ENV, + } + + async def install(self, environment: BaseEnvironment) -> None: + await super().install(environment) + + loopx_src = Path(os.environ["LOOPX_SRC_DIR"]).resolve() + portable_python = Path(os.environ["LOOPX_PORTABLE_PYTHON"]).resolve() + node_root = Path(os.environ["LOOPX_NODE_DIR"]).resolve() + await self.exec_as_root( + environment, + command=( + f"mkdir -p {_SRC} {_PYTHON} {_NODE} {_PROFILE_HOME} " + f"{_SHARED_CODEX_HOME} {_PROFILE}/bin {_PROFILE}/releases {_PROFILE}/man " + f"{_CONTROL} " + f"{_LOOPX_RUNTIME} {_CODEX_HOME} " + f"{_WAKE_LOG_DIR}; chmod -R 0777 {_ROOT} {_WAKE_LOG_DIR}" + ), + timeout_sec=180, + ) + actual_commit = await self._stage_source(environment, loopx_src) + await environment.upload_dir(portable_python, _PYTHON) + if self.execution.uses_loopx: + await environment.upload_dir(node_root, _NODE) + await self.exec_as_root( + environment, + command=( + f"printf '%s\\n' 'export PATH={_NODE}/bin:$PATH' > {_BASH_ENV}; " + f"chmod 0644 {_BASH_ENV}; " + f"find {_SRC} -maxdepth 2 \\( -name '*.egg-info' -o " + f"-name '*.dist-info' \\) -exec rm -rf {{}} +; " + f"chmod -R a+rX {_SRC} {_PYTHON} {_NODE}; " + f"chmod -R a+rwX {_PROFILE} {_CONTROL} {_CODEX_HOME} {_WAKE_LOG_DIR}" + ), + timeout_sec=300, + ) + auth_path = self._get_env("CODEX_AUTH_JSON_PATH") + if auth_path: + await environment.upload_file( + Path(auth_path).resolve(), f"{_CODEX_HOME}/auth.json" + ) + owner = shlex.quote(str(environment.default_user or "root")) + await self.exec_as_root( + environment, + command=( + f"chown {owner} {_CODEX_HOME}/auth.json && chmod 0600 {_CODEX_HOME}/auth.json" + ), + ) + if self.execution.uses_loopx: + install = await self.exec_as_agent( + environment, + command=f"bash {_SRC}/scripts/install-local.sh", + env=self._profile_env(), + timeout_sec=1200, + ) + if "error" in (install.stderr or "").lower(): + self.logger.debug("LoopX installer stderr: %s", install.stderr[-1000:]) + + doctor = await self.exec_as_agent( + environment, + command=f"{_CLI} --format json doctor --agent-type codex-cli", + env=self._profile_env(), + timeout_sec=300, + ) + try: + doctor_payload = json.loads(doctor.stdout or "") + except json.JSONDecodeError as exc: + raise RuntimeError("LoopX doctor returned invalid JSON") from exc + if doctor_payload.get("ok") is not True: + raise RuntimeError(f"LoopX doctor failed: {doctor_payload}") + + receipt = { + "loopx_commit": actual_commit, + "runtime_profile": "generic_cli", + "execution_mode": self.execution.mode, + "iteration_context": self.execution.context, + "task_entry": self.execution.task_entry, + "home_scope": "trial", + "login_shell_node_path": _BASH_ENV, + "scheduler_terminal_packet_compatibility": True, + "replan_after_completed_todos": self.replan_after_todos, + } + await self.exec_as_agent( + environment, + command=( + f"printf %s {shlex.quote(json.dumps(receipt, sort_keys=True))} " + f"> /logs/agent/loopx-install.json" + ), + env=self._profile_env(), + ) + + async def _write_task_document( + self, environment: BaseEnvironment, instruction: str + ) -> None: + descriptor, name = tempfile.mkstemp(prefix="benchmark-task-", suffix=".md") + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write("# Current benchmark task\n\n") + handle.write(instruction.strip()) + handle.write("\n") + await environment.upload_file(Path(name), _TASK_DOC) + await environment.upload_file(Path(name), self._task_document) + await self.exec_as_root( + environment, + command=f"chmod 0644 {_TASK_DOC} {self._task_document}", + ) + finally: + Path(name).unlink(missing_ok=True) + + @property + def _task_document(self) -> str: + # Existing Todos keep their original input when Harbor supplies a new phase. + return f"{_CONTROL}/task-phase-{self._phase_number:03d}.md" + + async def _loopx( + self, + environment: BaseEnvironment, + args: list[str], + *, + cwd: str, + require_ok: bool = True, + ) -> dict: + argv = [ + _CLI, + "--format", + "json", + "--registry", + _REGISTRY, + "--runtime-root", + _LOOPX_RUNTIME, + *args, + ] + result = await self.exec_as_agent( + environment, + command=shlex.join(argv), + env=self._profile_env(), + cwd=cwd, + timeout_sec=300, + ) + text = (result.stdout or "").strip() + if text.startswith("```"): + text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip() + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"LoopX command returned invalid JSON: {text[:300]}" + ) from exc + if require_ok and payload.get("ok") is False: + raise RuntimeError(f"LoopX command failed: {payload.get('error')}") + return payload + + async def _registry_exists(self, environment: BaseEnvironment) -> bool: + result = await environment.exec(command=f"test -s {_REGISTRY}") + return result.return_code == 0 + + async def _prepare_phase( + self, environment: BaseEnvironment, instruction: str, *, cwd: str + ) -> None: + pending = await environment.exec( + command=f"test -e {_LOOPX_RUNTIME}/benchmark-pending-turn.json" + ) + if pending.return_code == 0: + raise RuntimeError("Resolve the pending Turn before entering another task phase") + await self._write_task_document(environment, instruction) + if not await self._registry_exists(environment): + await self._loopx( + environment, + [ + "bootstrap", + "--project", + ".", + "--goal-id", + _GOAL_ID, + "--objective", + "Complete the current benchmark task through validated LoopX Todos.", + "--goal-doc", + _TASK_DOC, + "--adapter-kind", + "read_only_project_map_v0", + "--adapter-status", + "connected-read-only", + "--write-scope", + "**", + "--no-global-sync", + ], + cwd=cwd, + ) + await self._loopx( + environment, + [ + "configure-goal", + "--goal-id", + _GOAL_ID, + "--registered-agent", + _AGENT_ID, + "--boundary-authority-scope", + "**", + "--boundary-authority-source", + "harbor-task-workspace", + "--boundary-authority-decision-id", + "trial-workspace", + "--execution-replan-after-todos", + str(self.replan_after_todos), + "--agent-work-mode", + f"{_AGENT_ID}=active", + "--execute", + ], + cwd=cwd, + ) + else: + # New input is not evidence that an existing wait or gate was resolved. + await self._loopx( + environment, + [ + "configure-goal", + "--goal-id", + _GOAL_ID, + "--execution-replan-after-todos", + str(self.replan_after_todos), + "--execute", + ], + cwd=cwd, + ) + + if self.execution.task_entry == "seeded-todo": + await self._seed_phase(environment, cwd=cwd) + + cadence = await self._loopx( + environment, ["configure-goal", "--goal-id", _GOAL_ID], cwd=cwd, + ) + configured_state = cadence.get("after") or cadence.get("before") or {} + configured = configured_state.get("execution_profile", {}).get("replan_after_completed_todos") + if configured != self.replan_after_todos: + raise RuntimeError( + f"replan cadence readback mismatch: expected {self.replan_after_todos}, got {configured!r}" + ) + + async def _seed_phase(self, environment: BaseEnvironment, *, cwd: str) -> None: + text = ( + f"[P0] Execute benchmark phase {self._phase_number}. Read the exact " + f"current task from {self._task_document}; inspect the workspace, implement and " + "validate it, and create bounded successor Todos for remaining work." + ) + if self._seeded_todo_id: + listed = await self._loopx(environment, [ + "todo", "list", "--goal-id", _GOAL_ID, "--role", "agent", + "--todo-id", self._seeded_todo_id, + ], cwd=cwd) + current = next(iter(listed["todos"]), None) + if current and current.get("status") in {"open", "blocked"}: + if current.get("claimed_by") != _AGENT_ID: + raise RuntimeError("Seeded task Todo is no longer owned by this agent") + # New phase input revises our still-live generic task; do not + # strand it behind an unfinished predecessor or clear a wait. + await self._loopx(environment, [ + "todo", "update", "--goal-id", _GOAL_ID, + "--todo-id", self._seeded_todo_id, "--agent-id", _AGENT_ID, + "--text", text, "--execute", + ], cwd=cwd) + return + created = await self._loopx( + environment, + [ + "todo", + "add", + "--goal-id", + _GOAL_ID, + "--role", + "agent", + "--text", + text, + "--task-class", + "advancement_task", + "--action-kind", + "benchmark_task", + "--claimed-by", + _AGENT_ID, + "--status", + "open", + "--execute", + ], + cwd=cwd, + ) + self._seeded_todo_id = created["todo_id"] + + def _worker_env(self, *, cwd: str) -> dict[str, str]: + env = self._profile_env() + if not self.execution.uses_loopx: + env["PATH"] = "/usr/local/bin:/usr/bin:/bin" + env.update( + { + "PYTHONPATH": _SRC, + "LOOPX_CLI": _CLI, + "LOOPX_REGISTRY": _REGISTRY, + "LOOPX_RUNTIME_ROOT": _LOOPX_RUNTIME, + "LOOPX_GOAL_ID": _GOAL_ID, + "LOOPX_AGENT_ID": _AGENT_ID, + "LOOPX_PROJECT": cwd, + "LOOPX_TASK_DOC": self._task_document, + "LOOPX_WAKE_LOG_DIR": _WAKE_LOG_DIR, + "LOOPX_CODEX_HOME": _CODEX_HOME, + "LOOPX_SHARED_SKILLS": _SHARED_SKILLS, + "LOOPX_EXECUTION_MODE": self.execution.mode, + "LOOPX_TASK_ENTRY": self.execution.task_entry, + "LOOPX_ITERATION_CONTEXT": self.execution.context, + "LOOPX_CODEX_SANDBOX": self.execution.sandbox, + "LOOPX_VALIDATION_COMMAND_JSON": json.dumps( + self.execution.validation_command + ), + "LOOPX_CODEX_TURN_TIMEOUT_SEC": str(self.execution.timeout_seconds), + "CODEX_BIN": "/usr/local/bin/codex", + "MODEL_NAME": (self.model_name or "").split("/", 1)[-1], + "REASONING_EFFORT": str( + self._resolved_flags.get("reasoning_effort", "max") + ), + "OPENAI_BASE_URL": self._get_env("OPENAI_BASE_URL") or "", + "OPENAI_API_KEY": self._get_env("OPENAI_API_KEY") or "", + "CODEX_WIRE_API": self._get_env("CODEX_WIRE_API") or "responses", + } + ) + return env + + def _session_trajectories(self, roots: Iterable[Path]) -> list[Trajectory]: + sessions: set[Path] = set() + for root in roots: + if root.is_dir(): + sessions.update(root.glob("sessions/**/*.jsonl")) + trajectories: list[Trajectory] = [] + for session in sorted(sessions): + # Harbor's parser takes a directory. Isolate each native rollout + # so two sessions on the same date cannot collapse into one. + with tempfile.TemporaryDirectory(prefix="benchmark-session-") as directory: + (Path(directory) / session.name).symlink_to(session.resolve()) + trajectory = self._convert_events_to_trajectory(Path(directory)) + if trajectory is not None: + trajectories.append(trajectory) + return trajectories + + @staticmethod + def _totals(trajectories: Iterable[Trajectory]) -> dict[str, int | float | None]: + prompt = completion = cached = 0 + costs: list[float] = [] + for trajectory in trajectories: + metrics = trajectory.final_metrics + if metrics is None: + continue + prompt += metrics.total_prompt_tokens or 0 + completion += metrics.total_completion_tokens or 0 + cached += metrics.total_cached_tokens or 0 + if metrics.total_cost_usd is not None: + costs.append(metrics.total_cost_usd) + return { + "prompt": prompt, + "completion": completion, + "cached": cached, + "cost": sum(costs) if costs else None, + } + + def _write_aggregate_trajectory(self) -> list[Trajectory]: + trajectories = self._session_trajectories([self.logs_dir]) + if not trajectories: + return [] + steps = [] + for trajectory in trajectories: + for step in trajectory.steps: + copied = step.model_copy(deep=True) + copied.step_id = len(steps) + 1 + steps.append(copied) + totals = self._totals(trajectories) + aggregate = Trajectory( + schema_version="ATIF-v1.5", + session_id=f"benchmark-{self.logs_dir.parent.name}", + agent=trajectories[0].agent, + steps=steps, + final_metrics=FinalMetrics( + total_prompt_tokens=totals["prompt"] or None, + total_completion_tokens=totals["completion"] or None, + total_cached_tokens=totals["cached"] or None, + total_cost_usd=totals["cost"], + total_steps=len(steps), + extra={"native_sessions": len(trajectories)}, + ), + ) + (self.logs_dir / "trajectory.json").write_text( + format_trajectory_json(aggregate.to_json_dict()), encoding="utf-8" + ) + return trajectories + + def _populate_context( + self, context: AgentContext, before: dict | None = None + ) -> None: + totals = self._totals(self._session_trajectories([self.logs_dir])) + before = before or {} + context.n_input_tokens = int(totals["prompt"] or 0) - int( + before.get("prompt") or 0 + ) + context.n_output_tokens = int(totals["completion"] or 0) - int( + before.get("completion") or 0 + ) + context.n_cache_tokens = int(totals["cached"] or 0) - int( + before.get("cached") or 0 + ) + context.cost_usd = ( + totals["cost"] - (before.get("cost") or 0) + if totals["cost"] is not None + else None + ) + context.metadata = { + "execution_mode": self.execution.mode, + "iteration_context": self.execution.context, + "task_entry": self.execution.task_entry, + "home_scope": "trial", + "replan_after_completed_todos": self.replan_after_todos, + "benchmark_phase": self._phase_number, + } + self._write_aggregate_trajectory() + + def populate_context_post_run(self, context: AgentContext) -> None: + self._populate_context(context) + + @with_prompt_template + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + if not self.model_name: + raise ValueError("model_name is required") + self._phase_number += 1 + deadline = time.monotonic() + self.scheduler_timeout + pwd = await self.exec_as_agent(environment, command="pwd", timeout_sec=30) + cwd = (pwd.stdout or "").strip() + if not cwd.startswith("/"): + raise RuntimeError( + f"could not resolve container working directory: {cwd!r}" + ) + + before = self._totals(self._session_trajectories([self.logs_dir])) + try: + if self.execution.uses_loopx: + await self._prepare_phase(environment, instruction, cwd=cwd) + else: + await self._write_task_document(environment, instruction) + wake_command = [f"{_PYTHON}/bin/python3", "-m", _WORKER_MODULE] + env = self._worker_env(cwd=cwd) + if self.execution.task_entry == "loopx-planned": + result_path = f"{_CONTROL}/planning-phase-{self._phase_number:03d}.json" + planning_timeout = min(self.planning_timeout, deadline - time.monotonic() - 30) + if planning_timeout <= 0: + raise TimeoutError("Task budget exhausted before planning") + await self.exec_as_agent( + environment, command=shlex.join(wake_command), cwd=cwd, + env=env | { + "LOOPX_TASK_STAGE": "plan", + "LOOPX_PLANNING_TIMEOUT_SEC": str(planning_timeout), + "LOOPX_PLANNING_RESULT": result_path, + }, + timeout_sec=planning_timeout + 30, + ) + observed = await environment.exec(command=f"cat {result_path}") + entry = json.loads(observed.stdout or "") + if entry.get("state_readback_verified") is not True: + raise RuntimeError("Planning did not return verified Todo readback") + if entry["status"] == "blocked": + return + remaining = int(deadline - time.monotonic()) + if remaining <= 160: + raise TimeoutError("Task budget exhausted before execution handoff") + # Planning consumes the phase budget, including when the host later resumes. + # Keep ten seconds for scheduler startup before the worker checks + # its execution window plus the existing 150-second settlement reserve. + host_timeout = min(self.execution.timeout_seconds, remaining - 160) + env["LOOPX_CODEX_TURN_TIMEOUT_SEC"] = str(host_timeout) + if self.execution.mode in {"heartbeat", "turn"}: + command = [ + f"{_PYTHON}/bin/python3", + f"{_SRC}/scripts/external_scheduler_worker.py", + "--cli-bin", + _CLI, + "--registry", + _REGISTRY, + "--runtime-root", + _LOOPX_RUNTIME, + "--runtime-profile", + "generic_cli", + "--goal-id", + _GOAL_ID, + "--agent-id", + _AGENT_ID, + "--state-file", + _SCHEDULER_STATE, + "--wake-cmd", + "exec " + shlex.join(wake_command), + "--wake-timeout-seconds", + str(host_timeout + 150), + "--quota-timeout-seconds", + "30", + "--error-backoff-seconds", + "15", + ] + else: + command = wake_command + phase_log = f"/logs/agent/worker-phase-{self._phase_number:03d}.log" + shell = ( + "set +e; " + # Use the task environment's clock, including remote backends. + f"export LOOPX_PHASE_DEADLINE_EPOCH=$(( $(date +%s) + {remaining} )); " + f"timeout --signal=TERM --kill-after=30 {remaining}s " + f"{shlex.join(command)} >> {shlex.quote(phase_log)} 2>&1; " + "rc=$?; " + # Budget exhaustion retains partial artifacts for native scoring. + 'if [ "$rc" -eq 124 ]; then exit 0; fi; exit "$rc"' + ) + await self.exec_as_agent( + environment, + command=shell, + env=env, + cwd=cwd, + timeout_sec=remaining + 60, + ) + finally: + # Remote Harbor backends download logs after run(). Read them now + # before populating a non-empty context, which Harbor will retain. + mounted = getattr(environment, "is_mounted", None) + if mounted is None: + mounted = environment.capabilities.mounted + if not mounted: + await environment.download_dir("/logs/agent", self.logs_dir) + self._populate_context(context, before) diff --git a/benchmark/runtime/planning.py b/benchmark/runtime/planning.py new file mode 100644 index 0000000000..0d558c8e6b --- /dev/null +++ b/benchmark/runtime/planning.py @@ -0,0 +1,78 @@ +"""Consume the product task-planning checkpoint and verify its state readback.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + + +def task_plan_packet(env: dict[str, str], cli: list[str]) -> dict: + command = cli + [ + "todo", + "plan", + "--goal-id", + env["LOOPX_GOAL_ID"], + "--agent-id", + env["LOOPX_AGENT_ID"], + "--project", + env["LOOPX_PROJECT"], + "--text", + Path(env["LOOPX_TASK_DOC"]).read_text(encoding="utf-8"), + ] + response = subprocess.run( + command, + cwd=env["LOOPX_PROJECT"], + env=env, + text=True, + capture_output=True, + check=True, + timeout=120, + ) + packet = json.loads(response.stdout) + if ( + packet.get("ok") is not True + or packet.get("schema_version") != "loopx_task_planning_v0" + or packet.get("goal_id") != env["LOOPX_GOAL_ID"] + or packet.get("agent_id") != env["LOOPX_AGENT_ID"] + or packet.get("execution_handoff", {}).get("owner") != "caller" + ): + raise ValueError("product planning packet does not match the caller binding") + return packet + + +def validate_plan_readback(result: dict, before: dict, after: dict) -> dict: + if ( + result.get("input_digest") != before["input_digest"] + or after["input_digest"] != before["input_digest"] + ): + raise ValueError("planning input changed before readback") + status = result.get("status") + ids = result.get("todo_ids") + if ( + status not in {"ready", "blocked"} + or not isinstance(ids, list) + or not ids + or any(not isinstance(item, str) for item in ids) + or len(set(ids)) != len(ids) + ): + raise ValueError( + "planning result requires unique actual Todo ids and a typed status" + ) + todos = {item["todo_id"]: item for item in after["existing_todos"]} + if any(todo_id not in todos for todo_id in ids): + raise ValueError("planning referenced a missing or unrelated Todo") + if status == "ready" and not set(ids).issubset(after["runnable_todo_ids"]): + raise ValueError("planning referenced non-runnable or unclaimed work") + if status == "blocked" and not set(ids).issubset(after["blocking_todo_ids"]): + raise ValueError("blocked planning result requires unresolved blocking Todos") + return { + "input_digest": before["input_digest"], + "status": status, + "todo_ids": ids, + "goal_id": after["goal_id"], + "agent_id": after["agent_id"], + "state_readback_verified": True, + "execution_owner": "caller", + "planning_session_reused_for_execution": False, + } diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py new file mode 100644 index 0000000000..afd9fafd64 --- /dev/null +++ b/benchmark/runtime/worker.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""Execute one admitted wake in the task environment, through product APIs.""" + +from __future__ import annotations + +import json +import os +import shutil +import signal +import subprocess +import time +import uuid +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path + +from benchmark.runtime.codex import Execution, prepare_codex_home, process_environment + + +def read_json(text: str) -> dict: + payload = json.loads(text) + if not isinstance(payload, dict): + raise ValueError("expected a JSON object") + return payload + + +def loopx_command(env: dict[str, str]) -> list[str]: + return [ + env["LOOPX_CLI"], + "--format", + "json", + "--registry", + env["LOOPX_REGISTRY"], + "--runtime-root", + env["LOOPX_RUNTIME_ROOT"], + ] + + +def heartbeat_body(env: dict[str, str], turn_id: str, *, native_goal: bool) -> str: + command = loopx_command(env) + [ + "heartbeat-prompt", + "--thin", + "--runtime-profile", + "codex_app_ssh_goal" if native_goal else "generic_cli", + "--goal-id", + env["LOOPX_GOAL_ID"], + "--agent-id", + env["LOOPX_AGENT_ID"], + "--cli-bin", + env["LOOPX_CLI"], + "--available-capability", + "shell", + "--available-capability", + "filesystem_write", + ] + if not native_goal: + command += ["--turn-instance-id", turn_id] + result = subprocess.run( + command, + cwd=env["LOOPX_PROJECT"], + env=env, + capture_output=True, + text=True, + timeout=120, + check=True, + ) + payload = read_json(result.stdout) + if payload.get("ok") is not True or not payload.get("task_body"): + raise RuntimeError("formal LoopX heartbeat renderer is not ready") + if not native_goal and payload.get("turn_instance_id") != turn_id: + raise RuntimeError("heartbeat Turn identity mismatch") + return payload["task_body"].replace( + "$HOME/.codex/loopx/registry.global.json", env["LOOPX_REGISTRY"] + ) + + +@contextmanager +def child_process(command: list[str], *, env: dict[str, str], stdout, stderr): + process = subprocess.Popen( + command, + cwd=env["LOOPX_PROJECT"], + env=env, + stdin=subprocess.PIPE, + stdout=stdout, + stderr=stderr, + text=True, + start_new_session=True, + ) + try: + yield process + finally: + # Reap descendants even when their leader exited or Harbor cancelled us. + try: + # Python's standard SIGINT handler lets the Turn host's finally + # path reap its separately grouped Codex child before forced kill. + os.killpg(process.pid, signal.SIGINT) + except ProcessLookupError: + pass + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=10) + + +def turn_command( + env: dict[str, str], + execution: Execution, + turn_id: str, + resume_turn_key: str | None = None, +) -> list[str]: + return loopx_command(env) + [ + "turn", + "run-once", + "--execute", + "--host", + "codex-cli", + "--project", + env["LOOPX_PROJECT"], + "--goal-id", + env["LOOPX_GOAL_ID"], + "--agent-id", + env["LOOPX_AGENT_ID"], + "--scheduler-owner", + "outer_controller", + *( + ["--resume-turn-key", resume_turn_key, "--retry-failed-turn"] + if resume_turn_key + else ["--turn-instance-id", turn_id] + ), + "--iteration-context", + execution.context, + "--codex-bin", + env["CODEX_BIN"], + "--codex-model", + env["MODEL_NAME"], + "--codex-sandbox", + execution.sandbox, + "--timeout-seconds", + str(execution.timeout_seconds), + "--validation-command-json", + json.dumps(execution.validation_command), + "--available-capability", + "shell", + "--available-capability", + "filesystem_write", + ] + + +def run_native_goal( + env: dict[str, str], execution: Execution, body: str, receipt: dict, stderr +) -> None: + from loopx.capabilities.benchmark_toolkit.native_codex_goal import ( + NativeGoalConfig, + NativeGoalDeadlineExceeded, + StdioNativeGoalTransport, + compact_native_goal_receipt, + run_native_goal_until_terminal, + ) + + required_skills = () + if execution.uses_loopx: + from loopx.capabilities.benchmark_toolkit.native_codex_profile import ( + NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS, + ) + + required_skills = NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS + + observed = [] + config = NativeGoalConfig( + cwd=env["LOOPX_PROJECT"], + objective=body, + task_instruction=Path(env["LOOPX_TASK_DOC"]).read_text(encoding="utf-8"), + model=env["MODEL_NAME"], + effort=env["REASONING_EFFORT"], + sandbox=execution.sandbox, + approval_policy="never", + required_skill_ids=required_skills, + ) + with child_process( + [env["CODEX_BIN"], "app-server"], env=env, stdout=subprocess.PIPE, stderr=stderr + ) as process: + transport = StdioNativeGoalTransport(process, response_timeout_sec=120) + try: + run_native_goal_until_terminal( + transport, + config, + timeout_sec=execution.timeout_seconds, + on_turn_started=observed.append, + ) + receipt["ok"] = True + except NativeGoalDeadlineExceeded: + receipt["timed_out"] = True + finally: + if observed: + receipt["native_goal"] = compact_native_goal_receipt(observed[0]) + + +def run_once(env: dict[str, str]) -> dict: + execution = Execution( + mode=env.get("LOOPX_EXECUTION_MODE", "heartbeat"), + context=env.get("LOOPX_ITERATION_CONTEXT", "fresh"), + sandbox=env.get("LOOPX_CODEX_SANDBOX", "danger-full-access"), + timeout_seconds=float(env.get("LOOPX_CODEX_TURN_TIMEOUT_SEC", "4700")), + validation_command=json.loads(env.get("LOOPX_VALIDATION_COMMAND_JSON", "[]")), + task_entry=env.get("LOOPX_TASK_ENTRY", "seeded-todo"), + ) + stage = env.get("LOOPX_TASK_STAGE", "execute") + if stage not in {"plan", "execute"} or (stage == "plan" and execution.task_entry != "loopx-planned"): + raise ValueError("invalid task-entry stage") + turn_id = f"wake-{time.time_ns()}-{uuid.uuid4().hex[:12]}" + log_root = Path(env["LOOPX_WAKE_LOG_DIR"]) + wake = log_root / turn_id + wake.mkdir(parents=True, exist_ok=False) + home = Path(env["LOOPX_CODEX_HOME"]) + env = process_environment(home, base=env) + env["LOOPX_TURN"] = turn_id + receipt = { + "turn_id": turn_id, + "mode": execution.mode, + "context": execution.context, + "task_entry": execution.task_entry, + "stage": stage, + "home_scope": "trial", + "ok": False, + "timed_out": False, + } + pending_path = ( + Path(env.get("LOOPX_RUNTIME_ROOT", str(home))) / "benchmark-pending-turn.json" + ) + try: + if stage == "execute" and env.get("LOOPX_PHASE_DEADLINE_EPOCH"): + remaining = float(env["LOOPX_PHASE_DEADLINE_EPOCH"]) - time.time() + # Reserve startup and settlement on every wake, then allow the + # remaining time for work instead of reusing the initial timeout. + if remaining <= 160: + receipt.update(ok=True, budget_exhausted=True, host_invoked=False) + return receipt + execution = replace( + execution, timeout_seconds=min(execution.timeout_seconds, remaining - 160) + ) + prepare_codex_home( + home, + execution=execution, + workspace=Path(env["LOOPX_PROJECT"]), + model=env["MODEL_NAME"], + effort=env["REASONING_EFFORT"], + base_url=env.get("OPENAI_BASE_URL", ""), + api_key=env.get("OPENAI_API_KEY", ""), + wire_api=env.get("CODEX_WIRE_API", "responses"), + skills=Path(env["LOOPX_SHARED_SKILLS"]) if execution.uses_loopx else None, + ) + body = "Finish the task." + if stage == "plan": + from benchmark.runtime.planning import task_plan_packet + + planning_before = task_plan_packet(env, loopx_command(env)) + body = "$loopx\n\nHost-supplied planning checkpoint:\n" + json.dumps(planning_before, ensure_ascii=False) + (wake / "planning-input.json").write_text(body, encoding="utf-8") + (wake / "planning-schema.json").write_text(json.dumps(planning_before["result_schema"])) + elif execution.mode in {"heartbeat", "loopx-goal"}: + body = heartbeat_body(env, turn_id, native_goal=execution.native_goal) + elif execution.mode == "plain": + body = Path(env["LOOPX_TASK_DOC"]).read_text(encoding="utf-8") + with (wake / "stderr.log").open("w") as stderr: + if execution.native_goal and stage == "execute": + run_native_goal(env, execution, body, receipt, stderr) + else: + pending = {} + if execution.mode == "turn" and stage == "execute": + pending_path.parent.mkdir(parents=True, exist_ok=True) + if pending_path.exists(): + pending = read_json(pending_path.read_text()) + else: + pending = {"turn_instance_id": turn_id} + pending_path.write_text(json.dumps(pending)) + command = ( + turn_command( + env, + execution, + pending["turn_instance_id"], + pending.get("resume_turn_key"), + ) + if execution.mode == "turn" and stage == "execute" + else [ + env["CODEX_BIN"], + "exec", + "--json", + "--skip-git-repo-check", + "--sandbox", + execution.sandbox, + "--cd", + env["LOOPX_PROJECT"], + *([ + "-c", "features.goals=false", + "--output-schema", str(wake / "planning-schema.json"), + "--output-last-message", str(wake / "planning-result.json"), + ] if stage == "plan" else []), + "-", + ] + ) + with (wake / "stdout.jsonl").open("w") as stdout: + with child_process( + command, env=env, stdout=stdout, stderr=stderr + ) as process: + allowance = 150 if execution.mode == "turn" and stage == "execute" else 0 + timeout = (float(env["LOOPX_PLANNING_TIMEOUT_SEC"]) if stage == "plan" + else execution.timeout_seconds + allowance) + process.communicate( + input=body, timeout=timeout + ) + receipt["return_code"] = process.returncode + receipt["ok"] = process.returncode == 0 + if stage == "plan" and receipt["ok"]: + from benchmark.runtime.planning import task_plan_packet, validate_plan_readback + + result = read_json((wake / "planning-result.json").read_text()) + receipt["planning"] = validate_plan_readback( + result, planning_before, task_plan_packet(env, loopx_command(env)), + ) + target = Path(env["LOOPX_PLANNING_RESULT"]) + temporary = target.with_suffix(".tmp") + temporary.write_text(json.dumps(receipt["planning"])) + temporary.replace(target) + elif execution.mode == "turn" and stage == "execute": + result = read_json((wake / "stdout.jsonl").read_text()) + receipt["turn_execution"] = result + receipt["ok"] = receipt["ok"] and result.get("ok") is True + if receipt["ok"]: + pending_path.unlink() + elif result.get("resume_turn_key"): + # Product recovery owns eligibility and retry limits. + # Never disguise a failed transaction as a fresh Turn. + pending["resume_turn_key"] = result["resume_turn_key"] + temporary = pending_path.with_suffix(".tmp") + temporary.write_text(json.dumps(pending)) + temporary.replace(pending_path) + except subprocess.TimeoutExpired: + receipt["ok"] = False + receipt["timed_out"] = True + except BaseException as exc: + receipt["ok"] = False + receipt["error_kind"] = type(exc).__name__ + raise + finally: + if (home / "sessions").is_dir(): + # One authoritative copy per native session; resume must not count + # the same prefix again in every wake's aggregate trajectory. + shutil.copytree( + home / "sessions", log_root.parent / "sessions", dirs_exist_ok=True + ) + (wake / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + return receipt + + +def main() -> int: + def cancelled(signum, frame): + raise KeyboardInterrupt("worker cancelled") + + signal.signal(signal.SIGTERM, cancelled) + receipt = run_once(dict(os.environ)) + print(json.dumps(receipt)) + return 124 if receipt["timed_out"] else (0 if receipt["ok"] else 1) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/swe-marathon/README.md b/benchmark/swe-marathon/README.md index 5efe94b847..5414000029 100644 --- a/benchmark/swe-marathon/README.md +++ b/benchmark/swe-marathon/README.md @@ -77,7 +77,7 @@ agents/ Harbor 适配器（LoopX treatment + codex 原生 goal baseline） scoring/ 评分/聚合/可视化（口径见 _common.py） skills/ 历史五模式 benchmark skill（不代表当前公开证据范围） -runtime/ 模式框架 + turn 驱动，含 automation 唤醒循环 loopx_turn_runner.py（见 runtime/RUNTIME.md） +runtime/ 共享执行入口与迁移说明；实现位于 ../runtime/（见 runtime/RUNTIME.md） data.json pinned public-safe 聚合产物 case_insights.json 非官方 draft case-insight 记录（scoring/case_insights.py 由 data.json 生成） ``` diff --git a/benchmark/swe-marathon/agents/codex_goal_agent.py b/benchmark/swe-marathon/agents/codex_goal_agent.py index 48b0da64e4..2369565ddb 100644 --- a/benchmark/swe-marathon/agents/codex_goal_agent.py +++ b/benchmark/swe-marathon/agents/codex_goal_agent.py @@ -1,487 +1,14 @@ -"""codex 原生 goal 模式的 harbor agent。 +"""Compatibility entry for existing Harbor agent configs. -与基线 `CodexOffline` 的**唯一差别**是不用 `codex exec`，改用 -app-server + `thread/goal/set`。其余一切沿用：同一个离线二进制安装、同样的 -provider/auth 配置、同样的 harbor trial 流程、同样的 verifier 调用。 - -## 为什么必须做成 agent 类而不是脚本 - -打分要公平，就得走 harbor 原路：同样的 trial 目录结构、同样的 -`continue_until_timeout` 语义、同样的中途/最终 verifier 调用、同样的 -`result.json`。脚本跑出来的东西和基线不可比。 - -## 为什么不能用 environment.exec - -app-server 要持久双向 stdio，而 harbor 的 exec 把 stdin 设成 DEVNULL -（`docker.py` 里 `stdin=asyncio.subprocess.DEVNULL`）。agent 类本身跑在宿主机， -所以自己 `docker exec -i` 拿管道，codex 仍然跑在容器内——文件系统语义才对。 - -## goal 臂相对基线恰好多出三样（不多不少） - - 1. 每轮注入的 5338 字符 `` - 2. `update_goal` 工具 - 3. codex 的自动续跑调度 - -用法（config.yaml）： - agents: - - import_path: codex_goal_agent:CodexGoalAgent - model_name: openai/gpt-5.5 - override_timeout_sec: 5400 - kwargs: - reasoning_effort: medium +Use benchmark.runtime.harbor:BenchmarkCodex with execution_mode for new studies. +Historical assisted WEN modes are retired; results remain tied to their old revision. """ - -import asyncio -import json import os -import time -import shlex -import subprocess -import sys -from pathlib import Path - -from harbor.agents.installed.codex import EnvironmentPaths -from harbor.environments.base import BaseEnvironment - -from codex_offline import CodexOffline - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from native_codex_goal import ( # noqa: E402 - NativeGoalConfig, - NativeGoalProtocolError, - StdioNativeGoalTransport, - compact_native_goal_receipt, - refresh_native_goal_status, - start_native_goal_turn, - wait_native_goal_turn, -) - -# objective 用 LoopX 自己的写法（测试夹具与 CLI 测试路径都是这句）。 -# -# 极简通用、不含任务内容，三条理由： -# 1. codex 对 objective 有 4000 字符硬上限，46 个任务里 28 个指令超限， -# 而且超限的恰好是长程硬任务。完整指令走 turn 输入（不受限），一字不删。 -# 2. 逐任务撰写验收标准 = 给 goal 臂一条基线没有的信息通道， -# 测出来的会是"写标准的水平"而不是 goal 机制的价值。 -# 3. objective 的内容正是 LoopX treatment 臂的变量，基线臂先占了就没得比。 -_OBJECTIVE = "Finish the task." - -# goal 的单次时限。harbor 的 agent 基类**没有** _timeout_sec，agent 拿不到自己 -# 的预算，超时是 trial 从外面掐的。所以这里只能给一个略小于 override_timeout_sec -# (5400) 的值，让内部循环自己抛超时、走到 finally 落 receipt，而不是被外部 -# cancel 掉丢证据。 -# -# 可用 GOAL_TIMEOUT_SEC 覆盖——冒烟测试要在几分钟内触发超时路径（那条路径正是 -# 之前丢掉 20/27 份 receipt 的地方），不能等 90 分钟。 -# -# 注意 continue_until_timeout 会多次调 run()，后续阶段的剩余时间递减，这个常量 -# 会大于剩余量——那时仍会被外部掐断，靠 CancelledError 分支兜住。 -_GOAL_TIMEOUT_SEC = float(os.environ.get("GOAL_TIMEOUT_SEC") or 5340.0) - -# app-server 读 config.toml，`-c` 那套是给 `codex exec` 的，所以基线用 CLI flag -# 传的东西这里必须写进文件。内容与基线逐字对应： -# web_search="disabled" 基线 build_cli_flags() 里加的 -# retries 三件套 基线 _RETRY_FLAGS -# 注意**不写** features.goals——goal 臂就是要它开着。 -# -# 【踩过的坑】不能像上游那样分两次 `cat >>` 追加。TOML 里 table 头之后的裸键 -# 都归该 table，所以第二段开头的 `web_search = "disabled"` 会变成 -# `model_providers.harbor.web_search`；而再写一次 `[model_providers.harbor]` -# 是重复声明，直接解析失败。表现是每个 trial 都秒挂在 -# `app_server_request_failed:thread/start`。所以整份文件一次性写出，自己控制顺序。 -_CONFIG_TOML = """\ -web_search = "disabled" -# 关掉 codex 的**内层**沙箱。容器本身就是隔离边界，再套一层 bubblewrap 只会 -# 在启动时报 -# Codex's Linux sandbox uses bubblewrap and needs access to create user namespaces. -# 并让 app-server 的 initialize 握手失败（app_server_request_failed:initialize）。 -# 宿主机 kernel.unprivileged_userns_clone=1，是容器内的 seccomp/apparmor 挡住了 -# user namespace —— 装上 bwrap 二进制并不等于它能用。 -# -# 这与基线臂的 `codex exec --dangerously-bypass-approvals-and-sandbox` 等价： -# 两臂都把隔离交给容器，工具面一致。 -sandbox_mode = "danger-full-access" -# 把任务目录标成受信项目，否则 app-server 启动就报 -# Project-local config, hooks, and exec policies are disabled ... /app/.codex -projects."/app" = { trust_level = "trusted" } -model_provider = "harbor" -[model_providers.harbor] -name = "harbor" -base_url = "${OPENAI_BASE_URL}" -wire_api = "%(wire_api)s" -env_key = "OPENAI_API_KEY" -request_max_retries = 8 -stream_max_retries = 8 -stream_idle_timeout_ms = 300000 -""" - - -class CodexGoalAgent(CodexOffline): - # 驱动中的 turn，异常路径也要能落进 receipt（见 _drive 的说明） - _turn = None - - @staticmethod - def name() -> str: - return "codex-goal" - - # ── 容器定位 ────────────────────────────────────────────────────────── - def _container_id(self, environment: BaseEnvironment) -> str: - """拿到本 trial 的容器 ID。 - - harbor 用 docker compose 起容器，project name 是 session_id 消毒后的值、 - 服务名固定 `main`。按 compose 标签查比解析 compose 文件列表稳。 - """ - from harbor.environments.docker.docker import ( - _sanitize_docker_compose_project_name, - ) - - project = _sanitize_docker_compose_project_name(environment.session_id) - out = subprocess.run( - ["docker", "ps", "-q", - "--filter", f"label=com.docker.compose.project={project}", - "--filter", "label=com.docker.compose.service=main"], - capture_output=True, text=True, timeout=60, - ).stdout.split() - if not out: - raise RuntimeError(f"找不到 trial 容器（compose project={project}）") - return out[0] - - def _container_cwd(self, cid: str) -> str: - """任务的工作目录，从容器镜像的 WorkingDir 读。 - - 【踩过的坑】原来写死 `/app` 兜底。46 个任务镜像里 45 个确实是 /app， - 但 `tabular-data-feature-covshift` 是 **/workspace**——`docker exec -w /app` - 直接失败，而且 **docker 把这个错误写到 stdout**： - - OCI runtime exec failed: chdir to cwd ("/app") ... no such file or directory - - 于是 stderr 空、stdout 冒出一行非 JSON，撞上 LoopX 的 fail-closed 检查 - 抛 app_server_frame_invalid_json，整个 trial 作废且无从追查（tee 都没 - 执行到，raw stdout 也没有）。基线不受影响，因为上游走 exec_as_agent， - 用的是容器自己的 WORKDIR。 - """ - out = subprocess.run( - ["docker", "inspect", "-f", "{{.Config.WorkingDir}}", cid], - capture_output=True, text=True, timeout=60, - ).stdout.strip() - return out or "/app" - - def _user_flag(self, environment: BaseEnvironment) -> list[str]: - """任务要求非 root 时，给 `docker exec` 补上 `-u`。 - - 【公平性缺陷，必须修】上游走 `exec_as_agent`，harbor 会按 task.toml 的 - `user =` 降权；我这条 `docker exec` 路绕开了它，而 46 个镜像的 - `Config.User` 全是空 —— 于是**默认以 root 运行**。 - - 46 个任务里只有 `sudoku-recovery` 设了 `user = "agent"`，它的 task.toml - 写明这是反作弊基石：非 root 才读不到 /opt/sudoku/private 的 oracle 与 - 密钥、改不了引擎的节奏下限、杀不掉 root daemon。不补这个 flag， - goal / LoopX 臂就比基线和 Terminus 多一份权限——本轮审计确认没被利用 - （敏感路径命中 0），但敞口不能留着。 - """ - user = getattr(environment, "default_user", None) - return ["-u", str(user)] if user else [] - - # ── 环境准备 ────────────────────────────────────────────────────────── - async def _prepare(self, environment: BaseEnvironment) -> dict[str, str]: - """写 auth.json + config.toml，与上游 Codex.run() 的准备段等价。""" - codex_home = self._REMOTE_CODEX_HOME.as_posix() - secrets = self._REMOTE_CODEX_SECRETS_DIR.as_posix() - auth_path = (self._REMOTE_CODEX_SECRETS_DIR / "auth.json").as_posix() - env = {"CODEX_HOME": codex_home} - - await self.exec_as_agent( - environment, - command=(f'mkdir -p "$CODEX_HOME" {shlex.quote(secrets)} ' - f"{shlex.quote(EnvironmentPaths.agent_dir.as_posix())}"), - env=env, - ) - - setup = ( - f"cat >{shlex.quote(auth_path)} <` 而不是 `>>`：整份一次性写出，避免 table 作用域和重复声明问题 - setup += ( - '\ncat >"$CODEX_HOME/config.toml" < None: - if not self.model_name: - raise ValueError("Model name is required") - model = self.model_name.split("/")[-1] - - await self._prepare(environment) - cid = self._container_id(environment) - - # 凭证必须在宿主机环境里，才能被 docker exec -e 转发进容器。 - # 缺了不会报错，只会变成静默空转（见下面 command 里的注释）， - # 所以在这里硬失败——空转比失败难发现得多。 - for key in ("OPENAI_API_KEY", "OPENAI_BASE_URL"): - if not os.environ.get(key): - raise RuntimeError( - f"{key} 不在 harbor 进程的环境里，app-server 拿不到凭证，" - "会静默空转。检查 run_codex.sh 是否 export 了它。" - ) - - cwd = str(self._resolve_flag_values().get("cwd") or self._container_cwd(cid)) - - # treatment 臂（LoopX）在这里装 profile、渲染 goal body。 - # 基线 goal 臂是空实现，不产生任何行为差异。 - # instruction 挂到实例上供钩子取用（LoopX 的 --goal-doc 要它）。 - self._pending_instruction = instruction - # 任务要求的运行用户（非 root 时 LoopX 需要把 profile 目录 chown 过去）。 - self._task_user = getattr(environment, "default_user", None) - self._treatment_setup(cid, cwd) - - # app-server 的 stdout 经 tee 留证 + 过滤后才交给 transport。 - # - # 【踩过的坑】LoopX 的 _read_stream 对非 JSON 行是 fail-closed 的：一行 - # 不合法就抛 app_server_frame_invalid_json，整个 90 分钟的 trial 直接作废 - # （tabular-data-feature-covshift 就这么丢的，且 stderr 为空、无从还原）。 - # 协议层严格是对的，但一行杂音不该毁掉一次评测。 - # - # 所以：tee 一份原始 stdout 到容器里留证，再只把 `{` 开头的行喂给 - # transport。这样既不改 LoopX 的代码，下次出问题也能捞到那行到底是什么。 - inner = ( - "codex app-server --listen stdio:// " - "--enable goals --enable unified_exec" - ) - piped = ( - f"{inner} | tee /tmp/goal_raw_stdout.jsonl " - "| grep --line-buffered '^{'" - ) - command = [ - "docker", "exec", "-i", - # 【踩过的坑】必须把凭证转发进去。config.toml 里 env_key = - # "OPENAI_API_KEY"，基线是靠 exec_as_agent(env=...) 注入的，而 - # docker exec 这条路绕开了那个机制。漏掉的表现极隐蔽：轮次能起来、 - # goal_context 能注入，但模型请求根本不发，turn 立刻 complete， - # codex 见目标仍 active 又排一轮 —— 每 0.7 秒一圈的死循环， - # 1.5 小时空转 17945 轮、0 次 token_count、0 条 error 记录。 - # - # 用裸变量名（不带 =value）让 docker 从宿主机环境转发， - # 这样密钥不会出现在进程命令行里被 ps 看到。 - "-e", "OPENAI_API_KEY", - "-e", "OPENAI_BASE_URL", - "-e", f"CODEX_HOME={self._REMOTE_CODEX_HOME.as_posix()}", - ] - # 与基线 exec_as_agent 对齐的降权（详见 _user_flag）。 - command += self._user_flag(environment) - for k, v in self._extra_exec_env().items(): - command += ["-e", f"{k}={v}"] - command += [ - "-w", cwd, - cid, - "sh", "-c", piped, - ] - config = NativeGoalConfig( - cwd=cwd, - objective=self._objective(), - required_skill_ids=self._required_skill_ids(), - task_instruction=instruction, # harbor 已渲染，与基线同一份 - model=model, - effort=str(self._resolve_flag_values().get("reasoning_effort") or "medium"), - approval_policy="never", - # 基线是 --dangerously-bypass-approvals-and-sandbox，展开就是这个。 - # 不要用 LoopX 参考实现默认的 workspace-write——那会让 goal 臂弱于基线。 - sandbox="danger-full-access", - token_budget=None, # 基线没有预算概念，设了就多一个基线没有的机制 - ) - - out_dir = self.logs_dir - out_dir.mkdir(parents=True, exist_ok=True) - turn = None - err_path = out_dir / "goal_app_server.stderr" - - # transport 提到线程外持有：harbor 的超时是从外面掐的（agent 基类没有 - # _timeout_sec，拿不到自己的预算），被 cancel 时 asyncio.to_thread 里的 - # 线程不会停，app-server 会一直挂着、receipt 也丢。持有引用才能在 - # CancelledError 里主动 close()，让线程抛错退出、走到 finally 落 receipt。 - err = open(err_path, "w") - transport = StdioNativeGoalTransport.spawn( - command, cwd="/tmp", response_timeout_sec=180, stderr=err - ) - try: - turn = await asyncio.to_thread(self._drive, transport, config) - # 【踩过的坑】这里**不能**写 `except NativeGoalProtocolError`。 - # 同名异常类存在两份，来自两个不同模块： - # agents/native_codex_goal.py ← 符号链接到 wen/loopx 源码树（本文件用） - # loopx.capabilities.benchmark_toolkit.native_codex_goal ← 装在 .venv - # （codex_loopx_agent / codex_plain_appserver 用） - # 两者互不为子类。按类捕获的后果是**只对 goal 臂生效**： - # goal 臂的超时被吞掉、照常交给 verifier 打分；LoopX 三臂的超时逃到 - # harbor，被当成基础设施故障 → 重试烧掉 1.5–3 小时 → 最终记成 errored， - # 已完成的部分工作全部丢弃、不进评分。 - # 这是第四次"只打一边"的偏差，方向是压低 treatment 臂。 - # 超时是长程任务的**正常预算耗尽**，五臂必须一视同仁按部分进度评分。 - # 改按消息判定：两个类都继承 RuntimeError，消息不匹配的照样重抛。 - except RuntimeError as exc: - if str(exc) != "goal_timeout_before_terminal": - raise - self.logger.warning("goal 超时终止（预期行为）：%s", exc) - except asyncio.CancelledError: - self.logger.warning("harbor 掐断了 agent 阶段，关闭 app-server") - raise - finally: - try: - transport.close() - finally: - err.close() - self._save_sessions(cid) - self._write_receipt(out_dir, self._turn or turn, config) - self._assert_not_spinning(self._turn or turn) - - def _drive(self, transport, config): - """自己驱动续跑循环，而不是调 run_native_goal_until_terminal。 - - 【为什么不用现成的】那个函数超时时**抛异常而不返回 turn**，turn 对象 - 在函数内部就丢了。结果是最需要过程证据的场景（长程任务跑满时间）反而 - 什么都留不下——27 个已完成 trial 里 20 个的 receipt 是空壳。 - 这里把 turn 存到 self._turn，异常路径也能落到 receipt 里。 - - 循环体只有排事件和轮询状态，**不再调 turn/start**——续跑轮次由 codex - 自己排，这是续跑归属的全部依据，不能自己造轮次。 - """ - turn = start_native_goal_turn(transport, config) - self._turn = turn - deadline = time.monotonic() + _GOAL_TIMEOUT_SEC - completed_before = turn.turn_completed_count - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise NativeGoalProtocolError("goal_timeout_before_terminal") - try: - wait_native_goal_turn( - transport, turn, timeout_sec=remaining, - completed_before=completed_before, - ) - except NativeGoalProtocolError as exc: - if str(exc) == "goal_turn_timeout": - raise NativeGoalProtocolError( - "goal_timeout_before_terminal") from exc - raise - completed_before = turn.turn_completed_count - if refresh_native_goal_status(transport, turn) != "active": - return turn - - def _save_sessions(self, cid: str) -> None: - """把 codex 的 session rollout / goals 库 / 原始 stdout 拷进 /logs/agent。 - - 上游 Codex.run() 的 finally 里做了拷 sessions 这件事，我覆盖了整个 run() - 却没带上，导致 46 轮里 27 个已完成 trial 的轨迹**完全没有保存**， - 而容器随即被删除、永久丢失。 - - **故意用同步 subprocess 而不是 await exec_as_agent**：harbor 从外面掐断 - agent 阶段时（continue_until_timeout 的后续阶段剩余时间 < _GOAL_TIMEOUT_SEC - 就会发生），finally 里的 await 会在 CancelledError 传播中立刻再次抛出， - 拷贝根本执行不到——而那恰恰是长程任务最需要留证的场景。 - 同步调用不碰事件循环，取消中照样能跑完。 - """ - agent_dir = EnvironmentPaths.agent_dir.as_posix() - home = self._REMOTE_CODEX_HOME.as_posix() - script = ( - f'mkdir -p {agent_dir}; ' - f'if [ -d "{home}/sessions" ]; then ' - f' rm -rf {agent_dir}/sessions; cp -R "{home}/sessions" {agent_dir}/sessions; ' - f'fi; ' - f'cp "{home}"/goals_1.sqlite* {agent_dir}/ 2>/dev/null; ' - f'cp /tmp/goal_raw_stdout.jsonl {agent_dir}/ 2>/dev/null; true' - ) - try: - subprocess.run(["docker", "exec", cid, "sh", "-c", script], - capture_output=True, timeout=180) - except Exception as exc: - self.logger.warning("保存 goal 轨迹失败: %s", exc) - - @staticmethod - def _assert_not_spinning(turn) -> None: - """空转检测：起了很多轮却一个 item 都没产生，说明模型压根没被调到。 - - 2026-08-23 第二次开跑就栽在这：漏传 OPENAI_API_KEY，轮次能起、 - goal_context 能注入，但请求不发、turn 立刻 complete、codex 见目标仍 - active 又排一轮，每 0.7 秒一圈。1.5 小时空转 17945 轮， - **没有任何错误记录**，只有会话文件涨到 116MB 才看得出不对。 - - 与其让它安静地烧满 90 分钟再拿 0 分，不如让这个 trial 明确失败。 - """ - if turn is None: - return - turns = getattr(turn, "turn_started_count", 0) or 0 - items = getattr(turn, "item_event_count", 0) or 0 - if turns >= 20 and items == 0: - raise RuntimeError( - f"goal 空转：起了 {turns} 轮但 item_event_count=0，" - "模型未被真正调用（多半是凭证没进到 app-server）" - ) - - def _write_receipt(self, out_dir: Path, turn, config) -> None: - payload = {"objective_source": _OBJECTIVE, - "cwd": config.cwd, "model": config.model, - "effort": config.effort, "sandbox": config.sandbox, - "goal_timeout_sec": _GOAL_TIMEOUT_SEC} - if turn is not None: - payload.update(compact_native_goal_receipt(turn)) - # LoopX 臂的解锁次数存在 self._unblock_count 上，compact_native_goal_receipt - # 只认 turn 对象、带不出来。不记进 receipt 的话，报分时无法回答 - # "这个分数用了几次 harness 干预"——那是必须披露的。 - if getattr(self, "_unblock_count", None) is not None: - payload["_unblock_count"] = self._unblock_count - else: - payload["execution_mode"] = "goal_failed_before_receipt" - - # continue_until_timeout 会多次调 run()，每个阶段一份 receipt。 - # 只写固定文件名的话后面的阶段会把前面的覆盖掉——实测 - # langchain-version-migration 跑了 4 个阶段（每阶段模型都宣布 complete， - # 前 3 次被中途 verifier 驳回），最后只剩阶段 4 的数据。 - # 所以逐阶段追加进 jsonl，同时保留 goal_receipt.json 指向最后一个阶段。 - payload["phase"] = self._phase = getattr(self, "_phase", 0) + 1 - with (out_dir / "goal_receipts.jsonl").open("a") as fh: - fh.write(json.dumps(payload, sort_keys=True, ensure_ascii=False) + "\n") - (out_dir / "goal_receipt.json").write_text( - json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) - ) - - # ── treatment 钩子（基线臂全为空实现，子类覆盖）────────────────────── - # - # 这四个钩子是 LoopX treatment 臂唯一的接入点。基线 goal 臂走默认实现时 - # 行为与加钩子之前**逐字节相同**，所以两臂仍然只差 LoopX 那三样东西。 - - def _treatment_setup(self, cid: str, cwd: str) -> None: - """装 treatment 需要的东西（LoopX profile 等）。基线臂无操作。""" - return None - - def _objective(self) -> str: - """goal 的 objective。基线臂用 LoopX 夹具的那句极简写法。""" - return _OBJECTIVE - - def _required_skill_ids(self) -> tuple: - """交给 native_codex_goal 的 skills/list 保真度门禁。基线臂不需要。""" - return () - - def _extra_exec_env(self) -> dict: - """追加给 app-server 的环境变量（LoopX 要改 HOME/PATH）。""" - return {} +from benchmark.runtime.harbor import BenchmarkCodex + +class CodexGoalAgent(BenchmarkCodex): + def __init__(self, *args, **kwargs): + if any(os.environ.get(key) for key in ("WEN_MODE", "WEN_CLAIM_CODEX_APP", "LOOPX_UNGATED")): + raise ValueError("Retired WEN controls: select an explicit execution_mode; see runtime/RUNTIME.md") + kwargs.setdefault("execution_mode", "native-goal") + super().__init__(*args, **kwargs) diff --git a/benchmark/swe-marathon/agents/codex_loopx_agent.py b/benchmark/swe-marathon/agents/codex_loopx_agent.py index 9922cb547d..366d2e421b 100644 --- a/benchmark/swe-marathon/agents/codex_loopx_agent.py +++ b/benchmark/swe-marathon/agents/codex_loopx_agent.py @@ -1,604 +1,14 @@ -"""codex + LoopX treatment 臂的 harbor agent。 +"""Compatibility entry for existing Harbor agent configs. -继承 `CodexGoalAgent`（codex 原生 goal），在其之上加 LoopX 的三样东西——这正是 -LoopX 自己 `benchmark/deepswe/README.md` 定义的 treatment 与 baseline 的差别： - - 1. **skills**：LoopX 的 6 个 skill 装进 app-server 用的 `CODEX_HOME` - 2. **CLI**：`loopx` 可执行文件进 `PATH`，goal body 里指名的就是它 - 3. **goal body**：objective 从 `"Finish the task."` 换成 - `loopx heartbeat-prompt --thin` 渲染出的 thin dispatcher - -保真度靠 `required_skill_ids` 把关——`native_codex_goal.py` 会在 `thread/start` -之前发 `skills/list`，codex 没真的发现那些 skill 就直接失败，**一个 token 都不花**。 -README 原话：*"A filesystem check alone is not treatment-fidelity evidence."* - -## 为什么 LoopX 要装在容器里 - -codex 跑在容器内，goal body 让 agent 每轮调 `loopx` CLI 查状态——那个 CLI 必须 -在容器里可执行。所以整个 profile（含 skills、CLI、registry）都装进容器。 - -## 离线安装怎么做到的 - -- LoopX `dependencies = []`，零第三方依赖，`install-local.sh` 只拷文件+生成 wrapper， - 全脚本没有一处网络访问。 -- 但它要求 Python ≥3.11，而 46 个任务镜像里有 5 个是 3.10。所以**统一挂载一份 - 可移植 python**（uv 的 python-build-standalone），46 个容器一视同仁， - 避免 41/5 两套环境。 +Use benchmark.runtime.harbor:BenchmarkCodex with execution_mode for new studies. +Historical assisted WEN modes are retired; results remain tied to their old revision. """ - -from __future__ import annotations - -import json import os -import re -import subprocess -from pathlib import Path - -from harbor.environments.base import BaseEnvironment - -from codex_goal_agent import CodexGoalAgent, _GOAL_TIMEOUT_SEC -from loopx.capabilities.benchmark_toolkit.native_codex_goal import ( - NativeGoalProtocolError, -) - -# 容器内的路径。profile 布局照抄 benchmark_toolkit 的 `_profile_paths()`， -# 这样 LoopX 自己的 inspect/doctor 逻辑对得上。 -_SRC = "/opt/loopx-src" # LoopX 源码（docker cp 进去） -_PY = "/opt/loopx-py" # 可移植 python -_NODE = "/opt/loopx-node" # 可移植 node（doctor 的 TS 运行时必需检查要它） -_ROOT = "/opt/lxprofile" -_HOME = f"{_ROOT}/home" -_CODEX_HOME = f"{_ROOT}/codex-home" -_BIN = f"{_ROOT}/bin" -_CLI = f"{_BIN}/loopx" -_REGISTRY = f"{_ROOT}/registry.json" -_RUNTIME = f"{_ROOT}/runtime" - -_AGENT_ID = "lhtb-agent" - -# ── 两个变体开关（env 驱动，同时写进 receipt 以便事后追溯用的是哪一版）──── -# -# LOOPX_GOAL_DOC=1 -# bootstrap 时带 --goal-doc <任务原文>。LoopX 文档规定的接入方式是 -# 「--objective 一句话 + --goal-doc 全文」，全文会被登记为 primary -# authority source。第一版漏了这个参数，状态文件里明写 -# "No explicit goal document was provided during bootstrap"， -# 导致 LoopX 的 Next Action 一直钉在自带的 onboarding 项上、 -# Progress Ledger 只有 bootstrap 一条，任务进展没沉淀进持久状态。 -# -# 注意 46 个镜像里只有 6 个自带 /app/instruction.md，所以统一由 agent -# 把 harbor 传来的 instruction 写进容器，不依赖镜像。 -# -# LOOPX_GOAL_ID_MODE=task -# goal_id 用任务名而不是固定的 lhtb-goal。LoopX 文档只要求 "stable goal id"， -# benchmark 相关文档（deepswe README / RFC / 单元测试）**都没有规定** -# benchmark 场景该怎么取，所以两种都不违反约定。 -# 固定值的好处是 46 份 goal body 只差 goal-doc 一项；任务名的好处是更贴合 -# LoopX「一个项目一个持久目标」的语义。 -_GOAL_DOC = bool(os.environ.get("LOOPX_GOAL_DOC")) -_GOAL_ID_MODE = os.environ.get("LOOPX_GOAL_ID_MODE", "fixed") -# LOOPX_UNGATED=1 —— 第三版：把前两版自己关掉/卡死的三处打开。 -# -# 前两版的实测问题（见 LOOPX-DOC-46-RESULTS.md §二之二、§quota 分析）： -# -# ① 待办规划被关掉。bootstrap 带了 `--no-onboarding-scan`，它的 help 原文是 -# "Skip the fast first-connect repository scan and **todo candidate proposal**"。 -# 于是 LoopX 自己一条候选 todo 都没提，状态文件里那些任务专属 todo 全是模型 -# 运行时自己建的。`terminal_no_followup`（待办队列空）因此成为最大拦截源之一。 -# -# ② 人工门禁在无人场景下永不放行。`--codex-app-heartbeat ask` 不预授权； -# `coordination.write_scope` 是空的，agent 没有声明过的写权限；实测 -# `quota should-run` 真实返回里出现 state=operator_gate。 -# -# ③ 死锁。goal body 写明第三次相同阻塞轮就 `update_goal status=blocked`， -# 而「Only user `/goal resume` reactivates it」——benchmark 里没有 user。 -# v2 有 21/45 个任务、113/417 个阶段（27%）以 blocked 收尾。 -# -# 修复①和②的参数已随上游删除首连 onboarding 门禁而失效：现在无论哪一版， -# bootstrap 都只登记 goal，不再写 onboarding todo，也不再要求 -# connection validation / heartbeat 选择。LOOPX_UNGATED 目前只剩写权限声明 -# 这一处差异，仍不碰 LoopX 渲染出的 goal body，保真度门禁照旧。 -_UNGATED = bool(os.environ.get("LOOPX_UNGATED")) - -# ── 三个模式 ──────────────────────────────────────────────────────────────── -# WEN_MODE 选 LoopX README 里 Codex 的哪一行 host（README.md:289-291）。 -# 定义在 wen/modes/profiles.py，这里只取参数，不复制一份枚举。 -import sys as _sys # noqa: E402 -# modes/ 可能在同级（wen 布局）或 runtime/ 子目录（发布布局）下；探测哪个含 -# modes/profiles.py 再加进 path，`from modes.profiles` 在两种布局都能解析。 -_here = Path(__file__).resolve().parent -for _cand in (_here.parent, _here.parent / "runtime", _here.parent.parent): - if (_cand / "modes" / "profiles.py").exists(): - _sys.path.insert(0, str(_cand)) - break -from modes.profiles import profile_args as _profile_args, resolve as _resolve_mode # noqa: E402 - -_MODE = _resolve_mode( - os.environ.get("WEN_MODE", "ssh-goal"), - claim_codex_app=bool(os.environ.get("WEN_CLAIM_CODEX_APP")), -) -#: 渲染时传给 loopx 的 profile 参数（具名 profile 或 -H/-O/-M 三元组） -_PROFILE_ARGS = " ".join(_profile_args(_MODE)) - -_GOAL_DOC_PATH = f"{_ROOT}/goal-doc.md" -_FIXED_GOAL_ID = "lhtb-goal" -# 与 benchmark_toolkit 的 NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS 对应。 -# 实测 install-local.sh 物化出这 6 个。 -# 技能门禁向上游常量看齐，不写死。 -# 写死过一次 6 个，而 loopx 0.5.3 物化 7 个（多 loopx-benchmark）； -# 门禁只查那 6 个，于是容器里少装一个技能完全不会被发现。 -# 从 LOOPX_SRC_DIR 读常量，读不到就退回历史的 6 个并在日志里说明。 -def _load_required_skills() -> tuple[str, ...]: - src = os.environ.get("LOOPX_SRC_DIR", "") - if src and os.path.isdir(src): - import sys as _s - if src not in _s.path: - _s.path.insert(0, src) - try: - from loopx.capabilities.benchmark_toolkit.native_codex_profile import ( - NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS as _R, - ) - return tuple(_R) - except Exception: - pass - return ( - "loopx", "loopx-doc-registry", "loopx-pr-program", - "loopx-pr-review", "loopx-project", "loopx-self-repair", - ) - - -_REQUIRED_SKILLS = _load_required_skills() - -# `render_native_codex_goal_prompt` 里的同名常量：渲染出的 body 带这个占位符， -# 必须替换成真实 registry 路径，替换后还要验证占位符确实消失。 -_GLOBAL_REGISTRY_TOKEN = "$HOME/.codex/loopx/registry.global.json" - - -class CodexLoopxAgent(CodexGoalAgent): - _loopx_ready = False - - def _bootstrap_gates(self, cwd: str) -> str: - """bootstrap 末尾那串门禁/规划相关的参数。 - - 历史记录（v1/v2 用 `--no-onboarding-scan --codex-app-heartbeat ask`， - v3 用 `--accept-onboarding-agent-todos --begin-autonomous-advance - --codex-app-heartbeat yes`）描述的是 LoopX 当时的首连门禁参数。这些参数 - 已在上游删除：bootstrap 不再写入任何首连 onboarding todo，也不再有 - connection validation / heartbeat 选择项。现在两版只剩写权限声明不同， - objective、adapter、goal-doc、goal-id 仍逐字不变。 - """ - if not _UNGATED: - return "" - # 声明写权限（原来 coordination.write_scope 是空的） - return f"--write-scope {cwd}" - - def _goal_id(self) -> str: - """本 trial 的 goal id。 - - `LOOPX_GOAL_ID_MODE=task` 时取任务名。trial 目录名形如 - `__`，logs_dir 是 `.../__/agent`。 - LoopX 会把 goal_id 当路径段用，所以只保留安全字符、并保证首字符 - 是字母或数字（`2048` 这类纯数字任务名也合法）。 - """ - if _GOAL_ID_MODE != "task": - return _FIXED_GOAL_ID - raw = self.logs_dir.parent.name.rsplit("__", 1)[0] - safe = re.sub(r"[^A-Za-z0-9._-]", "-", raw).strip("-._") - return f"{safe}-goal" if safe and safe[0].isalnum() else _FIXED_GOAL_ID - - @staticmethod - def name() -> str: - return "codex-loopx" - - # ── app-server 要跑在 LoopX profile 的环境里 ────────────────────────── - # 覆盖父类的常量，让 config.toml / auth.json 写进 profile 的 CODEX_HOME， - # 否则 codex 找不到 skills。 - @property - def _REMOTE_CODEX_HOME(self): # noqa: N802 (与上游同名) - from pathlib import PurePosixPath - return PurePosixPath(_CODEX_HOME) - - def _sh(self, cid: str, script: str, env: dict[str, str] | None = None, - timeout: int = 600, as_root: bool = False) -> subprocess.CompletedProcess: - cmd = ["docker", "exec"] - # 默认以任务用户跑，这样 LoopX 写出的 registry / runtime / 状态文件归属 - # 正确，降权运行的 codex 后续才改得动。只有需要写 /opt 的目录创建和 - # 权限移交走 as_root=True。 - if not as_root and getattr(self, "_task_user", None): - cmd += ["-u", str(self._task_user)] - for k, v in (env or {}).items(): - cmd += ["-e", f"{k}={v}"] - cmd += [cid, "sh", "-c", script] - return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) - - async def _prepare(self, environment: BaseEnvironment): - """先用 root 建好 profile 根目录并移交给任务用户，再走父类准备段。 - - 【踩过的坑】父类 `_prepare()` 第一件事就是 - `exec_as_agent('mkdir -p "$CODEX_HOME" ...')`，而我把 CODEX_HOME 挪到了 - `/opt/lxprofile/codex-home`（codex 必须从 `$CODEX_HOME/skills` 发现那 6 个 - skill，不放一起过不了保真度门禁）。`/opt` 归 root，于是在 - `sudoku-recovery`——46 个任务里唯一设 `user = "agent"` 的——直接： - - mkdir: cannot create directory '/opt/lxprofile': Permission denied - - trial 在 setup 阶段就死，一个 token 没花，却被记成 0.000，等于把基础设施 - 失败当成了任务失败（同任务基线 0.429、goal 0.571/0.707）。v1 v2 共用 - 本文件，两轮报错一字不差。 - - 为什么不把 `_ROOT` 换成 `/tmp`：那会让这一个任务的环境和另外 45 个不同， - 修公平性的补丁反而制造新的不公平。这里保持路径完全一致，只补权限。 - """ - # _drive 破自锁时要用容器 id，但它没有 environment 形参，这里存一份。 - # 之前直接写 self._environment 导致 'CodexLoopxAgent' object has no - # attribute '_environment'，三个 LoopX 臂全挂。 - self._env_ref = environment - self._task_user = getattr(environment, "default_user", None) - if self._task_user: - cid = self._container_id(environment) - r = self._sh( - cid, - f"mkdir -p {_ROOT} {_SRC} {_PY} {_NODE} && " - f"chown -R {self._task_user} {_ROOT} {_SRC} {_PY} {_NODE}", - as_root=True, timeout=180, - ) - if r.returncode: - raise RuntimeError( - f"以 root 准备 LoopX profile 目录失败: {(r.stderr or r.stdout)[:200]}" - ) - self.logger.info("非 root 任务（user=%s），profile 目录已移交", - self._task_user) - return await super()._prepare(environment) - - def _install_env(self) -> dict[str, str]: - """照抄 benchmark_toolkit `_formal_install_environment()` 的每一项。 - - 少一项就可能装成 canary（未验证源码不自动提升为默认），实测过： - 不设 LOOPX_PROMOTE_DEFAULT=1 就只有 loopx-canary、skills 也不装。 - """ - return { - "HOME": _HOME, "SHELL": "/bin/sh", "CODEX_HOME": _CODEX_HOME, - # node 必须在 PATH 上，doctor 用 shutil.which("node") 找它 - "PATH": f"{_NODE}/bin:{_BIN}:/usr/local/bin:/usr/bin:/bin", - "LOOPX_PYTHON": f"{_PY}/bin/python3", - "LOOPX_PROMOTE_DEFAULT": "1", - "LOOPX_INSTALL_CANARY": "0", - "LOOPX_BIN_DIR": _BIN, - "LOOPX_RELEASES_DIR": f"{_ROOT}/releases", - "LOOPX_RELEASE_ID": "native-goal-profile", - "LOOPX_MAN_ROOT": f"{_ROOT}/man", - "LOOPX_MAN_DIR": f"{_ROOT}/man/man1", - "LOOPX_SHELL_PROFILE": f"{_HOME}/.profile", - "LOOPX_SKILLS_DIR": f"{_CODEX_HOME}/skills", - "LOOPX_INSTALL_SLASH_COMMANDS": "0", - "LOOPX_INSTALL_OPENCODE": "0", - "LOOPX_INSTALL_CLAUDE": "0", - "LOOPX_SKILL_DEDUPE_OTHER_ROOT": "0", - } - - def _cli_env(self) -> dict[str, str]: - return {"HOME": _HOME, "CODEX_HOME": _CODEX_HOME, - "PATH": f"{_NODE}/bin:{_BIN}:/usr/local/bin:/usr/bin:/bin"} - - # ── 安装 LoopX profile ──────────────────────────────────────────────── - def _install_loopx(self, cid: str) -> None: - # wen/ 版默认落在本机布局上（原默认曾是某台特定机器的绝对路径）。 - # env.sh 会显式导出这两个变量，这里的 fallback 只是脱离 env.sh 时的兜底。 - _wen = Path(__file__).resolve().parent.parent - src = os.environ.get("LOOPX_SRC_DIR", str(_wen.parent / "loopx")) - py = os.environ.get( - "LOOPX_PORTABLE_PYTHON", - os.path.expanduser( - "~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu" - ), - ) - node = os.environ.get("LOOPX_NODE_DIR", "") - if not node or not os.path.isdir(node): - raise RuntimeError( - "LOOPX_NODE_DIR 没指到可用的 node（>=22.6.0）。0.5.3 的 " - "install-local.sh 会跑 doctor --deep，其中 " - "typescript_effect_runtime_ready 是必需项，没有 node 就 missing，" - "整个 LoopX 安装会中止。" - ) - for host_path, dest in ((src, _SRC), (py, _PY), (node, _NODE)): - if not os.path.isdir(host_path): - raise RuntimeError(f"LoopX 依赖缺失: {host_path}") - r = subprocess.run(["docker", "cp", f"{host_path}/.", f"{cid}:{dest}"], - capture_output=True, text=True, timeout=900) - if r.returncode: - raise RuntimeError(f"拷贝 {host_path} 失败: {r.stderr[:200]}") - - # `docker cp` 总是以 root 身份写入，非 root 任务上拷完必须再移交一次， - # 否则 install-local.sh（降权运行）读不到源码、也写不进 releases。 - if getattr(self, "_task_user", None): - self._sh(cid, f"chown -R {self._task_user} {_SRC} {_PY} {_NODE} {_ROOT}", - as_root=True, timeout=180) - - self._sh(cid, f"mkdir -p {_HOME} {_CODEX_HOME}/skills {_BIN} " - f"{_ROOT}/releases {_ROOT}/man {_RUNTIME}") - # 拷进来的源码里可能带着宿主机 pip 留下的 *.egg-info / *.dist-info。 - # 那是致命污染：0.5.3 的 install-local.sh 会跑一道 RC doctor 深检， - # importlib.metadata.distribution("loopx") 会解析到这份树内元数据， - # 而 editable/in-tree 构建的 egg-info **不记录 console script**，于是 - # distribution_command: {"ok": false, "error": "console_script_not_recorded"} - # loopx installer error: release candidate doctor validation failed - # 整个 LoopX 臂装不进去。宿主机上删了 pip 还会再生成，所以在容器侧清， - # 不依赖宿主状态。 - self._sh(cid, - f"find {_SRC} -maxdepth 2 -name '*.egg-info' -o -maxdepth 2 -name '*.dist-info' " - f"| xargs -r rm -rf", - timeout=120) - - r = self._sh(cid, f"bash {_SRC}/scripts/install-local.sh", - env=self._install_env(), timeout=900) - if r.returncode: - raise RuntimeError(f"LoopX 安装失败: {(r.stderr or r.stdout)[-400:]}") - - # fail-closed：CLI 与 6 个 skill 必须都在，doctor 必须 ok - chk = self._sh(cid, f"test -x {_CLI} && ls {_CODEX_HOME}/skills", - env=self._cli_env()) - missing = [s for s in _REQUIRED_SKILLS if s not in (chk.stdout or "")] - if chk.returncode or missing: - raise RuntimeError(f"LoopX profile 不完整，缺 skill: {missing}") - doc = self._sh(cid, f"{_CLI} --format json doctor --agent-type codex-app-ssh", - env=self._cli_env()) - try: - if not json.loads(doc.stdout).get("ok"): - raise ValueError - except Exception: - raise RuntimeError(f"loopx doctor 未通过: {(doc.stdout or doc.stderr)[:300]}") - self.logger.info(f"LoopX profile 就绪（{len(_REQUIRED_SKILLS)} skills + CLI + doctor ok）") - - # ── 注册目标并渲染 goal body ────────────────────────────────────────── - def _render_goal_body(self, cid: str, cwd: str, instruction: str = "") -> str: - g = (f"{_CLI} --registry {_REGISTRY} --runtime-root {_RUNTIME}") - env = self._cli_env() - gid = self._goal_id() - - # bootstrap 幂等：registry 已有该目标就跳过。 - # harbor 的 continue_until_timeout 每阶段都会调 run()，重复 bootstrap - # 会被 LoopX 拒绝（它禁止在已有状态上强制 bootstrap）。 - has = self._sh(cid, f"test -f {_REGISTRY} && grep -q {gid} {_REGISTRY}", env=env) - if has.returncode: - doc_flag = "" - if _GOAL_DOC: - # 46 个镜像里只有 6 个自带 instruction.md，所以统一由我们写进去， - # 放在 profile 目录而不是任务工作区，避免多给 verifier 一个文件。 - if not instruction.strip(): - raise RuntimeError("LOOPX_GOAL_DOC=1 但 instruction 为空") - w = subprocess.run( - ["docker", "exec", "-i", cid, "sh", "-c", - f"cat > {_GOAL_DOC_PATH}"], - input=instruction, text=True, capture_output=True, timeout=120) - if w.returncode: - raise RuntimeError(f"写 goal-doc 失败: {w.stderr[:200]}") - doc_flag = f" --goal-doc {_GOAL_DOC_PATH}" - r = self._sh(cid, - f"cd {cwd} && {g} bootstrap --project . --goal-id {gid} " - f"--objective 'Finish the task.'{doc_flag} " - f"--adapter-kind read_only_project_map_v0 " - f"--adapter-status connected-read-only " - f"{self._bootstrap_gates(cwd)}", - env=env, timeout=300) - if r.returncode: - raise RuntimeError(f"loopx bootstrap 失败: {(r.stderr or r.stdout)[-300:]}") - r = self._sh(cid, f"cd {cwd} && {g} register-agent --goal-id {gid} " - f"--agent-id {_AGENT_ID} --require-new --execute", - env=env, timeout=300) - if r.returncode: - raise RuntimeError(f"loopx register-agent 失败: {(r.stderr or r.stdout)[-300:]}") - - # ── 每阶段解锁（破死锁）──────────────────────────────────────────── - # goal body 原文里写着： - # Third identical blocked turn with no progress: call update_goal with - # status=blocked ... **Only user `/goal resume` reactivates it** - # benchmark 里没有 user，所以这是个单向阀门。实测 v2 有 21/45 个任务、 - # 27% 的阶段以 blocked 收尾。这里由 harness 扮演「每阶段来解锁的 operator」， - # 用官方 CLI 清掉 waiting_on 并把 agent 置回 active，不碰 goal body。 - if _UNGATED: - self._sh(cid, - f"cd {cwd} && {g} configure-goal --goal-id {gid} " - f"--clear-waiting-on --agent-work-mode {_AGENT_ID}=active --execute", - env=env, timeout=180) - - r = self._sh(cid, - f"cd {cwd} && {g} --format json heartbeat-prompt --thin " - f"--goal-id {gid} --agent-id {_AGENT_ID} " - f"{_PROFILE_ARGS} --cli-bin {_CLI} " - f"--available-capability shell --available-capability filesystem_write", - env=env, timeout=300) - # 下面每一条校验都照抄 render_native_codex_goal_prompt()， - # 失败码沿用它的命名，方便和 LoopX 自己的实现对照。 - try: - payload = json.loads(r.stdout) - except json.JSONDecodeError: - raise RuntimeError("goal_prompt_cli_invalid_json") - if payload.get("ok") is not True: - raise RuntimeError(f"goal_prompt_cli_not_ready: {str(payload.get('error'))[:200]}") - if _MODE.runtime_profile and payload.get("runtime_profile") != _MODE.runtime_profile: - raise RuntimeError( - f"goal_prompt_runtime_profile_mismatch: " - f"want={_MODE.runtime_profile} got={payload.get('runtime_profile')}" - ) - budget = payload.get("interface_budget") or {} - if budget.get("within_budget") is not True: - raise RuntimeError("goal_prompt_interface_budget_invalid") - body = payload.get("task_body") - if not isinstance(body, str) or not body.strip(): - raise RuntimeError("goal_prompt_task_body_missing") - if _CLI not in body: - raise RuntimeError("goal_prompt_installed_cli_not_bound") - if _GLOBAL_REGISTRY_TOKEN in body: - body = body.replace(_GLOBAL_REGISTRY_TOKEN, _REGISTRY) - if _GLOBAL_REGISTRY_TOKEN in body or _REGISTRY not in body: - raise RuntimeError("goal_prompt_runtime_registry_not_bound") - # codex 的 objective 硬上限 4000；LoopX 的 interface_budget 也按 4000 设计 - if len(body) > 4000: - raise RuntimeError(f"goal body {len(body)} 字符，超过 codex 的 4000 上限") - self._goal_body = body - self._loopx_variant = {"goal_doc": _GOAL_DOC, "goal_id_mode": _GOAL_ID_MODE, - "goal_id": gid, "ungated": _UNGATED} - self.logger.info( - "LoopX goal body 已渲染（%d 字符，goal_id=%s，goal_doc=%s，ungated=%s）", - len(body), gid, _GOAL_DOC, _UNGATED) - return body - - # ── 覆盖父类的四个 treatment 钩子 ───────────────────────────────────── - def _treatment_setup(self, cid: str, cwd: str) -> None: - # profile 装一次即可；harbor 的 continue_until_timeout 每阶段都会调 - # run()，但容器不变，所以用实例标志 + 容器内探测双重保险。 - if not self._loopx_ready: - probe = self._sh(cid, f"test -x {_CLI}") - if probe.returncode: - self._install_loopx(cid) - self._loopx_ready = True - self._render_goal_body(cid, cwd, self._pending_instruction) - - def _objective(self) -> str: - body = getattr(self, "_goal_body", "") - if not body: - raise RuntimeError("LoopX goal body 未渲染，treatment 未生效") - return body - - def _required_skill_ids(self) -> tuple: - # 交给 native_codex_goal 的 skills/list 门禁：codex 没真的发现这些 - # skill 就在 thread/start 之前失败，不花 token。 - return _REQUIRED_SKILLS - - def _extra_exec_env(self) -> dict: - # app-server 必须跑在 profile 的环境里，否则 codex 看不到 skills、 - # goal body 里指名的 loopx 也不在 PATH 上。 - return {"HOME": _HOME, "PATH": f"{_BIN}:/usr/local/bin:/usr/bin:/bin"} - - - # ── 破自锁：blocked 不等于终态 ────────────────────────────────────────── - def _restart_turn_same_thread(self, transport, config, turn): - """在**已有 thread** 上再起一轮，只发 `turn/start`。 - - 【为什么不能用 start_native_goal_turn】它内部第一步是 attach_native_goal， - 而那会重发 `initialize`。同一条 transport 二次 initialize 是协议违规， - app-server 直接拒绝： - - Trial ... failed: app_server_request_failed:initialize - - 实测代价：kubernetes-rust-rewrite/codex-cli 三次尝试全挂在这里，$0.82 白花。 - 更隐蔽的是**这条路径直到那一刻才第一次被执行到** —— 在此之前 goal 从没 - 真的进过 blocked，receipt 里的解锁数一直是 0，我据此一轮轮报告"自锁未触发、 - 兜底备而未用"。兜底其实是坏的，只是没被叫到。 - - turn_params 逐字照抄上游 start_native_goal_turn，只去掉 attach 那一步， - 保证除"不重新握手"之外语义完全一致。 - """ - from loopx.capabilities.benchmark_toolkit.native_codex_goal import ( # noqa: E402 - NativeGoalProtocolError as _E, _nested, - ) - turn_params = { - "threadId": turn.thread_id, - "input": [{"type": "text", "text": config.task_instruction}], - "cwd": config.cwd, - "approvalPolicy": config.approval_policy, - } - if config.model: - turn_params["model"] = config.model - if config.effort: - turn_params["effort"] = config.effort - if config.sandbox_policy is not None: - turn_params["sandboxPolicy"] = dict(config.sandbox_policy) - turn_result = transport.request("turn/start", turn_params) - turn.methods.append("turn/start") - rt = _nested(turn_result, "turn") - rid = str(rt.get("id") or turn_result.get("turnId") or "") - if not rid: - raise _E("turn_start_id_missing") - turn.turn_id = rid - turn.response_turn_id = rid - turn.turn_status = str(rt.get("status") or "accepted") - return turn - - def _drive(self, transport, config): - """在父类续跑循环之上，把 `blocked` 当成可恢复而不是终态。 - - 父类（codex_goal_agent._drive）的判据是 `status != "active"` 就返回—— - `blocked` 也满足。而 LoopX 的 goal body 明写：连续三轮相同阻塞就 - `update_goal status=blocked`，且**只有 user 的 `/goal resume` 能复活它**。 - benchmark 里没有 user，于是模型一 block，驱动立刻收工。 - - 实测后果：find-network-alignments/codex-cli 只跑了 7 步、131 个输出 token - 就以 post_goal_status=blocked、continuation_turn_completed_count=0 结束， - 而这个 benchmark 单次平均 27.2M token。测到的是自锁，不是 harness 能力。 - - LOOPX_UNGATED=1 时由 harness 扮演那个不存在的 operator：清掉 waiting_on、 - 把 agent 置回 active、再起一轮，直到真正终态或预算耗尽。 - 每次解锁都计数并写进 receipt，报分时要能看出用了几次。 - """ - from loopx.capabilities.benchmark_toolkit.native_codex_goal import ( # noqa: E402 - refresh_native_goal_status, start_native_goal_turn, wait_native_goal_turn, - ) - import time as _t - - if not _UNGATED: - return super()._drive(transport, config) - - cid = self._container_id(self._env_ref) - gid = self._goal_id() - env = self._cli_env() - cwd = getattr(self, "_workdir", None) or "/app" - deadline = _t.monotonic() + _GOAL_TIMEOUT_SEC - unblocks = 0 - max_unblocks = int(os.environ.get("LOOPX_MAX_UNBLOCKS", "8")) - - turn = start_native_goal_turn(transport, config) - self._turn = turn - completed_before = turn.turn_completed_count - while True: - remaining = deadline - _t.monotonic() - if remaining <= 0: - self._unblock_count = unblocks - raise NativeGoalProtocolError("goal_timeout_before_terminal") - try: - wait_native_goal_turn(transport, turn, timeout_sec=remaining, - completed_before=completed_before) - except NativeGoalProtocolError as exc: - if str(exc) == "goal_turn_timeout": - self._unblock_count = unblocks - raise NativeGoalProtocolError("goal_timeout_before_terminal") from exc - # 【踩过的坑】这条裸 raise 原来不赋值 _unblock_count，于是 receipt 里 - # 该字段是 None 而不是数字。偏偏这是**最需要证据的**出口：非超时的 - # 协议错误（实测是 8 次流层重试被 TPM 限流耗尽后抛出的），trial 会 - # 提前几十分钟死掉。mastodon-clone/ssh-goal 就这么丢了解锁计数， - # 排查时只能靠"别的臂都是 0、就它是 -"这个差异反推。 - # 每条出口都要留下计数，否则出问题的那次恰好没有证据。 - self._unblock_count = unblocks - raise - completed_before = turn.turn_completed_count - status = refresh_native_goal_status(transport, turn) - if status == "active": - continue - if status != "blocked" or unblocks >= max_unblocks: - # 【实测教训】只在循环内判 blocked 是不够的：Goal 常常在 wait 返回、 - # codex 停止续跑之后才落到 blocked，那时已经走到这个 return。 - # 实测 ssh-goal cont=3 / codex-cli cont=2 都以 blocked 收尾而 - # 解锁一次未触发。这里在返回前再兜一次。 - if status == "blocked" and unblocks < max_unblocks: - unblocks += 1 - self.logger.info("收尾时仍 blocked，第 %d 次解锁后重试", unblocks) - self._sh(cid, - f"cd {cwd} && {_CLI} configure-goal --goal-id {gid} " - f"--clear-waiting-on --agent-work-mode {_AGENT_ID}=active " - f"--execute", env=env, timeout=180) - turn = self._restart_turn_same_thread(transport, config, turn) - self._turn = turn - completed_before = turn.turn_completed_count - continue - self._unblock_count = unblocks - return turn - # 扮演 operator：清阻塞、置回 active、再起一轮 - unblocks += 1 - self.logger.info("goal 进入 blocked，第 %d 次解锁", unblocks) - self._sh(cid, - f"cd {cwd} && {_CLI} configure-goal --goal-id {gid} " - f"--clear-waiting-on --agent-work-mode {_AGENT_ID}=active --execute", - env=env, timeout=180) - turn = self._restart_turn_same_thread(transport, config, turn) - self._turn = turn - completed_before = turn.turn_completed_count +from benchmark.runtime.harbor import BenchmarkCodex + +class CodexLoopxAgent(BenchmarkCodex): + def __init__(self, *args, **kwargs): + if any(os.environ.get(key) for key in ("WEN_MODE", "WEN_CLAIM_CODEX_APP", "LOOPX_UNGATED")): + raise ValueError("Retired WEN controls: select an explicit execution_mode; see runtime/RUNTIME.md") + kwargs.setdefault("execution_mode", "loopx-goal") + super().__init__(*args, **kwargs) diff --git a/benchmark/swe-marathon/agents/codex_offline.py b/benchmark/swe-marathon/agents/codex_offline.py index bf2156a5ae..d89adc07f7 100644 --- a/benchmark/swe-marathon/agents/codex_offline.py +++ b/benchmark/swe-marathon/agents/codex_offline.py @@ -1,204 +1,4 @@ -"""离线安装的 Codex agent。 +"""Compatibility import for existing Harbor configs.""" +from benchmark.runtime.codex_offline import CodexOffline -上游 `harbor.agents.installed.codex.Codex.install()` 在容器内 `npm install -g -@openai/codex`（非 musl 环境还要先装 NVM + Node 22），这需要容器能出公网。 -LHTB 有 22 个任务 `allow_internet=false`，装不上——这正是上一轮 codex-gpt5.5 -只能跑 24 个任务的原因。 - -但 `@openai/codex` 的 npm 包里带的是一个 **static-pie musl 二进制**，不依赖 -Node、不依赖任何动态库（已在断网容器里验证 `codex --version` 可跑）。所以把 -安装改成"从宿主机拷二进制进去"，就不需要容器出网了。同一个包里还带了 ripgrep， -一并拷进去（上游 install 也会装 rg）。 - -除 install() 外一切沿用上游 Codex：同样的 `codex exec` 命令行、同样的 -config.toml / auth.json 处理、同样的轨迹解析。所以这不是另一个 harness， -只是把"怎么把 codex 放进容器"换了个不需要网的做法。 - -用法（config.yaml）： - agents: - - import_path: codex_offline:CodexOffline - model_name: openai/gpt-5.5 -需要 PYTHONPATH 指到本文件所在目录，且 CODEX_OFFLINE_DIR 指向存放 -codex / rg 两个二进制的目录（默认见下）。 -""" - -import os -from pathlib import Path - -from harbor.agents.installed.base import CliFlag -from harbor.agents.installed.codex import Codex -from harbor.environments.base import BaseEnvironment - -# wen/ 版默认指向本工作区暂存的二进制（当前 0.151.0）。原值 -# 原默认曾指向某台特定机器的绝对路径，此处改为工作区相对路径。 -_DEFAULT_OFFLINE_DIR = str(Path(__file__).resolve().parent.parent / "codex") - -# 先落到 /tmp 再 install 到 /usr/local/bin：upload_file 以 root 落盘且不保留 -# 执行位，直接传到 /usr/local/bin 会得到一个不可执行的文件。 -_STAGE_DIR = "/tmp/codex-offline" - -# 上游限流会把 agent 打死：2026-08-21 第一次跑 46 全量时，5 个跑完的 trial 里 -# 有 3 个死于 -# "stream disconnected before completion: Requests have exceeded the throughput -# limit on your Provisioned-Managed deployment" -# nbody 的 todo 只完成 1/5 项就被切断（0.676 vs Terminus 的 0.973）。 -# -# 【第一次诊断错了，留着当教训】起初以为是"Terminus 配了 num_retries=4 会重试、 -# codex 不重试"，于是把这三个键设成 4。查了才知道 codex 的默认值本来就是 -# request_max_retries=4 / stream_max_retries=5——设成 4 等于没改，stream 那个 -# 还从 5 降到了 4。网关日志也证实 codex 一直在重试：那轮 1721 次调用里 109 次 -# 是 0-token 的失败调用，最惨的一个 session 连续重试 38 次仍然没救回来。 -# -# 真正的原因是**请求速率**：codex 约 17 次/分（4 路并发），Terminus 只有 2~3 次/分。 -# 同样 4 路并发，codex 的压强是 Terminus 的 5~8 倍，顶穿了 provisioned 部署的吞吐 -# 上限。Terminus 撞不到不是因为它会重试，是因为它根本达不到那个速率。 -# -# 所以主修法是**降并发**（见 config 里的 n_concurrent_trials），这里只是把重试 -# 抬到默认值以上做兜底——短暂抖动能扛过去，持续超限还是得靠降速率。 -_RETRY_FLAGS = ( - # codex 0.151.0 起 provider 必须有非空 name，否则起手就是 - # Error loading config.toml: model_providers.harbor: provider name must not be empty - # harbor 自己生成的 provider 段没写 name（0.147 之前不校验），于是 codex 直接 - # 退出、连 session 目录都不建，trial 记成 reward=0 —— 看着像模型没做出来， - # 其实一个 token 都没花。这一条必须排在其他 harbor provider 覆盖之前。 - "-c model_providers.harbor.name=harbor" - " -c model_providers.harbor.request_max_retries=8" - " -c model_providers.harbor.stream_max_retries=8" - " -c model_providers.harbor.stream_idle_timeout_ms=300000" -) - - -class CodexOffline(Codex): - # goals 在 0.133.0 里是 stable 且**默认开启**的：模型可以调 create_goal / - # get_goal / update_goal，运行时会自动续跑（"Continue working toward the active - # thread goal."）直到目标达成或预算耗尽。默认开意味着基线轮不显式关掉的话， - # 它自己就带了 goal 模式，三轮就不是三个条件了。 - # 所以这里做成必须显式声明：基线 goals="false"，goal 轮 goals="true"。 - CLI_FLAGS = Codex.CLI_FLAGS + [ - CliFlag( - "goals", - cli="-c", - type="enum", - choices=["true", "false"], - format="-c features.goals={value}", - ), - # 模型服务端的 web_search 由**模型侧**执行，不经过容器——internal 网络、 - # 无默认路由、iptables 对它全部无效。基线轮实测：46 个任务里 6 个用过， - # 其中 4 个是断网任务，而 allow_internet=false 是任务作者设的约束。 - # sokoban 搜关卡答案 22 次、apex-law433 顺着任务反查到 HuggingFace 上的 - # 源数据集 RUC-AIBOX/Evo-Bench 并试图在里面找答案原文。 - # - # 【键名踩过坑】正确的是 `web_search`，不是 `web_search_mode`。 - # 后者连同 disabled_tools / tools.web_search / --disable web_search_request - # 一共四种写法**codex 都接受但都不生效**——实测同一个必须联网才能答的 - # 问题，不加开关触发 86 次，四个候选仍触发 46/72/117/52 次。 - # `web_search="disabled"` 实测触发 0 次，且 agent 会明说 - # "this session does not have usable web access" 后退回 shell—— - # **它想用而用不了**，这比计数为 0 更能证明工具真的不在了。 - CliFlag( - "web_search", - cli="-c", - type="enum", - choices=["disabled", "cached", "live"], - format='-c web_search="{value}"', - ), - ] - - @staticmethod - def name() -> str: - return "codex-offline" - - def version(self) -> str | None: - return self._version or "offline" - - def get_version_command(self) -> str | None: - # 上游那条命令会先 source nvm；离线安装没有 nvm，直接问二进制。 - return "/usr/local/bin/codex --version" - - def build_cli_flags(self) -> str: - flags = super().build_cli_flags() - # 默认关掉模型服务端的 web_search，**对全部 46 个任务一律关闭**。 - # - # 理由是与 Terminus 的信息通道对齐：Terminus 结构上没有这个工具 - # （46 轮 × 每轮几十个 debug.json 里 `tools` 字段一次都没出现）， - # 所以留着它就等于给 codex 一条对方没有的信息通道。 - # - # 代价要认：对 24 个 allow_internet=true 的任务，搜索本是正当能力， - # 关掉等于让 codex 减配上场——实测 spice-ephemeris 因此从 0.939 掉到 0.030。 - # 所以这样测出来的是「**限定在 Terminus 同等信息通道下**的对比」， - # 而不是「codex 开箱能力」的对比。报告里必须写明这一点。 - # - # 键名是 `web_search` 不是 `web_search_mode`——后者连同 disabled_tools / - # tools.web_search / --disable web_search_request 四种写法 codex 都接受 - # 但都不生效（实测触发 46/72/117/52 次）。只有 web_search="disabled" - # 真正关掉：触发 0 次，且 agent 会明说没有可用的 web access 后退回 shell。 - if "web_search=" not in flags: - flags = f'{flags} -c web_search="disabled"'.strip() - return f"{flags} {_RETRY_FLAGS}" if flags else _RETRY_FLAGS - - async def install(self, environment: BaseEnvironment) -> None: - offline_dir = Path(os.environ.get("CODEX_OFFLINE_DIR", _DEFAULT_OFFLINE_DIR)) - # wen/codex 顶层的 codex / rg / codex-code-mode-host 都是指向 bin/ 与 - # codex-path/ 的软链（见 stage_codex_offline.sh）。upload_file 不跟随软链， - # 直传会得到 "install: cannot stat ... No such file or directory"， - # 所以这里一律 resolve 成真实文件再传。 - codex_bin = (offline_dir / "codex").resolve() - rg_bin = (offline_dir / "rg").resolve() - # unified_exec 的 sidecar。**少了它容器里每一次工具调用都失败**， - # 而且模型连"把 goal 标成 blocked"都做不到（那也是工具调用）， - # 结果是跑满预算、零产物、不报错。宿主机上实测空转 47 轮才发现。 - sidecar_bin = (offline_dir / "codex-code-mode-host").resolve() - # 沙箱助手。缺了它 app-server 每次起都报 - # "Codex could not find bubblewrap on PATH ... will use the bundled bubblewrap" - # 并计入 error_event_count。和当初漏 code-mode sidecar 是同一类错： - # vendor 树里有，但上传清单没带上。 - bwrap_bin = (offline_dir / "codex-resources" / "bwrap") - if not codex_bin.is_file(): - raise FileNotFoundError( - f"离线 codex 二进制不存在: {codex_bin}。" - " 用 stage_codex_offline.sh 从宿主机的 @openai/codex 包里取出来。" - ) - if not sidecar_bin.is_file(): - raise FileNotFoundError( - f"codex-code-mode-host 不存在: {sidecar_bin}。" - " 重跑 stage_codex_offline.sh —— 旧版脚本只抠 codex 和 rg，" - " 缺 sidecar 会让容器里的工具面静默全废。" - ) - - await self.exec_as_root(environment, command=f"mkdir -p {_STAGE_DIR}") - - await environment.upload_file(codex_bin, f"{_STAGE_DIR}/codex") - await environment.upload_file(sidecar_bin, f"{_STAGE_DIR}/codex-code-mode-host") - if rg_bin.is_file(): - await environment.upload_file(rg_bin, f"{_STAGE_DIR}/rg") - if bwrap_bin.is_file(): - await environment.upload_file(bwrap_bin, f"{_STAGE_DIR}/bwrap") - - # install 而不是 mv：一步搞定权限位，且目标已存在时直接覆盖。 - # 顺便把版本和二进制指纹落进 agent 产物目录：上游 codex.py:370 从事件流里 - # 读 cli_version，但 0.114.0 的 JSON 事件不吐这个字段，harbor 只能记 - # "unknown"，事后没法从产物反查跑的是哪个版本。 - await self.exec_as_root( - environment, - command=( - "set -eu; " - f"install -m 0755 {_STAGE_DIR}/codex /usr/local/bin/codex; " - # sidecar 必须和 codex 同目录：codex 按相对自身的位置找它 - f"install -m 0755 {_STAGE_DIR}/codex-code-mode-host " - " /usr/local/bin/codex-code-mode-host; " - f"if [ -f {_STAGE_DIR}/rg ]; then " - f" install -m 0755 {_STAGE_DIR}/rg /usr/local/bin/rg; " - "fi; " - f"if [ -f {_STAGE_DIR}/bwrap ]; then " - f" install -m 0755 {_STAGE_DIR}/bwrap /usr/local/bin/bwrap; " - "fi; " - f"rm -rf {_STAGE_DIR}; " - "mkdir -p /logs/agent; " - "{ /usr/local/bin/codex --version; " - " md5sum /usr/local/bin/codex /usr/local/bin/codex-code-mode-host; " - "} > /logs/agent/codex_version.txt 2>&1; " - "cat /logs/agent/codex_version.txt" - ), - ) - - self.logger.info(f"codex 离线安装完成（来源 {offline_dir}，含 code-mode sidecar）") +__all__ = ["CodexOffline"] diff --git a/benchmark/swe-marathon/agents/native_codex_goal.py b/benchmark/swe-marathon/agents/native_codex_goal.py index 71108420d2..55d53af407 100644 --- a/benchmark/swe-marathon/agents/native_codex_goal.py +++ b/benchmark/swe-marathon/agents/native_codex_goal.py @@ -1,740 +1,2 @@ -"""Reusable Codex app-server Goal runtime for benchmark adapters. - -The benchmark harness retains ownership of isolation, environment bridging, -timeouts, and scoring. This module owns the native Goal JSON-RPC transaction, -stdio process transport, event correlation, and public-safe receipts so runner -implementations do not carry a second copy of that state machine. -""" - -from __future__ import annotations - -import json -import queue -import re -import subprocess -import threading -import time -from collections import Counter, deque -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from hashlib import sha256 -from typing import Any, Protocol, Self, TextIO - - -class NativeGoalProtocolError(RuntimeError): - """The app-server exchange did not prove the required Goal transaction.""" - - -class NativeGoalTransport(Protocol): - def request(self, method: str, params: Mapping[str, Any]) -> Mapping[str, Any]: ... - - def notify(self, method: str, params: Mapping[str, Any]) -> None: ... - - -class NativeGoalEventTransport(NativeGoalTransport, Protocol): - def next_event(self, *, timeout_sec: float) -> Mapping[str, Any] | None: ... - - -def _digest(value: str) -> str: - return sha256(value.encode("utf-8")).hexdigest() - - -def _nested(value: Mapping[str, Any], key: str) -> Mapping[str, Any]: - nested = value.get(key) - return nested if isinstance(nested, Mapping) else {} - - -def _error_signature(params: Mapping[str, Any]) -> str: - error = params.get("error") - typed_parts: list[str] = [] - for container in (params, error): - if not isinstance(container, Mapping): - continue - for key in ("code", "type", "kind"): - value = container.get(key) - if isinstance(value, (str, int)): - normalized = re.sub(r"[^A-Za-z0-9_.:-]", "_", str(value))[:64] - typed_parts.append(f"{key}={normalized}") - canonical = json.dumps(params, sort_keys=True, default=str) - digest = sha256(canonical.encode("utf-8")).hexdigest()[:12] - typed = ",".join(dict.fromkeys(typed_parts)) - return f"{typed},sha256={digest}" if typed else f"sha256={digest}" - - -@dataclass(frozen=True) -class NativeGoalConfig: - cwd: str - objective: str - task_instruction: str - model: str | None = None - effort: str | None = None - token_budget: int | None = None - approval_policy: str = "never" - sandbox: str = "workspace-write" - sandbox_policy: Mapping[str, Any] | None = None - required_skill_ids: tuple[str, ...] = () - - def validate(self) -> None: - if not self.cwd.strip(): - raise ValueError("cwd must be non-empty") - if not self.objective.strip(): - raise ValueError("objective must be non-empty") - if not self.task_instruction.strip(): - raise ValueError("task_instruction must be non-empty") - if self.token_budget is not None and ( - isinstance(self.token_budget, bool) - or not isinstance(self.token_budget, int) - or self.token_budget <= 0 - ): - raise ValueError("token_budget must be a positive integer when provided") - for skill_id in self.required_skill_ids: - if not isinstance(skill_id, str) or not re.fullmatch( - r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", skill_id - ): - raise ValueError("required_skill_ids must contain safe skill ids") - - -@dataclass -class NativeGoalTurn: - thread_id: str - turn_id: str - response_turn_id: str - goal_status: str - objective_sha256: str - objective_chars: int - task_instruction_sha256: str - task_instruction_chars: int - token_budget_present: bool - methods: list[str] = field(default_factory=list) - notifications: list[str] = field(default_factory=list) - notification_counts: dict[str, int] = field(default_factory=dict) - error_signatures: list[str] = field(default_factory=list) - event_turn_id_observed: bool = False - terminal_event_observed: bool = False - turn_status: str = "not_started" - post_goal_status: str = "" - item_event_count: int = 0 - error_event_count: int = 0 - turn_started_count: int = 0 - turn_completed_count: int = 0 - goal_status_poll_count: int = 0 - required_skill_ids: tuple[str, ...] = () - discovered_required_skill_ids: tuple[str, ...] = () - skill_catalog_count: int = 0 - skill_error_count: int = 0 - - -def _read_required_skills( - transport: NativeGoalTransport, - *, - cwd: str, - required_skill_ids: tuple[str, ...], -) -> tuple[tuple[str, ...], int, int]: - """Prove required skills through the app-server discovery surface.""" - - required = tuple(dict.fromkeys(required_skill_ids)) - if not required: - return (), 0, 0 - result = transport.request( - "skills/list", - {"cwds": [cwd], "forceReload": True}, - ) - data = result.get("data") - if not isinstance(data, list) or not data: - raise NativeGoalProtocolError("skills_list_data_missing") - matching = [ - row - for row in data - if isinstance(row, Mapping) and str(row.get("cwd") or "") == cwd - ] - if len(matching) != 1: - raise NativeGoalProtocolError("skills_list_cwd_mismatch") - row = matching[0] - errors = row.get("errors") - if not isinstance(errors, list): - raise NativeGoalProtocolError("skills_list_errors_missing") - if errors: - raise NativeGoalProtocolError(f"skills_list_errors:{len(errors)}") - skills = row.get("skills") - if not isinstance(skills, list): - raise NativeGoalProtocolError("skills_list_catalog_missing") - names = { - str(skill.get("name") or "") - for skill in skills - if isinstance(skill, Mapping) and skill.get("enabled") is not False - } - missing = [skill_id for skill_id in required if skill_id not in names] - if missing: - raise NativeGoalProtocolError( - "required_skills_missing:" + ",".join(sorted(missing)) - ) - return required, len(skills), 0 - - -def attach_native_goal( - transport: NativeGoalTransport, - config: NativeGoalConfig, -) -> NativeGoalTurn: - """Initialize one app-server thread and attach an active native Goal.""" - - config.validate() - methods: list[str] = [] - - transport.request( - "initialize", - { - "clientInfo": { - "name": "loopx_benchmark_toolkit", - "title": "LoopX Benchmark Toolkit", - "version": "0.1.0", - }, - "capabilities": {"experimentalApi": True}, - }, - ) - methods.append("initialize") - transport.notify("initialized", {}) - methods.append("initialized") - - discovered_skills, skill_catalog_count, skill_error_count = _read_required_skills( - transport, - cwd=config.cwd, - required_skill_ids=config.required_skill_ids, - ) - if config.required_skill_ids: - methods.append("skills/list") - - thread_params: dict[str, Any] = { - "cwd": config.cwd, - "sandbox": config.sandbox, - "approvalPolicy": config.approval_policy, - } - if config.model: - thread_params["model"] = config.model - thread_result = transport.request("thread/start", thread_params) - methods.append("thread/start") - thread = _nested(thread_result, "thread") - thread_id = str(thread.get("id") or thread_result.get("threadId") or "") - if not thread_id: - raise NativeGoalProtocolError("thread_start_id_missing") - - goal_set: dict[str, Any] = { - "threadId": thread_id, - "objective": config.objective, - "status": "active", - } - if config.token_budget is not None: - goal_set["tokenBudget"] = config.token_budget - transport.request("thread/goal/set", goal_set) - methods.append("thread/goal/set") - - goal_result = transport.request("thread/goal/get", {"threadId": thread_id}) - methods.append("thread/goal/get") - goal = _nested(goal_result, "goal") - if str(goal.get("status") or "") != "active": - raise NativeGoalProtocolError("goal_not_active") - if str(goal.get("threadId") or "") != thread_id: - raise NativeGoalProtocolError("goal_thread_mismatch") - if str(goal.get("objective") or "") != config.objective: - raise NativeGoalProtocolError("goal_objective_mismatch") - - return NativeGoalTurn( - thread_id=thread_id, - turn_id="", - response_turn_id="", - goal_status="active", - objective_sha256=_digest(config.objective), - objective_chars=len(config.objective), - task_instruction_sha256=_digest(config.task_instruction), - task_instruction_chars=len(config.task_instruction), - token_budget_present=config.token_budget is not None, - methods=methods, - required_skill_ids=tuple(dict.fromkeys(config.required_skill_ids)), - discovered_required_skill_ids=discovered_skills, - skill_catalog_count=skill_catalog_count, - skill_error_count=skill_error_count, - ) - - -def start_native_goal_turn( - transport: NativeGoalTransport, - config: NativeGoalConfig, -) -> NativeGoalTurn: - """Attach an active Goal to a new thread and start one task turn.""" - - turn = attach_native_goal(transport, config) - turn_params: dict[str, Any] = { - "threadId": turn.thread_id, - "input": [{"type": "text", "text": config.task_instruction}], - "cwd": config.cwd, - "approvalPolicy": config.approval_policy, - } - if config.model: - turn_params["model"] = config.model - if config.effort: - turn_params["effort"] = config.effort - if config.sandbox_policy is not None: - turn_params["sandboxPolicy"] = dict(config.sandbox_policy) - turn_result = transport.request("turn/start", turn_params) - turn.methods.append("turn/start") - response_turn = _nested(turn_result, "turn") - response_turn_id = str(response_turn.get("id") or turn_result.get("turnId") or "") - if not response_turn_id: - raise NativeGoalProtocolError("turn_start_id_missing") - turn.turn_id = response_turn_id - turn.response_turn_id = response_turn_id - turn.turn_status = str(response_turn.get("status") or "accepted") - return turn - - -def observe_native_goal_event( - turn: NativeGoalTurn, - event: Mapping[str, Any], -) -> bool: - """Apply one app-server notification and return terminal observation state.""" - - method = str(event.get("method") or "") - params = _nested(event, "params") - if not method: - event_type = str(event.get("type") or "") - payload = _nested(event, "payload") - payload_type = str(payload.get("type") or "") - method = ( - f"{event_type}:{payload_type}" - if event_type and payload_type - else event_type - ) - params = payload - if not method: - return turn.terminal_event_observed - - turn.notifications.append(method) - turn.notification_counts[method] = turn.notification_counts.get(method, 0) + 1 - event_thread_id = str(params.get("threadId") or "") - if event_thread_id and event_thread_id != turn.thread_id: - return turn.terminal_event_observed - event_turn = _nested(params, "turn") - event_turn_id = str(event_turn.get("id") or params.get("turnId") or "") - - if method == "turn/started" and event_turn_id: - turn.turn_id = event_turn_id - turn.event_turn_id_observed = True - turn.turn_status = str(event_turn.get("status") or "inProgress") - turn.turn_started_count += 1 - return False - if event_turn_id and event_turn_id != turn.turn_id: - return turn.terminal_event_observed - if method.startswith(("item/", "response_item:")): - turn.item_event_count += 1 - if method == "error": - turn.error_event_count += 1 - turn.turn_status = "error" - if len(turn.error_signatures) < 3: - turn.error_signatures.append(_error_signature(params)) - if method == "turn/completed" or method in { - "event_msg:task_complete", - "event_msg:task_completed", - "event_msg:turn_completed", - }: - turn.terminal_event_observed = True - turn.turn_status = str(event_turn.get("status") or "completed") - turn.turn_completed_count += 1 - return turn.terminal_event_observed - - -def wait_native_goal_turn( - transport: NativeGoalEventTransport, - turn: NativeGoalTurn, - *, - timeout_sec: float, - completed_before: int | None = None, -) -> NativeGoalTurn: - """Drain events until one more correlated turn reaches a terminal event. - - ``completed_before`` lets a native Goal runtime wait for automatic - continuation turns without starting another model turn itself. Omitting it - preserves the single-turn behavior for existing callers. - """ - - if timeout_sec <= 0: - raise ValueError("timeout_sec must be positive") - deadline = time.monotonic() + timeout_sec - if completed_before is None: - if turn.terminal_event_observed: - return turn - completed_before = turn.turn_completed_count - while turn.turn_completed_count <= completed_before: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise NativeGoalProtocolError("goal_turn_timeout") - event = transport.next_event(timeout_sec=min(0.25, remaining)) - if event is not None: - observe_native_goal_event(turn, event) - return turn - - -def refresh_native_goal_status( - transport: NativeGoalTransport, - turn: NativeGoalTurn, -) -> str: - """Read the Goal status after a turn and bind it to the same thread.""" - - result = transport.request("thread/goal/get", {"threadId": turn.thread_id}) - turn.methods.append("thread/goal/get") - goal = _nested(result, "goal") - if str(goal.get("threadId") or "") != turn.thread_id: - raise NativeGoalProtocolError("post_goal_thread_mismatch") - status = str(goal.get("status") or "") - if not status: - raise NativeGoalProtocolError("post_goal_status_missing") - turn.post_goal_status = status - turn.goal_status_poll_count += 1 - return status - - -def run_native_goal_turn( - transport: NativeGoalEventTransport, - config: NativeGoalConfig, - *, - timeout_sec: float, -) -> NativeGoalTurn: - """Execute the complete native Goal transaction over an admitted transport.""" - - turn = start_native_goal_turn(transport, config) - wait_native_goal_turn(transport, turn, timeout_sec=timeout_sec) - refresh_native_goal_status(transport, turn) - return turn - - -def run_native_goal_until_terminal( - transport: NativeGoalEventTransport, - config: NativeGoalConfig, - *, - timeout_sec: float, -) -> NativeGoalTurn: - """Run one native Goal until its status leaves ``active``. - - Codex may schedule continuation turns while a Goal remains active. The - caller starts exactly one task turn, then keeps draining those correlated - continuation events and reading the Goal status under one total timeout. - """ - - if timeout_sec <= 0: - raise ValueError("timeout_sec must be positive") - turn = start_native_goal_turn(transport, config) - deadline = time.monotonic() + timeout_sec - completed_before = turn.turn_completed_count - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise NativeGoalProtocolError("goal_timeout_before_terminal") - try: - wait_native_goal_turn( - transport, - turn, - timeout_sec=remaining, - completed_before=completed_before, - ) - except NativeGoalProtocolError as exc: - if str(exc) == "goal_turn_timeout": - raise NativeGoalProtocolError("goal_timeout_before_terminal") from exc - raise - completed_before = turn.turn_completed_count - if refresh_native_goal_status(transport, turn) != "active": - return turn - - -_StreamItem = Mapping[str, Any] | Exception - - -def _read_stream(stream: TextIO, messages: queue.Queue[_StreamItem]) -> None: - try: - for line in stream: - try: - message = json.loads(line) - except json.JSONDecodeError: - messages.put(NativeGoalProtocolError("app_server_frame_invalid_json")) - continue - if not isinstance(message, Mapping): - messages.put(NativeGoalProtocolError("app_server_frame_not_object")) - continue - messages.put(message) - finally: - messages.put(EOFError("app_server_stream_closed")) - - -class StdioNativeGoalTransport: - """Line-delimited JSON-RPC transport backed by a real app-server process.""" - - def __init__( - self, - process: subprocess.Popen[str], - *, - response_timeout_sec: float = 30, - ) -> None: - if process.stdin is None or process.stdout is None: - raise ValueError("process must expose text stdin and stdout pipes") - if response_timeout_sec <= 0: - raise ValueError("response_timeout_sec must be positive") - self.process = process - self.response_timeout_sec = float(response_timeout_sec) - self._messages: queue.Queue[_StreamItem] = queue.Queue() - self._pending_events: deque[Mapping[str, Any]] = deque() - self._next_request_id = 1 - self._reader = threading.Thread( - target=_read_stream, - args=(process.stdout, self._messages), - daemon=True, - ) - self._reader.start() - - @classmethod - def spawn( - cls, - command: Sequence[str], - *, - cwd: str, - env: Mapping[str, str] | None = None, - response_timeout_sec: float = 30, - stderr: int | TextIO | None = subprocess.DEVNULL, - ) -> StdioNativeGoalTransport: - if not command: - raise ValueError("command must be non-empty") - process = subprocess.Popen( - list(command), - cwd=cwd, - env=dict(env) if env is not None else None, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=stderr, - text=True, - encoding="utf-8", - bufsize=1, - ) - return cls(process, response_timeout_sec=response_timeout_sec) - - def __enter__(self) -> Self: - return self - - def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: - self.close() - - def _send(self, message: Mapping[str, Any]) -> None: - if self.process.stdin is None or self.process.poll() is not None: - raise NativeGoalProtocolError("app_server_stdin_closed") - try: - self.process.stdin.write(json.dumps(message) + "\n") - self.process.stdin.flush() - except (BrokenPipeError, OSError) as exc: - raise NativeGoalProtocolError("app_server_write_failed") from exc - - def _next_message(self, *, timeout_sec: float) -> Mapping[str, Any] | None: - try: - message = self._messages.get(timeout=max(0.0, timeout_sec)) - except queue.Empty: - return None - if isinstance(message, Exception): - raise NativeGoalProtocolError(str(message)) from message - return message - - def request(self, method: str, params: Mapping[str, Any]) -> Mapping[str, Any]: - request_id = self._next_request_id - self._next_request_id += 1 - self._send({"id": request_id, "method": method, "params": dict(params)}) - deadline = time.monotonic() + self.response_timeout_sec - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise NativeGoalProtocolError(f"app_server_response_timeout:{method}") - message = self._next_message(timeout_sec=remaining) - if message is None: - if self.process.poll() is not None: - raise NativeGoalProtocolError("app_server_exited_before_response") - continue - if message.get("id") == request_id: - if message.get("error") is not None: - raise NativeGoalProtocolError(f"app_server_request_failed:{method}") - result = message.get("result") - return result if isinstance(result, Mapping) else {} - if message.get("method"): - self._pending_events.append(message) - continue - raise NativeGoalProtocolError("app_server_unexpected_response") - - def notify(self, method: str, params: Mapping[str, Any]) -> None: - self._send({"method": method, "params": dict(params)}) - - def next_event(self, *, timeout_sec: float) -> Mapping[str, Any] | None: - if self._pending_events: - return self._pending_events.popleft() - message = self._next_message(timeout_sec=timeout_sec) - if message is None: - if self.process.poll() is not None: - raise NativeGoalProtocolError("app_server_exited_before_terminal_event") - return None - if message.get("method"): - return message - raise NativeGoalProtocolError("app_server_unexpected_response") - - def close(self) -> None: - if self.process.stdin is not None: - try: - self.process.stdin.close() - except OSError: - pass - if self.process.poll() is None: - self.process.terminate() - try: - self.process.wait(timeout=2) - except subprocess.TimeoutExpired: - self.process.kill() - self.process.wait(timeout=2) - - -def _default_app_server_command(codex_bin: str) -> list[str]: - return [ - codex_bin, - "app-server", - "--listen", - "stdio://", - "--enable", - "goals", - ] - - -def probe_native_goal_process( - config: NativeGoalConfig, - *, - codex_bin: str = "codex", - process_command: Sequence[str] | None = None, - process_env: Mapping[str, str] | None = None, - process_cwd: str | None = None, - response_timeout_sec: float = 30, -) -> NativeGoalTurn: - """Exercise a real app-server through Goal attachment without starting a model turn.""" - - command = process_command or _default_app_server_command(codex_bin) - with StdioNativeGoalTransport.spawn( - command, - cwd=process_cwd or config.cwd, - env=process_env, - response_timeout_sec=response_timeout_sec, - ) as transport: - return attach_native_goal(transport, config) - - -def run_native_goal_process( - config: NativeGoalConfig, - *, - codex_bin: str = "codex", - process_command: Sequence[str] | None = None, - process_env: Mapping[str, str] | None = None, - process_cwd: str | None = None, - response_timeout_sec: float = 30, - goal_timeout_sec: float = 21_600, -) -> NativeGoalTurn: - """Spawn a real app-server and execute one complete native Goal turn.""" - - command = process_command or _default_app_server_command(codex_bin) - with StdioNativeGoalTransport.spawn( - command, - cwd=process_cwd or config.cwd, - env=process_env, - response_timeout_sec=response_timeout_sec, - ) as transport: - return run_native_goal_turn(transport, config, timeout_sec=goal_timeout_sec) - - -def run_native_goal_process_until_terminal( - config: NativeGoalConfig, - *, - codex_bin: str = "codex", - process_command: Sequence[str] | None = None, - process_env: Mapping[str, str] | None = None, - process_cwd: str | None = None, - response_timeout_sec: float = 30, - goal_timeout_sec: float = 21_600, -) -> NativeGoalTurn: - """Spawn app-server and keep the native Goal alive through continuations.""" - - command = process_command or _default_app_server_command(codex_bin) - with StdioNativeGoalTransport.spawn( - command, - cwd=process_cwd or config.cwd, - env=process_env, - response_timeout_sec=response_timeout_sec, - ) as transport: - return run_native_goal_until_terminal( - transport, - config, - timeout_sec=goal_timeout_sec, - ) - - -def compact_native_goal_receipt(turn: NativeGoalTurn) -> dict[str, Any]: - """Return public-safe transaction evidence without task or response content.""" - - counts = turn.notification_counts or dict(Counter(turn.notifications)) - return { - "schema_version": "native_codex_goal_turn_receipt_v0", - "thread_id_present": bool(turn.thread_id), - "turn_id_present": bool(turn.turn_id), - "response_turn_id_present": bool(turn.response_turn_id), - "event_turn_id_observed": turn.event_turn_id_observed, - "goal_status": turn.goal_status, - "post_goal_status": turn.post_goal_status or None, - "token_budget_present": turn.token_budget_present, - "objective_sha256": turn.objective_sha256, - "objective_chars": turn.objective_chars, - "task_instruction_sha256": turn.task_instruction_sha256, - "task_instruction_chars": turn.task_instruction_chars, - "methods": list(turn.methods), - "notifications": sorted(set(turn.notifications)), - "notification_counts": dict(sorted(counts.items())), - "turn_status": turn.turn_status, - "terminal_event_observed": turn.terminal_event_observed, - "item_event_count": turn.item_event_count, - "error_event_count": turn.error_event_count, - "turn_started_count": turn.turn_started_count, - "turn_completed_count": turn.turn_completed_count, - "goal_continuation_turn_completed_count": max(0, turn.turn_completed_count - 1), - "goal_status_poll_count": turn.goal_status_poll_count, - "required_skill_ids": list(turn.required_skill_ids), - "discovered_required_skill_ids": list(turn.discovered_required_skill_ids), - "skill_catalog_count": turn.skill_catalog_count, - "skill_error_count": turn.skill_error_count, - "required_skills_discovered": ( - turn.discovered_required_skill_ids == turn.required_skill_ids - if turn.required_skill_ids - else None - ), - "error_signatures": list(turn.error_signatures), - "public_boundary": { - "raw_objective_recorded": False, - "raw_task_instruction_recorded": False, - "raw_assistant_message_recorded": False, - "raw_tool_events_recorded": False, - "credentials_recorded": False, - "local_paths_recorded": False, - }, - } - - -__all__ = [ - "NativeGoalConfig", - "NativeGoalEventTransport", - "NativeGoalProtocolError", - "NativeGoalTransport", - "NativeGoalTurn", - "StdioNativeGoalTransport", - "attach_native_goal", - "compact_native_goal_receipt", - "observe_native_goal_event", - "probe_native_goal_process", - "refresh_native_goal_status", - "run_native_goal_process", - "run_native_goal_process_until_terminal", - "run_native_goal_turn", - "run_native_goal_until_terminal", - "start_native_goal_turn", - "wait_native_goal_turn", -] +"""Compatibility import; all callers share the installed runtime and exception types.""" +from loopx.capabilities.benchmark_toolkit.native_codex_goal import * # noqa: F403 diff --git a/benchmark/swe-marathon/configs/shared-heartbeat.yaml b/benchmark/swe-marathon/configs/shared-heartbeat.yaml new file mode 100644 index 0000000000..da9300877f --- /dev/null +++ b/benchmark/swe-marathon/configs/shared-heartbeat.yaml @@ -0,0 +1,13 @@ +# Merge this agent section into the native SWE-Marathon job configuration. +# Keep native datasets/tasks, environment, feedback and verifier settings. +agents: + - import_path: benchmark.runtime.harbor:BenchmarkCodex + model_name: openai/gpt-5.6-sol + override_timeout_sec: 5400 + kwargs: + execution_mode: heartbeat + task_entry: seeded-todo # Use loopx-planned for the product planning checkpoint. + iteration_context: fresh + reasoning_effort: high + turn_timeout_sec: 4700 + scheduler_timeout_sec: 5080 diff --git a/benchmark/swe-marathon/runtime/RUNTIME.md b/benchmark/swe-marathon/runtime/RUNTIME.md index 1a34d2a0a0..5f8278cf80 100644 --- a/benchmark/swe-marathon/runtime/RUNTIME.md +++ b/benchmark/swe-marathon/runtime/RUNTIME.md @@ -1,38 +1,20 @@ -# runtime：模式框架 + automation 驱动 - -codex×LoopX 对照所用的运行时。`heartbeat` 模式的续跑由**外部 driver**（`turn/loopx_turn_runner.py`） -拥有：每轮发 `--turn-instance-id` + 过 `quota should-run` 闸门，回合边界由 driver 保证，因此 -无人自动化下最稳——不依赖人值守（codex-cli TUI）或 Codex 自身的 visible-Goal 循环（ssh-goal）。 - -## 结构 - -``` -runtime/ - modes/ - profiles.py # 声明式 Mode 表：ssh-goal / codex-cli / heartbeat - run_mode.py # CLI 入口：选模式、装 profile、跑 session - session.py # session 生命周期 - codex_host.py # host 接线 - profile_install.py # 把 loopx runtime profile / skills 装进 CODEX_HOME - turn/ - loopx_turn_runner.py # automation 驱动：每轮 turn-instance + quota 闸门 + turn 超时 + HEAD-moved 进度检查 - loopx_native_codex.py# 原生 codex turn 驱动 - goal_codex.py # visible-Goal turn 驱动（ssh-goal / codex-cli） - codex_nosandbox_wrapper.py -``` - -## 三模式（见 modes/profiles.py） - -| 模式 | runtime_profile | 续跑归属 | 备注 | -|---|---|---|---| -| ssh-goal | codex_app_ssh_goal | Codex（visible Goal） | guard 带 `--begin-turn`，原生无人值守路径 | -| codex-cli | codex_cli | Codex（visible Goal） | guard **不带** `--begin-turn`（人值守 TUI 设计） | -| heartbeat | generic_cli | **driver**（本运行时） | driver 拥有唤醒，无人自动化下最鲁棒 | - -## 依赖 - -- `loopx.capabilities.benchmark_toolkit`（native_codex_goal / native_codex_profile）。 -- 试验/agent 框架（harbor）提供环境与已安装的 Codex agent。 -- 环境变量旋钮，如 `MR_LOOPX_TURN_TIMEOUT`（每轮超时秒，默认 1200）。 - -未内嵌任何内部网络拓扑或凭证；环境相关接线（网关、代理）由环境变量在外部提供。 +# SWE-Marathon execution + +Use the [shared Codex runtime](../../runtime/RUNTIME.md), retaining SWE-Marathon's +native dataset, task environment, phase/feedback rules and scoring. The entry +`benchmark.runtime.harbor:BenchmarkCodex` is also used by LHTB. A native job +agent fragment is available in `../configs/shared-heartbeat.yaml`. + +The previous modes/ and turn/ implementations are retired. They duplicated +installation and continuation, and the old Turn validator accepted empty commits +or subsequent no-op iterations after the first changed HEAD. Public Turn CLI +execution and caller-provided independent validation replace that behavior. + +Named agent imports remain thin compatibility entries. codex_loopx_agent now +selects loopx-goal explicitly. Retired WEN/assisted controls fail with migration +guidance; select execution_mode and iteration_context for new studies. + +Prior code is preserved in Git at 8330a974cc2631ffd006d1fb7bd1627d2d690e85. +Historical results and withdrawal notices are unchanged and do not describe the +new runtime. Consumers of the former runtime directories, including pending TB4 +work, must migrate to the shared entry before claiming compatibility. diff --git a/benchmark/swe-marathon/runtime/modes/__init__.py b/benchmark/swe-marathon/runtime/modes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/benchmark/swe-marathon/runtime/modes/codex_host.py b/benchmark/swe-marathon/runtime/modes/codex_host.py deleted file mode 100644 index 466e3f68f9..0000000000 --- a/benchmark/swe-marathon/runtime/modes/codex_host.py +++ /dev/null @@ -1,188 +0,0 @@ -"""用 codex app-server 承载一轮 Goal。 - -三种模式共用这一个 host：LoopX 渲染出的 task_body 作为 Goal objective 送进 -app-server 的 Goal 事务，Codex 执行。模式差别在于**谁拥有续跑**： - - continuation_owner="codex" —— visible Goal，起首轮后 Codex 自己续跑到终态， - 驱动只观察（run_until_terminal）。 - continuation_owner="driver" —— 心跳，每次唤醒是全新一轮，驱动起完这轮就收， - 下次 tick 用新的 body 再起（run_single_turn）。 - -Goal 状态机本身不在这里实现：直接用 benchmark_toolkit 里那份安装态运行时， -上游 benchmark/deepswe/README.md 明确要求适配器 import 它而不是抄第二份。 -""" - -from __future__ import annotations - -import os -import time -from dataclasses import dataclass -from typing import Any - -from loopx.capabilities.benchmark_toolkit.native_codex_goal import ( - NativeGoalConfig, - NativeGoalDeadlineExceeded, - NativeGoalProtocolError, - StdioNativeGoalTransport, - compact_native_goal_receipt, - probe_native_goal_process, - refresh_native_goal_status, - start_native_goal_turn, - wait_native_goal_turn, -) - -from .profiles import Mode - - -class HostError(RuntimeError): - """app-server 那一侧没能按契约走完。""" - - -def app_server_command(codex_bin: str) -> list[str]: - """起 app-server 的 argv。 - - 两个 feature 都要开：goals 是 Goal 事务本身，unified_exec 是工具面。基线臂 - 也开 unified_exec，两臂工具面必须一致，否则比的是工具不是 harness。 - """ - - return [ - codex_bin, - "app-server", - "--listen", "stdio://", - "--enable", "goals", - "--enable", "unified_exec", - ] - - -@dataclass -class CodexHost: - """把一个 task_body 交给 codex 跑。""" - - codex_bin: str - mode: Mode - model: str | None = None - effort: str | None = None - """推理档位，进 turn/start 的 turn_params.effort（native_codex_goal.py:277）。""" - sandbox: str = "danger-full-access" - required_skill_ids: tuple[str, ...] = () - response_timeout_sec: float = 60.0 - goal_timeout_sec: float = 1800.0 - process_env: dict[str, str] | None = None - - def _config(self, *, cwd: str, objective: str, task_instruction: str) -> NativeGoalConfig: - return NativeGoalConfig( - cwd=cwd, - objective=objective, - task_instruction=task_instruction, - model=self.model, - effort=self.effort, - sandbox=self.sandbox, - required_skill_ids=self.required_skill_ids, - ) - - def _env(self) -> dict[str, str]: - """app-server 的环境。 - - process_env 里带着 profile 的 CODEX_HOME —— 少了它，codex 的 skills/list - 找不到装好的技能，required_skill_ids 门禁会以 - required_skills_missing 失败。preflight 和 run 必须用同一份，否则 - preflight 过了 run 才炸（或者反过来），白跑。 - """ - - return {**os.environ, **(self.process_env or {})} - - def preflight(self, *, cwd: str, objective: str, task_instruction: str, - process_cwd: str | None = None) -> dict[str, Any]: - """只证 initialize / thread / Goal 挂载，不起模型 turn。不烧 token。""" - - turn = probe_native_goal_process( - self._config(cwd=cwd, objective=objective, task_instruction=task_instruction), - process_command=app_server_command(self.codex_bin), - process_env=self._env(), - process_cwd=process_cwd, - response_timeout_sec=self.response_timeout_sec, - ) - receipt = compact_native_goal_receipt(turn) - receipt["execution_mode"] = "goal_attachment_preflight" - return receipt - - def run(self, *, cwd: str, objective: str, task_instruction: str, - process_cwd: str | None = None) -> dict[str, Any]: - """跑一轮，跑到 Goal 离开 active 或预算耗尽为止。 - - **不用上游的 run_native_goal_process_until_terminal**：那个函数在预算耗尽时 - 抛 NativeGoalDeadlineExceeded，而异常不携带 turn 对象，于是整份收据全丢。 - 实测后果是——模型已经把任务做完（文件改了、测试建了、跑通了），只因为 Goal - 还没离开 active 就被记成"彻底失败、零信息"。 - - 长程任务里预算耗尽是**正常终止**而不是异常：SWE-Marathon 的任务超时上到 - 10 小时，本来就指望跑满预算再交给验证器判分。所以这里照抄上游的循环 - （native_codex_goal.py:412-449 同样的 start/wait/refresh 三步），只把终止 - 原因作为数据返回，收据一律保留。 - """ - - config = self._config(cwd=cwd, objective=objective, - task_instruction=task_instruction) - config.validate() - deadline = time.monotonic() + self.goal_timeout_sec - stop_reason = "goal_terminal" - - with StdioNativeGoalTransport.spawn( - app_server_command(self.codex_bin), - env=self._env(), - cwd=process_cwd or cwd, # spawn 的 cwd 是必填 - ) as transport: - turn = start_native_goal_turn(transport, config) - completed_before = turn.turn_completed_count - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - stop_reason = "budget_exhausted" - break - try: - wait_native_goal_turn( - transport, turn, - timeout_sec=remaining, - completed_before=completed_before, - ) - except NativeGoalDeadlineExceeded: - stop_reason = "budget_exhausted" - break - except NativeGoalProtocolError as exc: - if str(exc) == "goal_turn_timeout": - stop_reason = "budget_exhausted" - break - raise - completed_before = turn.turn_completed_count - if refresh_native_goal_status(transport, turn) != "active": - break - # 心跳模式一次唤醒只做一段：Codex 的续跑归外部调度器管， - # 这一轮到此为止，下一 tick 由 LoopX 重新渲染 body 再起。 - if self.mode.continuation_owner == "driver": - stop_reason = "single_wake_complete" - break - - receipt = compact_native_goal_receipt(turn) - receipt["execution_mode"] = ( - "goal_until_terminal" if self.mode.continuation_owner == "codex" - else "goal_single_wake" - ) - receipt["continuation_owner"] = self.mode.continuation_owner - receipt["stop_reason"] = stop_reason - return receipt - - -def classify(receipt: dict[str, Any]) -> str: - """把 Goal 收据折成一个 refresh-state 能吃的分类标签。 - - 只看运行时自己的 typed 计数，不去解析模型说了什么——模型自称完成不是完成。 - """ - - status = str(receipt.get("post_goal_status") or "") - turns = int(receipt.get("turn_completed_count") or 0) - if receipt.get("terminal_event_observed") and status and status != "active": - return "validated_progress" if turns else "replan_required" - # 预算耗尽但确实推进过：算进展，不算需要修复——判分交给 benchmark 的验证器。 - if turns: - return "validated_progress" - return "repair_required" diff --git a/benchmark/swe-marathon/runtime/modes/profile_install.py b/benchmark/swe-marathon/runtime/modes/profile_install.py deleted file mode 100644 index 8501392d56..0000000000 --- a/benchmark/swe-marathon/runtime/modes/profile_install.py +++ /dev/null @@ -1,102 +0,0 @@ -"""安装一份隔离的 LoopX profile，供三种模式共用。 - -上游 benchmark/deepswe/README.md 要求 treatment 臂有三项独立的产品路径证据： - - 1. 该 profile 渲染出的 Goal body； - 2. LoopX 技能装进 app-server 实际使用的那个 CODEX_HOME； - 3. body 里点名的那个 release-snapshot CLI 确实存在。 - -只做第 1 项是不够的——实测过：body 里写着让模型用 `loopx-project` / -`loopx-self-repair` 技能、跑 `loopx ...` 命令，但技能没装、PATH 上的 `loopx` 还是 -另一个安装，于是模型拿到一份自己无法执行的指令，跑满预算、工作区零改动、 -且不报错。三项必须一起给。 - -`install_native_codex_profile` 一次把三项都办了，返回的 NativeCodexProfile 里 -codex_home / cli_bin / required_skill_ids 就是那三项证据。 -""" - -from __future__ import annotations - -import os -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from loopx.capabilities.benchmark_toolkit.native_codex_profile import ( - NativeCodexProfile, - NativeCodexProfileError, - install_native_codex_profile, - native_codex_profile_environment, -) - - -class ProfileError(RuntimeError): - """隔离 profile 没装成。""" - - -@dataclass(frozen=True) -class InstalledProfile: - """装好的 profile + 它的公开身份摘要。""" - - profile: NativeCodexProfile - - @property - def cli_bin(self) -> str: - return str(self.profile.cli_bin) - - @property - def codex_home(self) -> str: - return str(self.profile.codex_home) - - @property - def required_skill_ids(self) -> tuple[str, ...]: - return tuple(self.profile.required_skill_ids) - - def env(self, *, base: dict[str, str] | None = None) -> dict[str, str]: - """app-server 该用的环境。 - - native_codex_profile_environment 会把 HOME/CODEX_HOME/PATH 指到 profile - 里，并且是 credential-free 的：模型凭证只通过 provider 网关的 URL 和一个 - 固定的非密 env 哨兵进去，不把宿主机的密钥暴露给 danger-full-access 的 - agent。 - """ - - return dict(native_codex_profile_environment( - self.profile, base_env=base if base is not None else dict(os.environ) - )) - - def receipt(self) -> dict[str, Any]: - """写进产物的公开身份，只有摘要和 id，不含本地路径。""" - - p = self.profile - return { - "source_revision": p.source_revision, - "source_clean": p.source_clean, - "skills_digest": p.skills_digest, - "required_skill_ids": list(p.required_skill_ids), - "materialized_skill_ids": list(p.materialized_skill_ids), - } - - -def install(source_root: str | Path, profile_root: str | Path, *, - python_executable: str | None = None, - require_clean_source: bool = False) -> InstalledProfile: - """装一份 profile。 - - profile_root 必须不存在或为空——上游刻意不修复半装的 profile，因为混着两个 - 安装版本会让整个 treatment 失效。所以每次跑用一个新目录。 - - require_clean_source 默认放宽：wen 里的 loopx 是带本地改动的工作副本时， - 严格模式会直接拒装。跑正式对照时应当传 True。 - """ - - try: - profile = install_native_codex_profile( - source_root, - profile_root, - python_executable=python_executable, - require_clean_source=require_clean_source, - ) - except NativeCodexProfileError as exc: - raise ProfileError(f"profile 安装失败: {exc}") from exc - return InstalledProfile(profile=profile) diff --git a/benchmark/swe-marathon/runtime/modes/profiles.py b/benchmark/swe-marathon/runtime/modes/profiles.py deleted file mode 100644 index 05d96944f6..0000000000 --- a/benchmark/swe-marathon/runtime/modes/profiles.py +++ /dev/null @@ -1,189 +0,0 @@ -"""三种 Codex × LoopX 运行模式的声明式定义。 - -模式取自 LoopX README 的 Codex 三行 host 表（upstream README.md:289-291）。 -三者的差别不是"接法不同"，而是 LoopX 渲染出的 body、闸门命令、结算来源不同—— -这里把差别集中成数据，驱动逻辑（session.py / codex_host.py）对三者一视同仁。 - -实测三个 profile 在同一个 goal 上渲染出的差别（loopx 0.5.3）： - - codex_app_ssh_goal body 2720 字符 guard 带 --begin-turn spend --source visible-goal - codex_cli body 2698 字符 guard 不带 --begin-turn spend --source visible-goal - codex_app_heartbeat body 1557 字符 guard --codex-app + LOOPX_TURN spend --source heartbeat - -前两者是 visible-Goal 渲染器（body 开头"in this visible Codex `/goal`"）， -Codex 自己拥有续跑；第三个是精简派发器 body，每次唤醒是全新 turn，续跑由外部 -调度器拥有。 -""" - -from __future__ import annotations - -from dataclasses import dataclass - - -class ModeError(RuntimeError): - """模式定义或使用方式不成立。""" - - -@dataclass(frozen=True) -class Mode: - """一种运行模式。""" - - name: str - """命令行上的短名。""" - - runtime_profile: str - """传给 loopx 的 --runtime-profile。""" - - host_surface: str - scheduler_owner: str - execution_mode: str - """profile 展开后的三元组，仅用于自检与记录，不重复传给 CLI。""" - - continuation_owner: str - """'codex' = Codex 自己续跑（visible Goal）；'driver' = 本驱动按节拍再唤醒。""" - - needs_turn_instance: bool = False - """是否每 tick 需要一个 LOOPX_TURN turn-instance-id。""" - - spend_source: str = "visible-goal" - - notes: str = "" - """这个模式在无人值守环境下的真实边界，写进产物收据里。""" - - substitution: str = "" - """若本适配器用了替代传输/替代 host_surface，在这里写清楚。空 = 无替代。""" - - -#: `Codex App over SSH` —— LoopX 自己的 benchmark 方法唯一认可的 treatment 臂。 -#: benchmark/deepswe/README.md 要求三项产品路径证据：本 profile 渲染的 Goal body、 -#: 装进 app-server 所用 CODEX_HOME 的 skills、以及 body 里点名的 release CLI。 -SSH_GOAL = Mode( - name="ssh-goal", - runtime_profile="codex_app_ssh_goal", - host_surface="codex_app_ssh", - scheduler_owner="agent_cli_loop", - execution_mode="interactive", - continuation_owner="codex", - spend_source="visible-goal", - notes=( - "LoopX benchmark/deepswe 的官方 treatment 臂。app-server 承载 visible Goal，" - "Codex 拥有自动续跑；驱动只起首轮 turn 并观察终态。" - ), -) - -#: `Codex CLI` —— 可见 `/goal`。文档（docs/product/runtimes/codex-cli/ -#: codex-cli-tui-loop.md 的 "Headless Disabled Boundary"）明确：这条路默认不提供 -#: headless 回退，连 opt-in 都没有；`codex-cli-exec-handoff` 已从"输出可运行脚本" -#: 改成"报告禁用边界"。真正的 TUI 注入（Session-Attached Automation）在上游是 -#: 一串 dry-run 诊断，没有任何代码真的往活 TUI 里写。 -#: -#: 所以本适配器对这个模式做的是**传输替代**：body 仍由 `--runtime-profile codex_cli` -#: 渲染（渲染器、闸门命令、结算来源都是真的），但承载它的是 app-server 的 Goal -#: 事务而不是人值守的 TUI。这样模式语义可测，且不假装有人在场。 -CODEX_CLI = Mode( - name="codex-cli", - runtime_profile="codex_cli", - host_surface="codex_cli", - scheduler_owner="agent_cli_loop", - execution_mode="interactive", - continuation_owner="codex", - spend_source="visible-goal", - notes=( - "上游把有人值守的 TUI 当作本模式的定义特征，headless 回退被显式禁用。" - "无人值守跑分时 body/闸门/结算是真的，承载传输是替代的。" - ), - substitution=( - "transport: 用 codex app-server 的 Goal 事务承载 codex_cli 渲染的 visible " - "body，替代人值守 TUI 里的 `/goal` 粘贴。渲染 profile 未被替换。" - ), -) - -#: `Codex App` 心跳 —— host_automation + hosted_automation。 -#: -#: host_surface=codex_app 在上游是留给真 Codex App 产品的：它会回一套 -#: scheduler_hint.codex_app.stateful_backoff（apply_needed / recommended_rrule / -#: reset_token / identity_signature），期待宿主去调 App 自己的 automation_update -#: 改 RRULE 再 ACK。没有真 App 就兑现不了，只会永远悬着或者伪造 ACK。 -#: -#: 上游为自建定时器指定的对口是 `--runtime-profile generic_cli`：shell_worker -#: 参考实现 `scripts/external_scheduler_worker.py` 默认就是它，其 help 明写 -#: "Quota runtime profile that emits the local_scheduler hint"。 -#: -#: 不要直接传 -H local_scheduler：`--turn-instance-id` 只接受 generic_cli 或 -#: codex_app_heartbeat（否则报 "requires runtime-profile generic_cli or -#: codex_app_heartbeat so quota guard creates a heartbeat receipt"），而没有 -#: turn instance 就拿不到心跳收据。 -HEARTBEAT = Mode( - name="heartbeat", - runtime_profile="generic_cli", - host_surface="generic_cli", - scheduler_owner="agent_cli_loop", - execution_mode="interactive", - continuation_owner="driver", - needs_turn_instance=True, - spend_source="heartbeat", - notes=( - "自建定时器拥有唤醒，对应上游 shell_worker 连接器。闸门发 local_scheduler " - "提示（初始间隔 + 递进阶梯 + 未变轮询上限），驱动照 external_scheduler_" - "worker.py 的做法推进阶梯。" - ), - substitution=( - "host_surface: generic_cli 代替 codex_app —— 这是上游为自建定时器指定的" - "对口 profile，不是权宜之计。用 --claim-codex-app 可切到硬声明 codex_app。" - ), -) - -#: `--claim-codex-app` 时用的变体：硬声明 codex_app。会拿到 App 形状的 -#: stateful_backoff，但本驱动没有真 App 去 automation_update，apply_needed / -#: ack_needed 会一直悬着。仅用于观察这套义务在无 App 环境下如何卡住。 -HEARTBEAT_CLAIM_APP = Mode( - name="heartbeat-codex-app", - runtime_profile="codex_app_heartbeat", - host_surface="codex_app", - scheduler_owner="host_automation", - execution_mode="hosted_automation", - continuation_owner="driver", - needs_turn_instance=True, - spend_source="heartbeat", - notes=( - "硬声明 codex_app。校验只查枚举组合、不验身份，所以能过；但没有真 App，" - "scheduler_hint 的 apply_needed/ack_needed 无法诚实兑现。" - ), - substitution=( - "host_surface: 声明为 codex_app 但没有真 Codex App 支撑。" - "属上游文档意义上的误用，只用于观察义务如何悬空。" - ), -) - - -MODES: dict[str, Mode] = {m.name: m for m in (SSH_GOAL, CODEX_CLI, HEARTBEAT)} -MODES[HEARTBEAT_CLAIM_APP.name] = HEARTBEAT_CLAIM_APP - - -def resolve(name: str, *, claim_codex_app: bool = False) -> Mode: - """按短名取模式；heartbeat 可切成硬声明 codex_app 的变体。""" - - if name == HEARTBEAT.name and claim_codex_app: - return HEARTBEAT_CLAIM_APP - try: - return MODES[name] - except KeyError: - raise ModeError( - f"未知模式 {name!r}；可用：{', '.join(sorted(MODES))}" - ) from None - - -def profile_args(mode: Mode) -> list[str]: - """渲染成 loopx CLI 的 profile 参数。 - - 有具名 profile 就用 --runtime-profile（上游 round-trip 测试保证它与三元组 - 等价）；没有的（local_scheduler）就显式传 -H/-O/-M。 - """ - - if mode.runtime_profile: - return ["--runtime-profile", mode.runtime_profile] - return [ - "-H", mode.host_surface, - "-O", mode.scheduler_owner, - "-M", mode.execution_mode, - ] diff --git a/benchmark/swe-marathon/runtime/modes/run_mode.py b/benchmark/swe-marathon/runtime/modes/run_mode.py deleted file mode 100644 index 5c4f92d0fd..0000000000 --- a/benchmark/swe-marathon/runtime/modes/run_mode.py +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env python3 -"""统一驱动 Codex × LoopX 的三种模式。 - - python3 -m modes.run_mode --mode ssh-goal --project --task-file - python3 -m modes.run_mode --mode codex-cli --project --task-file - python3 -m modes.run_mode --mode heartbeat --project --task-file --ticks 4 - -模式取自 LoopX README 的 Codex 三行 host 表；差别见 profiles.py。 -`--preflight-only` 走通全部 LoopX 侧契约（渲染 + 闸门 + Goal 挂载）但不起模型 -turn，不烧 token，适合当冒烟。 - -产物是一份 JSON 收据，只含稳定标签、计数、摘要，不含任务原文/轨迹/凭证。 -""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -from pathlib import Path -from typing import Any - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from modes.codex_host import CodexHost, classify # noqa: E402 -from modes.profile_install import ProfileError, install as install_profile # noqa: E402 -from modes.profiles import MODES, resolve # noqa: E402 -from modes.session import LoopxSession, SessionError # noqa: E402 - - -def _default_cli() -> str: - here = Path(__file__).resolve().parent.parent - venv = here / ".venv" / "bin" / "loopx" - return str(venv) if venv.exists() else "loopx" - - -def _default_codex() -> str: - here = Path(__file__).resolve().parent.parent - staged = here / "codex" / "codex" - return str(staged) if staged.exists() else "codex" - - -def _default_loopx_src() -> str: - """默认 LoopX 源根：优先 env MR_LOOPX_ROOT，其次从已安装 loopx 包推导， - 再次回退到仓库根下的 loopx/。不硬编码机器/用户专属布局。""" - root = os.environ.get("MR_LOOPX_ROOT") - if root: - return root - try: - import importlib.util - spec = importlib.util.find_spec("loopx") - if spec and spec.origin: - return str(Path(spec.origin).resolve().parent.parent) - except Exception: - pass - return str(Path(__file__).resolve().parent.parent / "loopx") - - -def _parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("--mode", required=True, choices=sorted(MODES), - help="运行模式") - p.add_argument("--claim-codex-app", action="store_true", - help="heartbeat 模式下硬声明 host_surface=codex_app（上游文档" - "意义上的误用，只用于观察 RRULE/ACK 义务如何悬空）") - p.add_argument("--project", required=True, help="任务工作区（Codex 可见的 cwd）") - p.add_argument("--task-file", required=True, help="任务正文文件（UTF-8）") - p.add_argument("--goal-id", default="wen-goal") - p.add_argument("--agent-id", default="wen-codex") - p.add_argument("--objective", default=None, - help="goal 目标；默认取任务正文首行") - p.add_argument("--loopx-bin", default=_default_cli()) - p.add_argument("--codex-bin", default=_default_codex()) - p.add_argument("--model", default=os.environ.get("MR_MODEL")) - p.add_argument("--effort", default=os.environ.get("MR_EFFORT"), - help="推理档位，如 xhigh；进 turn/start 的 turn_params.effort") - p.add_argument("--sandbox", default="danger-full-access") - p.add_argument("--ticks", type=int, default=1, - help="heartbeat 模式的最大唤醒次数；visible 模式恒为 1") - p.add_argument("--tick-seconds", type=float, default=0.0, - help="heartbeat 两次唤醒之间的睡眠秒数") - p.add_argument("--turn-timeout", type=float, default=1800.0, - help="单轮 Goal 预算（秒）") - p.add_argument("--preflight-only", action="store_true", - help="只证 LoopX 契约与 Goal 挂载，不起模型 turn") - p.add_argument("--loopx-src", default=None, - help="LoopX 源码根，用来装隔离 profile；默认 wen/loopx") - p.add_argument("--profile-root", default=None, - help="隔离 profile 装到哪；必须是空目录，默认在 --receipt 旁边") - p.add_argument("--no-profile", action="store_true", - help="不装隔离 profile（body 会引用裸 loopx、技能不装，" - "模型多半执行不了——只在调试渲染时用）") - p.add_argument("--codex-config", default=None, - help="provider 配置（config.toml），装完 profile 后拷进它的 " - "codex-home；不给的话 app-server 不知道往哪调模型") - p.add_argument("--require-clean-source", action="store_true", - help="要求 LoopX 源码干净才肯装 profile；正式对照应当开") - p.add_argument("--receipt", default=None, help="收据写到这个文件") - return p - - -def _one_turn(session: LoopxSession, host: CodexHost, *, task: str, - turn_instance: str | None, preflight: bool, - project: Path) -> dict[str, Any]: - """一轮：渲染 body → 过闸门 → 交给 codex → 结算。""" - - body = session.render_body(turn_instance=turn_instance) - objective = body["task_body"] - - decision = session.should_run(turn_instance=turn_instance) - obligations = session.scheduler_obligations(decision) - turn: dict[str, Any] = { - "turn_instance": turn_instance, - "objective_chars": len(objective), - "should_run": decision.get("should_run"), - "effective_action": decision.get("effective_action"), - "gate_reason": str(decision.get("reason") or "")[:200], - "scheduler_obligations": obligations, - } - - if not decision.get("should_run"): - turn["outcome"] = "gate_declined" - return turn - - if preflight: - turn["goal_receipt"] = host.preflight( - cwd=str(project), objective=objective, task_instruction=task, - process_cwd=str(project), - ) - turn["outcome"] = "preflight_only" - return turn - - receipt = host.run(cwd=str(project), objective=objective, - task_instruction=task, process_cwd=str(project)) - turn["goal_receipt"] = receipt - classification = classify(receipt) - turn["classification"] = classification - turn["settlement"] = _compact_settlement(session.settle( - classification=classification, - todo_id=session.selected_todo_id(decision), - )) - turn["outcome"] = "ran" - return turn - - -def _compact_settlement(settled: dict[str, Any]) -> dict[str, Any]: - """结算结果只留可公开的状态位。""" - - spend = settled.get("spend_slot") or {} - refresh = settled.get("refresh_state") or {} - before = spend.get("before") or {} - quota = before.get("quota") or {} - return { - "refresh_ok": refresh.get("ok"), - "spend_ok": spend.get("ok"), - # agent 在 turn 内自己结算过时 spend_ok=false 是正常的，reason 说明原因 - "spend_reason": str(spend.get("reason") or "")[:160] or None, - "spent_slots_total": quota.get("spent_slots"), - "quota_state": quota.get("state") or before.get("state"), - } - - -def main(argv: list[str] | None = None) -> int: - args = _parser().parse_args(argv) - mode = resolve(args.mode, claim_codex_app=args.claim_codex_app) - project = Path(args.project).expanduser().resolve() - if not project.is_dir(): - print(f"FATAL: --project 不是目录: {project}", file=sys.stderr) - return 2 - task = Path(args.task_file).read_text(encoding="utf-8").strip() - if not task: - print("FATAL: --task-file 是空的", file=sys.stderr) - return 2 - objective = args.objective or task.splitlines()[0][:200] - - # ── 隔离 profile：一次给齐上游要求的三项产品路径证据 ────────────────── - # 1) 本模式渲染的 Goal body 2) 技能装进 app-server 实际用的 CODEX_HOME - # 3) body 里点名的那个 release CLI 真的存在 - installed = None - if not args.no_profile: - src = args.loopx_src or _default_loopx_src() - root = args.profile_root or str( - Path(args.receipt).resolve().parent / f"profile-{mode.name}" - if args.receipt else Path(f"/tmp/wen-profile-{mode.name}-{os.getpid()}") - ) - try: - installed = install_profile(src, root, python_executable=sys.executable, - require_clean_source=args.require_clean_source) - except ProfileError as exc: - print(f"FATAL: {exc}", file=sys.stderr) - return 2 - if args.codex_config: - dest = Path(installed.codex_home) / "config.toml" - dest.parent.mkdir(parents=True, exist_ok=True) - dest.write_text(Path(args.codex_config).read_text(encoding="utf-8"), - encoding="utf-8") - - cli = installed.cli_bin if installed else args.loopx_bin - session = LoopxSession(cli=cli, project=project, goal_id=args.goal_id, - agent_id=args.agent_id, mode=mode, - cli_bin=installed.cli_bin if installed else "", - env=installed.env() if installed else {}) - host = CodexHost(codex_bin=args.codex_bin, mode=mode, model=args.model, - effort=args.effort, sandbox=args.sandbox, - goal_timeout_sec=args.turn_timeout, - required_skill_ids=installed.required_skill_ids if installed else (), - process_env=installed.env() if installed else None) - - out: dict[str, Any] = { - "schema": "wen_mode_run_v0", - "mode": mode.name, - "runtime_profile": mode.runtime_profile, - "scheduler_context": { - "host_surface": mode.host_surface, - "scheduler_owner": mode.scheduler_owner, - "execution_mode": mode.execution_mode, - }, - "continuation_owner": mode.continuation_owner, - "substitution": mode.substitution or None, - "notes": mode.notes, - "model": args.model, - "effort": args.effort, - "preflight_only": bool(args.preflight_only), - "loopx_profile": installed.receipt() if installed else None, - "turns": [], - } - - try: - session.bootstrap(objective) - # 任务必须作为 todo 进 goal —— 闸门按 todo 选工作，只放进 turn 输入的话 - # 模型没有可执行 todo，会跑满预算却零产出且不报错。 - session.add_task_todo(task) - except SessionError as exc: - out["error"] = f"bootstrap 失败: {exc}" - _emit(out, args.receipt) - return 1 - - # visible Goal 由 Codex 自己续跑，驱动只起一轮；心跳由驱动按节拍唤醒。 - ticks = args.ticks if mode.continuation_owner == "driver" else 1 - - for i in range(ticks): - if i and args.tick_seconds: - time.sleep(args.tick_seconds) - ti = session.new_turn_instance() if mode.needs_turn_instance else None - try: - turn = _one_turn(session, host, task=task, turn_instance=ti, - preflight=args.preflight_only, project=project) - except Exception as exc: # noqa: BLE001 — 收据要如实记录失败 - turn = {"turn_instance": ti, "outcome": "error", - "error": f"{type(exc).__name__}: {str(exc)[:300]}"} - out["turns"].append(turn) - if turn.get("outcome") == "error": - break - # Goal 已到终态就不用再唤醒 - receipt = turn.get("goal_receipt") or {} - if str(receipt.get("post_goal_status") or "") not in ("", "active"): - out["stopped_early"] = "goal_left_active" - break - - out["turns_run"] = len(out["turns"]) - _emit(out, args.receipt) - return 0 if out["turns"] and out["turns"][-1].get("outcome") != "error" else 1 - - -def _emit(payload: dict[str, Any], path: str | None) -> None: - text = json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=True) - print(text) - if path: - Path(path).parent.mkdir(parents=True, exist_ok=True) - Path(path).write_text(text + "\n", encoding="utf-8") - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmark/swe-marathon/runtime/modes/session.py b/benchmark/swe-marathon/runtime/modes/session.py deleted file mode 100644 index 03108b272f..0000000000 --- a/benchmark/swe-marathon/runtime/modes/session.py +++ /dev/null @@ -1,293 +0,0 @@ -"""LoopX 侧的一轮：渲染 body → 过闸门 → 结算。 - -对三种模式一视同仁；模式差别全部来自 profiles.Mode，本模块不写 if mode ==。 - -刻意不复用 benchmark_toolkit.native_codex_profile.render_native_codex_goal_prompt： -那个函数把 `codex_app_ssh_goal` 写死在三处（native_codex_profile.py:337-338、366、 -399），换 profile 就会在 `runtime_profile != "codex_app_ssh_goal"` 那道断言上失败。 -这里直接调同一个 CLI 子命令，把 profile 参数化，其余校验照抄它的意图。 -""" - -from __future__ import annotations - -import json -import os -import subprocess -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from .profiles import Mode, profile_args - -#: body 里出现这些字样说明 LoopX 认为该模式是 visible Goal 而非心跳自动化。 -_VISIBLE_MARKER = "visible Codex" -#: 心跳 body 会要求宿主提供 turn instance。 -_TURN_ENV = "LOOPX_TURN" - - -class SessionError(RuntimeError): - """LoopX 侧的一轮没能按契约走完。""" - - -def _utc_now_iso() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -@dataclass -class LoopxSession: - """绑定到一个 goal + agent 的 LoopX 会话。""" - - cli: str - """执行用的 loopx 可执行文件路径。""" - - project: Path - goal_id: str - agent_id: str - mode: Mode - cli_bin: str = "" - """写进 body 的 CLI 路径（--cli-bin）。留空则 body 里渲染成裸 `loopx`。""" - timeout_sec: float = 120.0 - env: dict[str, str] = field(default_factory=dict) - - # ── 底层 ──────────────────────────────────────────────────────────────── - def _run(self, args: list[str], *, extra_env: dict[str, str] | None = None, - expect_ok: bool = True) -> dict[str, Any]: - """跑一条 loopx 子命令，要求 --format json 且能解析。 - - expect_ok 默认开：loopx 的写命令失败时**照样退出 0**，只在 JSON 里把 - ok 置 false。不查这一位的话，写不进去的 todo、注册不上的 agent 都会静默 - 通过，一直到几百秒后模型行为不对才发现。 - """ - - argv = [self.cli, "--format", "json", *args] - run_env = {**os.environ, **self.env, **(extra_env or {})} - proc = subprocess.run( - argv, - cwd=str(self.project), - capture_output=True, - text=True, - timeout=self.timeout_sec, - env=run_env, - ) - text = proc.stdout.strip() - # loopx 的 json 输出有时包在 ``` 围栏里 - if text.startswith("```"): - text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip() - if not text: - raise SessionError( - f"loopx {' '.join(args[:2])} 无输出 (exit={proc.returncode}): " - f"{proc.stderr.strip()[:300]}" - ) - try: - payload = json.loads(text) - except json.JSONDecodeError as exc: - raise SessionError( - f"loopx {' '.join(args[:2])} 输出不是 JSON: {text[:300]}" - ) from exc - if expect_ok and isinstance(payload, dict) and payload.get("ok") is False: - raise SessionError( - f"loopx {' '.join(args[:2])} 失败: " - f"{str(payload.get('error'))[:400]}" - ) - return payload - - # ── 一次性准备 ────────────────────────────────────────────────────────── - def bootstrap(self, objective: str) -> dict[str, Any]: - """建 goal 并把本 agent 注册进 coordination.registered_agents。 - - 注册这步不能省：不注册的话 heartbeat-prompt 会返回 ok=false，错误信息是 - "cannot be used because goal has no coordination.registered_agents list"， - 而它仍然退出 0，很容易被当成渲染成功。 - - bootstrap 不再写入任何首连 onboarding todo（user gate / 候选 todo / - connection validation 都已删除），所以接 goal 之后闸门里不会再有 - 需要人工放行的条目，无人值守环境也不会静默空转。 - """ - - boot = self._run([ - "bootstrap", - "--project", ".", - "--goal-id", self.goal_id, - "--objective", objective, - ]) - self._run([ - "configure-goal", - "--goal-id", self.goal_id, - "--registered-agent", self.agent_id, - "--execute", - ]) - return boot - - def add_task_todo(self, task_text: str, *, todo_id: str = "wen-task") -> dict[str, Any]: - """把任务正文作为一条 P0 agent todo 写进 goal。 - - 这一步不能省，也不能只靠 turn/start 的输入：闸门是按 todo 选工作的， - 任务不在 todo 里就不会被选中。实测过只把任务放进 turn 输入的情况，900 秒 - 里只建了 .loopx/ 和 .codex/，任务文件一个字没改，而且不报错——闸门放行、 - Goal 活着、收据干净，看起来一切正常。 - - 闸门是按 todo 选工作的，任务不在 todo 里就不会被选中。 - """ - - return self._run([ - "todo", "add", - "--goal-id", self.goal_id, - "--role", "agent", - "--todo-id", todo_id, - "--text", task_text, - "--task-class", "advancement_task", - "--status", "open", - "--execute", - ]) - - # ── 每轮 ──────────────────────────────────────────────────────────────── - def render_body(self, *, turn_instance: str | None = None) -> dict[str, Any]: - """渲染本模式的 task_body，并校验它确实属于本模式。""" - - args = [ - "heartbeat-prompt", "--thin", - "--goal-id", self.goal_id, - "--agent-id", self.agent_id, - *profile_args(self.mode), - ] - # body 里会写"用 ` ...` 跑下一步"。不传的话渲染成裸 `loopx`， - # 模型会去 PATH 上找——那多半是另一个安装、另一个 registry。实测这条不传 - # 的后果是模型拿到一份自己执行不了的指令，跑满预算、零改动、且不报错。 - if self.cli_bin: - args += ["--cli-bin", self.cli_bin] - - if self.mode.needs_turn_instance: - if not turn_instance: - raise SessionError( - f"{self.mode.name} 每轮需要 turn instance（body 里会引用 " - f"{_TURN_ENV}），调用方没给" - ) - args += ["--turn-instance-id", turn_instance] - - payload = self._run(args, extra_env={_TURN_ENV: turn_instance} if turn_instance else None) - - if not payload.get("ok"): - raise SessionError( - f"heartbeat-prompt 失败({self.mode.name}): " - f"{str(payload.get('error'))[:400]}" - ) - body = (payload.get("task_body") or "").strip() - if not body: - raise SessionError(f"heartbeat-prompt 没给 task_body({self.mode.name})") - - self._assert_body_matches_mode(body, payload) - return payload - - def _assert_body_matches_mode(self, body: str, payload: dict[str, Any]) -> None: - """确认渲染出来的确实是本模式的 body，而不是别的模式的。 - - 这一条是仿 native_codex_profile.py:366 的 runtime_profile 断言。渲染器按 - profile 分岔（visible-Goal vs 心跳派发器），拿错了不会报错、只会静默测成 - 另一个模式——那种失败最难发现，所以在这里挡住。 - """ - - visible = _VISIBLE_MARKER in body - if self.mode.continuation_owner == "codex" and not visible: - raise SessionError( - f"{self.mode.name} 期望 visible Goal body，实际拿到的不含" - f"{_VISIBLE_MARKER!r}——渲染 profile 可能没生效" - ) - if self.mode.continuation_owner == "driver" and visible: - raise SessionError( - f"{self.mode.name} 期望心跳派发器 body，实际拿到的是 visible Goal body" - ) - - spend = payload.get("quota_spend_command") or "" - want = f"--source {self.mode.spend_source}" - if want not in spend: - raise SessionError( - f"{self.mode.name} 的 spend 命令应含 {want!r}，实际: {spend[:200]}" - ) - - def should_run(self, *, turn_instance: str | None = None) -> dict[str, Any]: - """闸门。返回完整决策，调用方读 should_run / interaction_contract。""" - - args = [ - "quota", "should-run", - "--goal-id", self.goal_id, - "--agent-id", self.agent_id, - *profile_args(self.mode), - ] - if self.mode.needs_turn_instance: - if not turn_instance: - raise SessionError(f"{self.mode.name} 的闸门需要 turn instance") - args += ["--turn-instance-id", turn_instance] - return self._run(args, extra_env={_TURN_ENV: turn_instance} if turn_instance else None) - - @staticmethod - def selected_todo_id(decision: dict[str, Any]) -> str: - """闸门这一轮选中的 todo。 - - visible Goal 的结算要求绑定恰好一个 todo_id，否则 spend-slot 回 - "visible Goal settlement requires exactly one todo_id or - replan_obligation_id binding"。 - """ - - sel = (decision.get("selected_todo") or {}) - return str(sel.get("todo_id") or "") - - def settle(self, *, classification: str, todo_id: str = "") -> dict[str, Any]: - """一轮做完之后写回状态并花一次配额。 - - 顺序不能反：LoopX 的契约是 spend_only_after_artifact_validation_writeback， - 先 refresh-state 再 spend-slot。 - - **两步都不强制成功**。visible Goal 模式下 body 本身就要求 agent 在 turn 内 - 自己 refresh + spend，等驱动来收尾时 goal 往往已经是 terminal_no_followup、 - 配额也已经花过（实测 spent_slots=2）。这时再 spend 会返回 ok=false， - reason 是"validated closure evidence derives terminal no-follow-up..."—— - 那是**正常的已结算**，不是失败。强行当错误处理会把一次成功的跑判成失败。 - """ - - refresh = self._run([ - "refresh-state", - "--goal-id", self.goal_id, - "--agent-id", self.agent_id, - "--project", ".", - "--classification", classification, - ], expect_ok=False) - spend_args = [ - "quota", "spend-slot", - "--goal-id", self.goal_id, - "--agent-id", self.agent_id, - "--slots", "1", - "--source", self.mode.spend_source, - "--execute", - ] - if todo_id: - spend_args += ["--todo-id", todo_id] - spend = self._run(spend_args, expect_ok=False) - return {"refresh_state": refresh, "spend_slot": spend} - - # ── 心跳专用 ──────────────────────────────────────────────────────────── - @staticmethod - def new_turn_instance() -> str: - """心跳每次唤醒用一个新的 turn instance；重试复用同一个。""" - - return _utc_now_iso() - - @staticmethod - def scheduler_obligations(decision: dict[str, Any]) -> dict[str, Any]: - """从闸门决策里摘出宿主该兑现的调度义务。 - - 对 local_scheduler，这里应该是空的（cadence 由宿主自己拥有）。 - 对硬声明 codex_app 的变体，这里会有 apply_needed/ack_needed——本驱动没有 - 真 App 可以 automation_update，所以只记录、不假装兑现。 - """ - - hint = decision.get("scheduler_hint") or {} - app = hint.get("codex_app") or {} - backoff = app.get("stateful_backoff") or {} - return { - "applicability": app.get("applicability"), - "apply_needed": backoff.get("apply_needed"), - "ack_needed": backoff.get("ack_needed"), - "recommended_rrule": backoff.get("recommended_rrule"), - "host_action": app.get("host_action"), - } diff --git a/benchmark/swe-marathon/runtime/turn/codex_nosandbox_wrapper.py b/benchmark/swe-marathon/runtime/turn/codex_nosandbox_wrapper.py deleted file mode 100644 index 95aec76d0b..0000000000 --- a/benchmark/swe-marathon/runtime/turn/codex_nosandbox_wrapper.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -"""A `codex` stand-in that drops LoopX's sandbox flags before running the real one. - -LoopX's codex-cli host is the path that works: driven through it, a Turn loop -ran four times on one task, committed a 22 KB patch and scored f2p 31/35. Its -one problem is the sandbox — it always passes `--sandbox ` (or -`-c sandbox_mode=...` when resuming), only permits read-only and -workspace-write, and both need bubblewrap, which needs unprivileged user -namespaces these containers do not have: - - bwrap: No permissions to create a new namespace - -Switching to `--host generic-cli` avoided that but bought a worse problem: the -generic host carries its own scheduler contract, and eleven of sixteen turns -died at "LoopX Turn route is not host executable" before any model work, with -no route recorded to explain why. - -So keep the working host and fix the flag instead. This sits earlier on PATH -than the real codex, strips the sandbox arguments, and substitutes the same -`--dangerously-bypass-approvals-and-sandbox` the other two arms already use — -which is also what keeps the three arms identical in permissions. LoopX's -contracts are untouched: it still believes it is driving codex-cli, because it -is. - -Set MR_REAL_CODEX to the real binary; defaults to /usr/local/bin/codex. -MR_LOOPX_CODEX_LOG names the log file; defaults to /tmp/loopx-goal/codex-wrapper.log. -""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import sys -from pathlib import Path - -# Resolve the real binary rather than assuming /usr/local/bin/codex. Codex is -# installed into the image through nvm, so it lives under the Node version's -# bin directory and the hardcoded path does not exist — which made the wrapper -# die before it ever reached Codex, and LoopX report the indistinguishable -# `codex_cli_exit_nonzero`. The wrapper is invoked by absolute path through -# --codex-bin and is not itself on PATH, so a PATH lookup finds the real one. -REAL = ( - os.environ.get("MR_REAL_CODEX") - or shutil.which("codex") - or "/usr/local/bin/codex" -) -BYPASS = "--dangerously-bypass-approvals-and-sandbox" -LOG = Path( - os.environ.get("MR_LOOPX_CODEX_LOG", "/tmp/loopx-goal/codex-wrapper.log") -) - - -def rewrite(argv: list[str]) -> list[str]: - out: list[str] = [] - skip_next = False - for i, arg in enumerate(argv): - if skip_next: - skip_next = False - continue - # `--sandbox ` — new-session form. - if arg == "--sandbox": - skip_next = True - continue - if arg.startswith("--sandbox="): - continue - # `-c sandbox_mode="..."` — resume form. The value is a separate argv - # item after -c, so both have to go, and only when it is that key: -c - # carries every other config override too. - if arg == "-c" and i + 1 < len(argv) and argv[i + 1].startswith("sandbox_mode="): - skip_next = True - continue - out.append(arg) - - # Insert the bypass right after the subcommand so it lands before `--`, - # which codex treats as the end of flags. - if out and out[0] == "exec": - out.insert(1, BYPASS) - else: - out.insert(0, BYPASS) - return out - - -def _log(text: str) -> None: - try: - LOG.parent.mkdir(parents=True, exist_ok=True) - with LOG.open("a", encoding="utf-8") as handle: - handle.write(text.rstrip("\n") + "\n") - except OSError: - pass - - -def main() -> int: - argv = rewrite(sys.argv[1:]) - # Run the real codex as a child rather than execv'ing it, so its stderr can - # be recorded. LoopX reports a failed Turn as `codex_cli_exit_nonzero` and - # keeps neither the exit code's cause nor any output, and the container is - # gone by the time anyone looks — so an execv here means the only evidence - # of why Codex refused is destroyed at the moment it is produced. - # - # stdout stays inherited and untouched: LoopX parses Codex's `--json` - # stream off it, so anything written there would corrupt the Turn. - _log(f"--- argv in : {sys.argv[1:]}") - _log(f"--- argv out: {argv}") - _log(f"--- real : {REAL} (exists={os.path.exists(REAL)})") - try: - completed = subprocess.run( # noqa: S603 - [REAL, *argv], stderr=subprocess.PIPE, check=False - ) - except OSError as exc: - # Without this the wrapper's own failure to start Codex is reported by - # LoopX as `codex_cli_exit_nonzero`, which reads as "the model refused" - # rather than "the binary is not there". - _log(f"--- launch failed: {type(exc).__name__}: {exc}") - sys.stderr.write(f"codex wrapper could not launch {REAL}: {exc}\n") - return 127 - stderr = completed.stderr.decode("utf-8", "replace") if completed.stderr else "" - _log(f"--- exit {completed.returncode}") - if stderr: - _log(stderr) - sys.stderr.write(stderr) - return completed.returncode - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmark/swe-marathon/runtime/turn/goal_codex.py b/benchmark/swe-marathon/runtime/turn/goal_codex.py deleted file mode 100644 index c62d4a02fe..0000000000 --- a/benchmark/swe-marathon/runtime/turn/goal_codex.py +++ /dev/null @@ -1,552 +0,0 @@ -"""Codex driven through its native Goal API, as a Pier agent. - -Pier's stock Codex agent runs `codex exec` once. That is enough to *create* a -Goal — with `features.goals` on, the model will call `create_goal` when asked — -but not to exercise one: the process exits after the first turn, so the -automatic continuation loop that is the entire point of Goal mode never runs. -Measured that way, Goal mode looks like a no-op, and the experiment would -conclude the wrong thing for a purely mechanical reason. - -So this subclass keeps everything Pier does to stand Codex up — npm install -through the CN mirror, CODEX_HOME, auth.json, config.toml, skills, MCP, session -capture — and swaps only the final invocation for the app-server transaction: - - initialize(experimentalApi=true) -> thread/start -> thread/goal/set(active) - -> turn/start -> observe continuation turns while the Goal stays active - -That transaction is not reimplemented here. `native_codex_goal.py` from LoopX -already owns it, is stdlib-only, and is the same code path the LoopX arm will -use later — sharing it is what keeps the two arms differing in LoopX alone -rather than in how each one talks to Codex. It is copied into the container at -run time rather than baked into the image so that the two arms cannot drift. - -The swap is done by intercepting `exec_as_agent` rather than by reimplementing -`run()`. Pier's `run()` is one long method whose setup (auth resolution, -ownership fixes, config blocks) would have to be duplicated and kept in step -with upstream; intercepting the one command that matters leaves that setup -untouched. If Pier ever changes how it invokes Codex, the marker below stops -matching and this fails loudly instead of silently reverting to plain -`codex exec` — which would look like a successful Goal run with no Goal in it. - -Usage: - - MR_AGENT=goal_codex:GoalCodex MR_MODEL=openai/gpt-5.5 ./run.sh --all -i - -Environment: - - MR_GOAL_PREFLIGHT=1 prove Goal attachment and stop before any model - turn — costs nothing, use it first on a new box - MR_GOAL_TIMEOUT_SEC ceiling for the continuation loop (default 5100, - under the 5400 s task budget so the loop stops - itself instead of being killed mid-turn) - MR_GOAL_TOKEN_BUDGET optional Goal token budget - MR_NATIVE_GOAL_MODULE path to LoopX's native_codex_goal.py on the host -""" - -from __future__ import annotations - -import os -import shlex -import subprocess -import tempfile -from pathlib import Path - -from pier.agents.installed.codex import Codex -from pier.models.trial.paths import EnvironmentPaths - -# Pier builds exactly one command containing this; see -# pier/agents/installed/codex.py, Codex.run(). -_CODEX_EXEC_MARKER = "codex exec " - -_REMOTE_DIR = "/tmp/loopx-goal" -_LOOPX_MOUNT = "/opt/loopx" - - -def _installed_loopx_root() -> str: - """已安装 loopx 包的根目录（含 `loopx/` 的上级）。找不到返回空串。 - - 不硬编码任何机器/用户专属路径：优先环境变量 MR_LOOPX_ROOT，其次从已安装包推导。 - """ - root = os.environ.get("MR_LOOPX_ROOT") - if root: - return root - try: - import importlib.util - spec = importlib.util.find_spec("loopx") - if spec and spec.origin: - return str(Path(spec.origin).resolve().parent.parent) - except Exception: - pass - return "" - - -_DEFAULT_LOOPX_ROOT = _installed_loopx_root() -_DEFAULT_MODULE = os.environ.get("MR_NATIVE_GOAL_MODULE") or ( - str(Path(_DEFAULT_LOOPX_ROOT) - / "loopx/capabilities/benchmark_toolkit/native_codex_goal.py") - if _DEFAULT_LOOPX_ROOT else "" -) - -# The Goal objective is fixed rather than derived from the task text. A Goal is -# meant to state the durable intent that survives across continuation turns, -# while the task file already carries the specifics; restating the task as the -# objective gave the model two copies of the same thing and nothing to hold on -# to between turns. DeepSWE grades a committed patch, so committing belongs in -# the objective — a run that solves the task and never commits scores zero. -_OBJECTIVE = ( - "Complete the software engineering task described in the task file. " - "Work in the repository, keep existing behaviour intact, verify the change " - "against the repository's own tests, and commit the finished work to a new " - "branch off main. The goal is complete only once the change is committed." -) - -# A Goal only continues while it is still active, so an objective that one turn -# can satisfy never exercises the continuation loop — and the objective above -# says outright that committing completes it. Across 53 runs the continuation -# count was zero every time, which makes the measured "Goal API has no effect" -# a statement about an objective that never needed the API, not about the API. -# -# This variant withholds completion until work that cannot plausibly finish in -# one turn is done: pass, then re-derive from the tests, then hunt regressions, -# then edge cases. Whether that actually keeps the Goal active is the thing -# being tested — if the continuation count is still zero, single-turn -# termination is Codex's behaviour here rather than an artefact of the wording. -_OBJECTIVE_STAGED = ( - "Complete the software engineering task described in the task file, in " - "stages, and do not consider the goal complete until every stage is done.\n" - "Stage 1: make the target behaviour work and commit it.\n" - "Stage 2: re-read the task description and check your implementation " - "against every requirement it states, including ones you did not address " - "in stage 1. Fix what is missing and commit.\n" - "Stage 3: look for behaviour you may have broken elsewhere in the " - "repository, run the wider test suite, and fix any regression you find.\n" - "Stage 4: consider edge cases the tests may not cover — empty inputs, " - "concurrent use, error paths — and handle the ones the task implies.\n" - "The goal is complete only after stage 4." -) - - -def _objective() -> str: - return _OBJECTIVE_STAGED if os.environ.get("MR_GOAL_OBJECTIVE") == "staged" else _OBJECTIVE - - -# All three arms keep Pier's own agent name. Overriding name() per arm looked -# tidy but fed straight into AgentInstallSpec.fingerprint(), whose first input is -# agent_name — so each arm produced a different PIER_AGENT_INSTALL_FINGERPRINT, -# invalidated the Docker layer cache, and rebuilt `nvm install 22` plus the npm -# install of Codex for every task in every arm: 162 builds where 54 would do. -# It also removed the only fallback for a network outage, since a cached layer -# needs no proxy. The arm is selected by pier_cn.py rebinding AgentName.CODEX, -# which needs no distinct name. - -_WEB_SEARCH_OFF = 'printf "\\nweb_search = \\"disabled\\"\\n" >> "$CODEX_HOME/config.toml"' - - -class PlainCodex(Codex): - """The control arm: same everything, no Goal attached. - - Exists so that Goal vs no-Goal differs in the Goal API and nothing else. - Two things have to be carried over from GoalCodex or the comparison measures - the wrong difference: - - * ``web_search`` off. Stock Codex leaves it at its default, and it is a - hosted tool the container's egress allowlist cannot block, so one arm - could look answers up. - * the objective text. A Goal cannot exist without an objective, so the Goal - arm is necessarily prompted with those three sentences. Withholding them - here would fold "the effect of that wording" into the measured difference. - Appending them leaves the API as the only variable. - - What remains different is intrinsic: `codex exec` runs one turn and exits, - while the app-server keeps serving continuations while the Goal stays - active. That *is* the treatment. - """ - - async def run(self, instruction, environment, context): # type: ignore[override] - return await super().run( - f"{instruction}\n\n{_objective()}", environment, context - ) - - async def exec_as_agent(self, environment, command: str = "", env=None, **kwargs): # type: ignore[override] - if _CODEX_EXEC_MARKER in command: - await super().exec_as_agent(environment, command=_WEB_SEARCH_OFF, env=env) - return await super().exec_as_agent( - environment, command=command, env=env, **kwargs - ) - - -class LoopxCodex(Codex): - """The third arm: Codex driven by LoopX's governed Turn loop. - - The other two arms both stop when the model says it is finished — `codex - exec` exits, and the Goal API marked every one of 53 runs complete on the - first turn. LoopX is the only arm where something other than the model - decides: it runs one Turn, requires an independent validator to prove the - postcondition, and only then commits and spends quota. Turn two happens - because the controller asks for it. - - Everything else is held to the other arms: same model, same disabled - web_search, same bypassed sandbox, same task set. The staged Todo text - mirrors the four-stage objective already tested on the Goal arm, where it - did not produce a single continuation — so any multi-turn behaviour here is - attributable to the loop rather than to the wording. - - LoopX is bind-mounted rather than installed: it declares no runtime - dependencies, and a read-only mount cannot drift between tasks the way 54 - separate installs could. - - Sandbox is ``workspace-write`` rather than the bypass the other two arms - use, because LoopX rejects anything else in two places — the argparse - choices and again in the driver ("Codex CLI sandbox must be read-only or - workspace-write"). Whether Codex can actually execute under it inside - these containers is the thing this arm has to establish first: the app- - server path could not, but that is a different code path from - ``codex exec --sandbox``, and assuming they behave alike is what a smoke - test is for. If it works, the other two arms should move to the same value - so permissions stop being a second difference between the arms. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._loopx_instruction: str | None = None - self._loopx_swapped = False - - async def run(self, instruction, environment, context): # type: ignore[override] - self._loopx_instruction = instruction - self._loopx_swapped = False - try: - return await super().run(instruction, environment, context) - finally: - if not self._loopx_swapped: - raise RuntimeError( - "LoopxCodex never intercepted a `codex exec` command — this " - "run would have been plain Codex with no LoopX loop." - ) - - async def exec_as_agent(self, environment, command: str = "", env=None, **kwargs): # type: ignore[override] - if _CODEX_EXEC_MARKER not in command: - return await super().exec_as_agent( - environment, command=command, env=env, **kwargs - ) - - self._loopx_swapped = True - model = self._command_model_name or (self.model_name or "").split("/")[-1] - - loopx_root = Path(os.environ.get("MR_LOOPX_ROOT", _DEFAULT_LOOPX_ROOT)) - if not (loopx_root / "loopx" / "__init__.py").is_file(): - raise FileNotFoundError(f"LoopX package not found under {loopx_root}") - runner_src = Path(__file__).resolve().parent / "loopx_turn_runner.py" - - await super().exec_as_agent( - environment, command=f"mkdir -p {shlex.quote(_REMOTE_DIR)}", env=env - ) - await super().exec_as_agent(environment, command=_WEB_SEARCH_OFF, env=env) - - # LoopX arrives as one tarball rather than a bind mount: the container is - # created by Pier from its own compose file, so an agent cannot add a - # mount to it, and uploading 710 files one at a time is not a serious - # option. Packed once per run rather than once per task would be nicer - # still, but the tar is ~13 MB and building it is far cheaper than the - # model turn that follows. - with tempfile.TemporaryDirectory() as tmp: - tarball = Path(tmp) / "loopx.tar.gz" - subprocess.run( - ["tar", "czf", str(tarball), "-C", str(loopx_root), "loopx"], - check=True, - ) - task_path = Path(tmp) / "task.txt" - task_path.write_text(self._loopx_instruction or "", encoding="utf-8") - for local, remote in ( - (tarball, f"{_REMOTE_DIR}/loopx.tar.gz"), - (task_path, f"{_REMOTE_DIR}/task.txt"), - (runner_src, f"{_REMOTE_DIR}/loopx_turn_runner.py"), - (runner_src.parent / "codex_nosandbox_wrapper.py", - f"{_REMOTE_DIR}/codex_nosandbox_wrapper.py"), - ): - await environment.upload_file(str(local), remote) - - if environment.default_user is not None: - await self.exec_as_root( - environment, - command=f"chown -R {environment.default_user} {shlex.quote(_REMOTE_DIR)} && chmod +x {shlex.quote(_REMOTE_DIR)}/codex_nosandbox_wrapper.py", - ) - await super().exec_as_agent( - environment, - command=( - f"mkdir -p {shlex.quote(_LOOPX_MOUNT)} && " - f"tar xzf {shlex.quote(_REMOTE_DIR)}/loopx.tar.gz " - f"-C {shlex.quote(_LOOPX_MOUNT)} && " - f"python3 -c 'import sys; sys.path.insert(0, \"{_LOOPX_MOUNT}\"); " - "import loopx.cli_commands.turn'" - ), - env=env, - ) - - args = [ - "python3", - f"{_REMOTE_DIR}/loopx_turn_runner.py", - "--project", "__PWD__", - "--task-file", f"{_REMOTE_DIR}/task.txt", - "--runtime-root", f"{_REMOTE_DIR}/runtime", - "--codex-bin", "codex", - "--model", model, - "--sandbox", os.environ.get("MR_LOOPX_SANDBOX", "workspace-write"), - "--quota", os.environ.get("MR_LOOPX_QUOTA", "4"), - ] - rendered = shlex.join(args).replace("'__PWD__'", '"$(pwd)"').replace( - "__PWD__", '"$(pwd)"' - ) - output = (EnvironmentPaths.agent_dir / "loopx-turns.json").as_posix() - return await super().exec_as_agent( - environment, - command=( - "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " - f"PYTHONPATH={shlex.quote(_LOOPX_MOUNT)} {rendered} " - f"2>&1 {shlex.quote(_REMOTE_DIR)}/run_goal.py <<'LOOPX_EOF'\n{runner}\nLOOPX_EOF", - env=env, - ) - - timeout = os.environ.get("MR_GOAL_TIMEOUT_SEC", "5100") - budget = os.environ.get("MR_GOAL_TOKEN_BUDGET", "") - preflight = os.environ.get("MR_GOAL_PREFLIGHT", "") not in ("", "0") - - args = [ - "python3", - f"{_REMOTE_DIR}/run_goal.py", - "--cwd", - "__PWD__", - "--objective-file", - # The LoopX arm renders its objective with LoopX's own CLI and - # points here; a subclass cannot substitute it by rewriting the - # command, because this method calls `super().exec_as_agent` rather - # than `self.`, so an override never sees the built command at all. - os.environ.get("MR_GOAL_OBJECTIVE_FILE", f"{_REMOTE_DIR}/objective.txt"), - "--task-file", - f"{_REMOTE_DIR}/task.txt", - "--codex-bin", - "codex", - "--model", - model, - "--goal-timeout-seconds", - timeout, - # NativeGoalConfig defaults to sandbox="workspace-write", which on - # Linux is enforced with landlock/seccomp and cannot be set up - # inside these task containers. Codex then declines to run - # commands: the first attempt produced 431 assistant-message deltas, - # zero command_execution events, zero file_change events and an - # empty model.patch, which the verifier scored 0/24 — a harness - # failure that reads exactly like the model failing the task. - # `codex exec` avoids it with --dangerously-bypass-approvals-and- - # sandbox; this is the app-server equivalent, so both arms execute - # under the same permissions. - "--sandbox", - os.environ.get("MR_GOAL_SANDBOX", "danger-full-access"), - ] - if budget: - args += ["--token-budget", budget] - # Empty on this arm. The LoopX arm sets it to the skill ids its - # installed profile materialized, which arms `skills/list` as a - # precondition of thread creation: without it a run in which Codex never - # discovered LoopX still finishes and still scores, and is then filed as - # a LoopX result. One such run has already happened. - skills = os.environ.get("MR_GOAL_REQUIRED_SKILL_IDS", "").strip() - if skills: - args += ["--required-skill-ids", skills] - if preflight: - args.append("--preflight-only") - - # No task declares a working directory, so `codex exec` would have run - # in whatever WORKDIR the image sets — different per repository. Resolve - # it in the container instead of guessing. Both spellings are replaced - # because shlex.join only quotes a token that needs it, and this one - # (letters and underscores) comes back bare: matching only the quoted - # form silently leaves the placeholder in the command, which surfaces as - # FileNotFoundError('__PWD__') from inside the runner. - rendered = shlex.join(args) - rendered = rendered.replace("'__PWD__'", '"$(pwd)"').replace( - "__PWD__", '"$(pwd)"' - ) - - output = (EnvironmentPaths.agent_dir / "codex-goal.json").as_posix() - # The LoopX arm points CODEX_HOME at its installed profile so app-server - # discovers the LoopX skills; this arm leaves it as Pier set it up. - codex_home = os.environ.get("MR_GOAL_CODEX_HOME", "").strip() - prefix = f"export CODEX_HOME={shlex.quote(codex_home)}; " if codex_home else "" - # The profile's `loopx` launcher was built on the host and records the - # host's own Python path, which does not exist in the container. This - # was fixed for the one-shot bootstrap script by exporting the variable - # in that command's own shell -- but bootstrap and this long-lived - # app-server process are separate `docker exec` invocations, each with - # its own shell, so the export did not carry over. Any `loopx` command - # Codex itself runs *during* a turn -- todo claim, quota should-run, - # heartbeat-prompt -- inherits app-server's process environment, not - # bootstrap's, and failed with the same "configured Python executable - # not found" every time until this export is repeated here. A session - # transcript showed exactly that: two failed `loopx` calls mid-turn. - if os.environ.get("MR_GOAL_ARM_LOOPX_PYTHON", "") not in ("", "0"): - prefix += 'export LOOPX_PYTHON="$(command -v python3)"; ' - return await super().exec_as_agent( - environment, - command=( - "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " - f"{prefix}{rendered} 2>&1 /ACTIVE_GOAL_STATE.md, - with its own execution profile - loopx configure-goal registers the peer identity - render_native_codex_goal_prompt the installed CLI renders the real Goal - body -- the thing I used to hand-write - run_native_goal_process_until_terminal - codex app-server owns continuation while - the Goal stays active; nothing here counts - turns - -`required_skill_ids` makes `skills/list` a precondition of thread creation, so a -run in which Codex never discovered the LoopX skills fails before model work -instead of quietly scoring as a LoopX result. - -Everything Pier does to stand Codex up is inherited from GoalCodex, which also -already owns the app-server transaction (it copies LoopX's own -`native_codex_goal.py` into the container). Only three things differ from the -Goal arm: where CODEX_HOME points, who wrote the objective, and whether the -skills gate is armed. That is the intended contrast -- the same host, the same -transaction, LoopX present or absent. - -The profile is built on the host and shipped, rather than installed in the -container: `install-local.sh` is offline (no pip, npm, curl, wget, git clone or -apt in 872 lines, so the sandbox is not the obstacle), but it verifies that its -source tree is a clean checkout, and the container receives a tarball of the -`loopx` package with no `.git` to verify. Building it once on the host keeps -`source_clean` a real claim. Both sides use the same absolute path so the -release snapshot's symlinks stay valid. - -Usage: - - MR_AGENT=loopx_native_codex:LoopxNativeCodex MR_MODEL=openai/gpt-5.5 \ - ./run.sh --all -i - -Environment: - - MR_LOOPX_PREFLIGHT=1 prove skills discovery and Goal attachment, then - stop before any model turn -- costs nothing - MR_LOOPX_PROFILE_ROOT where the profile lives on host and in container - (default /tmp/loopx-profile; must match on both) - MR_LOOPX_ROOT LoopX checkout to install from - MR_GOAL_TIMEOUT_SEC ceiling for the continuation loop -""" - -from __future__ import annotations - -import json -import os -import shlex -import subprocess -import sys -import tempfile -from pathlib import Path - -from goal_codex import GoalCodex, _CODEX_EXEC_MARKER, _REMOTE_DIR, _installed_loopx_root - -_PROFILE_ROOT = os.environ.get("MR_LOOPX_PROFILE_ROOT", "/tmp/loopx-profile") -# LoopX 源根：优先 env，其次从已安装 loopx 包推导；不硬编码机器/用户专属路径。 -_LOOPX_ROOT = os.environ.get("MR_LOOPX_ROOT") or _installed_loopx_root() -_GOAL_ID = "deepswe-task" -_AGENT_ID = "deepswe-codex" -_PROJECT = "/app" -# Deliberately not GoalCodex's objective.txt: that file is written after this -# runs, so sharing the name would let the Goal arm's hand-written objective -# overwrite the one LoopX rendered. -_OBJECTIVE_FILE = f"{_REMOTE_DIR}/loopx_objective.txt" - - -def build_host_profile(loopx_root: str = _LOOPX_ROOT, - profile_root: str = _PROFILE_ROOT) -> dict: - """Install the formal release snapshot once, on the host. - - Reuses an existing profile: the installer refuses a non-empty target on - purpose, because mixing installation revisions would invalidate the - treatment, and re-installing per task would repeat that work 54 times. - """ - sys.path.insert(0, loopx_root) - from loopx.capabilities.benchmark_toolkit.native_codex_profile import ( - compact_native_codex_profile_receipt, - inspect_native_codex_profile, - install_native_codex_profile, - ) - - target = Path(profile_root) - if target.exists() and any(target.iterdir()): - profile = inspect_native_codex_profile(target, source_root=loopx_root) - else: - profile = install_native_codex_profile(loopx_root, target) - return compact_native_codex_profile_receipt(profile) - - -class LoopxNativeCodex(GoalCodex): - """Codex with LoopX installed, driven by LoopX's rendered Goal.""" - - async def exec_as_agent(self, environment, command, env=None, **kwargs): # noqa: ANN001 - # Intercept the same marker GoalCodex does, not the command it builds. - # GoalCodex reaches the launch through `super().exec_as_agent`, so an - # override keyed on `run_goal.py` is never called: the first version of - # this arm silently shipped no profile, armed no skills gate, and still - # produced a receipt that looked like a LoopX run. Everything below is - # therefore handed over through the environment, which GoalCodex reads - # while building its own command. - if _CODEX_EXEC_MARKER not in str(command): - return await super().exec_as_agent(environment, command, env=env, **kwargs) - - profile_receipt = build_host_profile() - parent = str(Path(_PROFILE_ROOT).parent) - name = Path(_PROFILE_ROOT).name - - await super().exec_as_agent( - environment, command=f"mkdir -p {shlex.quote(_REMOTE_DIR)}", env=env - ) - - # Ship the installed profile to the identical absolute path: the release - # snapshot's `bin/loopx` is a symlink into releases/, so a different path - # would leave a dangling CLI and no Goal body could be rendered at all. - with tempfile.TemporaryDirectory() as tmp: - tarball = Path(tmp) / "profile.tar.gz" - subprocess.run( - ["tar", "czf", str(tarball), "-C", parent, name], check=True - ) - receipt_path = Path(tmp) / "profile_receipt.json" - receipt_path.write_text( - json.dumps(profile_receipt, indent=2), encoding="utf-8" - ) - bootstrap_path = Path(tmp) / "loopx_product_bootstrap.py" - bootstrap_path.write_text(_BOOTSTRAP, encoding="utf-8") - for local, remote in ( - (tarball, f"{_REMOTE_DIR}/profile.tar.gz"), - (receipt_path, f"{_REMOTE_DIR}/profile_receipt.json"), - (bootstrap_path, f"{_REMOTE_DIR}/loopx_product_bootstrap.py"), - ): - await environment.upload_file(str(local), remote) - - await super().exec_as_agent( - environment, - command=( - f"mkdir -p {shlex.quote(parent)} && " - f"tar xzf {shlex.quote(_REMOTE_DIR)}/profile.tar.gz " - f"-C {shlex.quote(parent)} && " - f"test -x {shlex.quote(_PROFILE_ROOT)}/bin/loopx" - ), - env=env, - ) - - # LoopX writes its own registry, goal state and Goal body. Nothing in - # this arm authors goal content: the hand-written four-stage document - # the previous arm used is exactly what made its results describe a - # prompt of mine rather than this product. - await super().exec_as_agent( - environment, - command=( - # The profile is installed on the host, so its launcher records - # the host's Python path -- a uv-managed interpreter that does - # not exist in the task image, which made every CLI call exit 2 - # with "configured Python executable not found". LOOPX_PYTHON - # redirects the launcher at the container's own interpreter - # without reinstalling, so the release snapshot stays the one - # whose cleanliness was proven on the host. - f"cd {shlex.quote(_PROJECT)} && " - "export LOOPX_PYTHON=\"$(command -v python3)\" && " - "python3 -c 'import sys; assert sys.version_info >= (3, 11), sys.version' && " - f"python3 {shlex.quote(_REMOTE_DIR)}/loopx_product_bootstrap.py " - f"--profile-root {shlex.quote(_PROFILE_ROOT)} " - f"--project {shlex.quote(_PROJECT)} " - f"--goal-id {_GOAL_ID} --agent-id {_AGENT_ID} " - f"--objective-out {shlex.quote(_OBJECTIVE_FILE)} " - f"--receipt-out {shlex.quote(_REMOTE_DIR)}/loopx_product.json" - ), - env=env, - ) - - os.environ["MR_GOAL_OBJECTIVE_FILE"] = _OBJECTIVE_FILE - os.environ["MR_GOAL_CODEX_HOME"] = f"{_PROFILE_ROOT}/codex-home" - os.environ["MR_GOAL_REQUIRED_SKILL_IDS"] = ",".join( - profile_receipt["required_skill_ids"] - ) - # Arm GoalCodex's LOOPX_PYTHON export for the actual app-server process, - # not just the earlier bootstrap script -- Codex's own mid-turn `loopx` - # calls run inside app-server's environment and hit the same failure - # bootstrap did until this is set here too. - os.environ["MR_GOAL_ARM_LOOPX_PYTHON"] = "1" - if os.environ.get("MR_LOOPX_PREFLIGHT", "") not in ("", "0"): - os.environ["MR_GOAL_PREFLIGHT"] = "1" - - # Give the profile's CODEX_HOME the credentials and provider settings - # Pier wrote into its own. Pointing app-server at the formally - # installed CODEX_HOME is what makes `skills/list` find LoopX, but that - # directory ships only `skills/` -- no auth.json, no config.toml, so no - # API key and no gateway base_url. A run configured that way discovers - # every skill, starts, and then waits for a model it has no address - # for: one smoke hung for 85 minutes and the gateway's call count never - # moved. The preflight cannot catch it, because Goal attachment stops - # before the first model turn and needs no credentials. - # - # Only top-level files are copied. `cp -a` of the whole directory - # would merge Pier's own skills over the formal install, and which - # skills app-server discovers is the one thing this arm must not - # improvise. - profile_home = f"{_PROFILE_ROOT}/codex-home" - await super().exec_as_agent( - environment, - command=( - 'for f in "$CODEX_HOME"/*; do ' - f'[ -f "$f" ] && cp -f "$f" {shlex.quote(profile_home)}/; ' - "done; " - # GoalCodex disables web_search by appending to *its* CODEX_HOME - # after this runs, so the copy above would leave the profile - # without it and give this arm a hosted search tool the other - # two do not have -- an advantage no network isolation would - # reveal, since the provider runs the search. - f'grep -q "^web_search" {shlex.quote(profile_home)}/config.toml ' - f'|| printf "\\nweb_search = \\"disabled\\"\\n" ' - f'>> {shlex.quote(profile_home)}/config.toml; ' - f'test -s {shlex.quote(profile_home)}/config.toml ' - '|| { echo "no config.toml reached the LoopX profile" >&2; exit 1; }' - ), - env=env, - ) - - # Capture LoopX's own trace before the container is torn down. Pier's - # own teardown copies `$CODEX_HOME/sessions` into the job directory, - # but that is Pier's CODEX_HOME -- this arm points app-server at the - # profile's instead, so codex writes its session transcript to - # `{profile_home}/sessions` and Pier's copy finds nothing. The first - # smoke run's job directory logged "No Codex session directory found" - # for exactly this reason: the transcript existed, just one directory - # over from where anyone looked for it. - # - # This has to be a second, separate call rather than the launch command - # rewritten to append a capture step. `command` at this point still - # reads "codex exec ..." -- GoalCodex.exec_as_agent below only checks - # for that marker's presence and then throws the string away, building - # its own `run_goal.py` invocation from internal state. Appending shell - # onto a string GoalCodex never looks at silently does nothing, which is - # exactly the bug that made the CODEX_HOME swap above necessary: this - # arm keeps stumbling on places where a value must go through GoalCodex - # rather than through the string it happens to be holding. - capture_dir = "/logs/agent/loopx_trace" - try: - return await super().exec_as_agent( - environment, command=command, env=env, **kwargs - ) - finally: - await super().exec_as_agent( - environment, - command=( - f"mkdir -p {shlex.quote(capture_dir)}; " - f'if [ -d {shlex.quote(profile_home)}/sessions ]; then ' - f'cp -R {shlex.quote(profile_home)}/sessions ' - f'{shlex.quote(capture_dir)}/sessions; fi; ' - f'find /app/.codex/goals -name ACTIVE_GOAL_STATE.md ' - f'-exec cp {{}} {shlex.quote(capture_dir)}/ACTIVE_GOAL_STATE.md \\; ' - f'2>/dev/null; ' - # Same LOOPX_PYTHON fix as the app-server launch above: this - # CLI call is yet another separate shell, and without its own - # export it fails with the same "configured Python - # executable not found" that showed up twice in mid-turn - # calls before the launch-side fix existed. - 'export LOOPX_PYTHON="$(command -v python3)"; ' - f'{shlex.quote(_PROFILE_ROOT)}/bin/loopx ' - f'--registry {shlex.quote(_PROJECT)}/.loopx/registry.json ' - f'--runtime-root {shlex.quote(_PROJECT)}/.loopx/runtime ' - f'--format json todo list --goal-id {_GOAL_ID} ' - f'> {shlex.quote(capture_dir)}/todo_list.json 2>&1; ' - "true" - ), - env=env, - ) - - -# Runs inside the container: bootstrap, register the peer, render the Goal body. -_BOOTSTRAP = '''\ -import argparse, json, subprocess, sys -from pathlib import Path - -p = argparse.ArgumentParser() -p.add_argument("--profile-root", required=True) -p.add_argument("--project", required=True) -p.add_argument("--goal-id", required=True) -p.add_argument("--agent-id", required=True) -p.add_argument("--objective-out", required=True) -p.add_argument("--receipt-out", required=True) -a = p.parse_args() - -cli = f"{a.profile_root}/bin/loopx" -registry = f"{a.project}/.loopx/registry.json" -runtime = f"{a.project}/.loopx/runtime" -base = [cli, "--registry", registry, "--runtime-root", runtime, "--format", "json"] -steps = {} - - -def run(name, args): - out = subprocess.run(base + args, capture_output=True, text=True) - try: - payload = json.loads(out.stdout) - except Exception: - payload = {} - # Keep returncode and both streams for every step. Pier reports a failed - # agent command as "Command failed" and discards its output, so a step that - # dies here is otherwise invisible: the first run of this arm failed exactly - # once and left nothing but that phrase in the log. - steps[name] = { - "returncode": out.returncode, - "ok": payload.get("ok"), - "error": payload.get("error"), - "stdout_tail": out.stdout[-400:] if not payload else None, - "stderr_tail": out.stderr[-400:], - } - return payload - - -run("bootstrap", ["bootstrap", "--project", a.project, "--goal-id", a.goal_id, - "--objective", "Complete the software engineering task described " - "in the task file and commit the finished work."]) -run("configure_goal", ["configure-goal", "--goal-id", a.goal_id, - "--registered-agent", a.agent_id, "--execute"]) -prompt = run("heartbeat_prompt", - ["heartbeat-prompt", "--thin", "--goal-id", a.goal_id, - "--agent-id", a.agent_id, "--available-capability", "shell", - "--available-capability", "filesystem_write", - "--runtime-profile", "codex_app_ssh_goal", "--cli-bin", cli]) - -body = prompt.get("task_body") -if body: - Path(a.objective_out).write_text(body, encoding="utf-8") - steps["goal_body_chars"] = len(body) -else: - steps["fatal"] = "heartbeat-prompt returned no task_body" -Path(a.receipt_out).write_text(json.dumps(steps, indent=2), encoding="utf-8") -print(json.dumps(steps, indent=2)) -raise SystemExit(0 if body else 1) -''' diff --git a/benchmark/swe-marathon/runtime/turn/loopx_turn_runner.py b/benchmark/swe-marathon/runtime/turn/loopx_turn_runner.py deleted file mode 100644 index d58f2d17ba..0000000000 --- a/benchmark/swe-marathon/runtime/turn/loopx_turn_runner.py +++ /dev/null @@ -1,623 +0,0 @@ -#!/usr/bin/env python3 -"""Drive one DeepSWE task through LoopX's governed Turn loop, inside the container. - -Run by LoopxCodex after the LoopX package, the Codex profile and the goal state -have been staged. Kept as a standalone script for the same reason -native_codex_goal.py is: it must execute where the repository is, and the -repository is inside the task container. - -Why this calls ``handle_turn_command`` instead of the ``loopx turn run-once`` -CLI: the CLI restricts ``--codex-sandbox`` to read-only and workspace-write, -and neither can be set up inside these task images — the Linux sandbox needs -kernel features the container does not grant, and Codex responds by narrating -instead of executing (measured once: 431 assistant messages, zero command -executions, an empty patch scored 0/24). The other two arms run Codex with -approvals and sandbox bypassed, so the LoopX arm has to as well or the three -differ in permissions as well as in looping. Editing LoopX's argparse choices -would also have made the source tree dirty, and install_native_codex_profile -refuses an unclean source because mixing revisions invalidates a benchmark -treatment. Building the Namespace directly avoids both problems and touches -no file in the LoopX checkout. - -The loop is the point of the arm. LoopX runs exactly one governed Turn per -call: it selects a Todo, has the host adapter invoke Codex, requires an -independent validator to prove the postcondition, and only then commits the -result and spends quota. Multi-turn behaviour comes from calling it again -- -which is precisely what the other two arms never do, since `codex exec` and the -app-server both stop as soon as the model says it is finished. -""" - -from __future__ import annotations - -import argparse -import io -import json -import os -import subprocess -import sys -from contextlib import redirect_stdout -from pathlib import Path - -REMOTE_DIR = Path(__file__).resolve().parent -DEFAULT_QUOTA = int(os.environ.get("MR_LOOPX_QUOTA", "4")) -DEFAULT_TURN_TIMEOUT = float(os.environ.get("MR_LOOPX_TURN_TIMEOUT", "1200")) - -GOAL_ID = "deepswe-task" -AGENT_ID = "deepswe-codex" -TODO_ID = "deepswe-todo-1" - - -def hide_loopx_state_from_git(project: Path) -> None: - """Keep LoopX's own files out of git's view. - - The goal document and registry have to live at paths inside the project — - LoopX resolves ``state_file`` relative to the repo — but they are control- - plane state, not the agent's work. Left visible they break the run twice: - ``git status`` never comes back clean, so the validator rejects every Turn, - and they land in ``git diff base..HEAD``, which is exactly the patch the - benchmark grades. - - Written to ``.git/info/exclude`` rather than ``.gitignore`` because that - file is local to the clone and never becomes part of the diff itself. - """ - exclude = project / ".git" / "info" / "exclude" - if not exclude.parent.is_dir(): - return - existing = exclude.read_text(encoding="utf-8") if exclude.exists() else "" - additions = [p for p in (".codex/", ".loopx/") if p not in existing] - if additions: - with exclude.open("a", encoding="utf-8") as fh: - fh.write("\n# LoopX control-plane state (benchmark harness)\n") - fh.write("\n".join(additions) + "\n") - - -def stage_goal_state(project: Path, instruction: str) -> Path: - """Write the goal document LoopX reads Todos from. - - The Todo, not the objective, is what LoopX plans against: it selects one per - Turn and asks the host to advance it. Phrasing it as staged work is what - gives the loop somewhere to go on turn two — a Todo that one turn satisfies - ends the goal exactly the way the Goal-API arm already ends, and the arm - would measure nothing. - """ - state = project / ".codex" / "goals" / GOAL_ID / "ACTIVE_GOAL_STATE.md" - state.parent.mkdir(parents=True, exist_ok=True) - state.write_text( - "\n".join( - [ - "---", - "status: active", - "updated_at: 2026-01-01T00:00:00+00:00", - "---", - "", - "# DeepSWE task", - "", - "## Agent Todo", - "", - "- [ ] [P0] Advance the task below by exactly one stage per Turn, " - "and report which stage you completed and what remains. " - "Stage 1: implement the target behaviour and commit. " - "Stage 2: re-check the implementation against every requirement " - "in the task text, fix gaps, commit. " - "Stage 3: run the wider test suite and repair any regression. " - "Stage 4: handle edge cases the tests do not cover.", - f" ", - "", - "## How to report each Turn", - "", - # gpt-5.5 reached for path_delta_mode=material_replan on a plain - # first implementation Turn, which LoopX rejects four ways at - # once: that mode is reserved for Turns that overturn a prior - # assumption, and it then demands result_kind=replan_required - # plus a goal_path_delta_v0 vision packet the model had not - # produced. Every Turn failed validation, so nothing committed - # and the quota bought nothing. Advancing a stage is routine - # continuation, so say so explicitly rather than leaving the - # model to pick. - "This Todo is routine staged continuation, never a replan. In the", - "typed result set `path_delta_mode=unchanged`, leave", - "`agent_vision_json` empty, and give a one-line", - "`vision_unchanged_reason` such as \"routine stage advance\".", - "Set `delivery_batch_scale` to `implementation` when you changed", - "source, or `test_only` when you only touched tests.", - "Use `result_kind=validated_progress` when the stage advanced and", - "`repair_required` when it did not.", - "", - "## Task", - "", - instruction, - "", - ] - ), - encoding="utf-8", - ) - return state - - -def stage_registry(project: Path, runtime: Path, state: Path) -> Path: - """Write the registry LoopX plans against. - - Shape follows LoopX's own e2e fixture rather than a guess: a goal needs - ``state_file`` to find its Todos, ``quota`` for the scheduler to spend - against, and a ``coordination`` block with ``registered_agents`` — without - the last one every Turn fails at planning with "quota should-run - --agent-id requires coordination.registered_agents", before the host is - ever invoked. - - ``write_scope`` is the repository rather than the fixture's ``docs/**``: - the agent's whole job here is to change source. - """ - registry = project / ".loopx" / "registry.json" - registry.parent.mkdir(parents=True, exist_ok=True) - registry.write_text( - json.dumps( - { - "schema_version": 1, - "common_runtime_root": str(runtime), - "goals": [ - { - "id": GOAL_ID, - "domain": "deepswe-benchmark", - "status": "active", - "repo": str(project), - "state_file": str(state.relative_to(project)), - "adapter": { - "kind": "fixture_v0", - "status": "connected-delivery", - }, - "quota": {"compute": 1.0, "window_hours": 24}, - "coordination": { - "agent_model": "peer_v1", - "registered_agents": [AGENT_ID], - "agent_profiles": { - AGENT_ID: { - "schema_version": "agent_profile_v1", - "profile_role": "benchmark", - "scope": "deepswe task", - } - }, - "write_scope": ["**"], - }, - } - ], - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - return registry - - -def validator_command(project: Path, base_sha: str) -> list[str]: - """Postcondition: the Turn committed something new and left a clean tree. - - Comparing against the base commit is the whole point. An earlier version - only asked for a clean tree and a non-empty history, which any run - satisfies without doing anything at all — four Turns passed validation - while producing an empty patch. A Turn that has not moved HEAD has not - advanced the Todo, whatever the model reports. - - Deliberately structural, never the hidden tests: the benchmark's rules put - verifier invocation after the run and outside the controller, so that a - loop cannot steer on the grade. What this proves is that the agent really - committed work rather than declaring success over an unchanged tree. - """ - program = ( - "import json,subprocess,sys;" - "json.load(sys.stdin);" - f"p={str(project)!r};b={base_sha!r};" - "st=subprocess.run(['git','-C',p,'status','--porcelain']," - "capture_output=True,text=True);" - "hd=subprocess.run(['git','-C',p,'rev-parse','HEAD']," - "capture_output=True,text=True);" - "head=hd.stdout.strip();" - "clean=st.returncode==0 and not st.stdout.strip();" - "raise SystemExit(0 if clean and head and head!=b else 9)" - ) - return [sys.executable, "-c", program] - - -def head_sha(project: Path) -> str: - out = subprocess.run( - ["git", "-C", str(project), "rev-parse", "HEAD"], - capture_output=True, text=True, - ) - return out.stdout.strip() - - -def run_turn(*, project: Path, registry: Path, runtime: Path, codex_bin: str, - model: str, sandbox: str, turn_index: int, base_sha: str, - ) -> dict: - from loopx.cli_commands.turn import handle_turn_command, register_turn_commands - - # Build the Namespace from LoopX's own parser rather than by hand. Hand- - # writing it meant discovering missing attributes one failed Turn at a time - # (`resume_turn_key` was the third), and every LoopX upgrade would restart - # that game. Parsing real argv fills every default the handler expects and - # fails loudly here if an option is ever renamed. - # - # Host is generic-cli, not codex-cli. LoopX's built-in Codex host launches - # `codex exec --sandbox `, and both modes it permits need bubblewrap, - # which needs unprivileged user namespaces that these containers do not - # grant — every Turn came back with "bwrap: No permissions to create a new - # namespace" and an empty patch. The generic-cli seam lets the adapter - # launch Codex with the same approvals-and-sandbox bypass the other two - # arms use, so the loop stays LoopX's and the permissions stay identical. - # - # Which adapter is the only thing that differs between the Codex arm and the - # Claude Code arm: both go through this same generic-cli seam, the same - # validator and the same quota, and only the CLI that executes a Turn - # changes. Defaulting to the Codex one keeps that arm byte-identical to the - # runs already recorded against it. - wrapper = str(Path(__file__).resolve().parent / "codex_nosandbox_wrapper.py") - parser = argparse.ArgumentParser() - sub = parser.add_subparsers(dest="command") - register_turn_commands(sub, lambda p: p.add_argument("--format", default="json")) - args = parser.parse_args([ - "turn", "run-once", - "--goal-id", GOAL_ID, - "--agent-id", AGENT_ID, - "--turn-instance-id", f"{GOAL_ID}-turn-{turn_index}", - # codex-cli, not generic-cli. The generic host was an attempt to get - # around the sandbox flag, and it cost more than it saved: it carries - # its own scheduler contract, and eleven of sixteen turns died at - # "LoopX Turn route is not host executable" before any model work, with - # no route recorded to say why. The codex host is the one that - # demonstrably works — four turns, a 22 KB patch, f2p 31/35 — so keep it - # and neutralise the sandbox at the binary instead, via a wrapper that - # strips --sandbox and substitutes the approvals-and-sandbox bypass. - "--host", "codex-cli", - "--execution-mode", "isolated-headless", - "--project", str(project), - "--codex-bin", wrapper, - # An accepted value that the wrapper then removes; LoopX only permits - # read-only and workspace-write, and this argument has to satisfy its - # parser rather than the container. - "--codex-sandbox", "workspace-write", - "--codex-model", model, - "--validation-command-json", json.dumps(validator_command(project, base_sha)), - "--validation-timeout-seconds", "60", - # Without --execute LoopX plans the Turn and stops: every receipt comes - # back ok=True with dry_run=True, the host is never invoked, and four - # turns of nothing look exactly like four turns of success. - "--execute", - # Deliberately no --scan-root. It is not "where the work is"; LoopX - # documents it as "public files to scan for obvious private material", - # and pointing it at the task repository made LoopX run its public - # boundary scanner over the repository's own history. On the OPA task - # that produced - # public_boundary_violation CHANGELOG.md:3588: private_ip - # public_boundary_violation CHANGELOG.md:4859: credential - # which sets contract health to not-ok, which makes the quota decision - # `should_run=false / quota_skip`, which makes the route `wait`, which - # is what "LoopX Turn route is not host executable" finally reports — - # four layers away from the file it actually objected to. - # - # It also explains the shape of the failure: any repository whose text - # happens to contain something resembling an IP or a credential is - # rejected before a model runs, which is most of them, while the odd - # clean repository sails through and looks like proof the setup works. - # `turn plan` kept answering ready_for_host because the plan probe never - # passed --scan-root and so scanned LoopX's own directory instead. - # - # The default is LoopX's own public root, which is what the scanner is - # for. The task repository reaches the Turn through --project. - "--no-global-sync", - "--timeout-seconds", str(DEFAULT_TURN_TIMEOUT), - "--format", "json", - ]) - - # PrintPayload is (payload, fmt, renderer) -> None and FormatSelector is - # (...) -> str; passing single-argument lambdas made every Turn die with a - # TypeError before any model work, which the loop then dutifully repeated - # four times. Capture the payload instead of printing it, so the receipt - # survives whatever the CLI would have rendered. - captured: list[dict] = [] - - def _print_payload(payload, fmt=None, renderer=None): # noqa: ANN001 - if isinstance(payload, dict): - captured.append(payload) - - # `run-once` decided `route: wait`, which `_typed_route` returns only when - # the envelope says should_run is false. The decision behind it blamed - # "status or contract health is not ok" while reporting the quota itself as - # eligible with zero slots spent — so the gate is `goal_status_health_ok`, - # which reads `contract` and `global_registry` straight off the status - # payload. `turn plan`, run first in this same process, said - # ready_for_host, so the two calls are seeing different status. Spy on - # collect_status for both and record what differs; the two subcommands do - # not take the same options (`--scan-root` differs, `--no-global-sync` is - # run-once only), and that is the remaining candidate. - from loopx.cli_commands import turn as _turn_mod - _statuses: list[dict] = [] - _original_collect = _turn_mod.collect_status - - def _spy_collect(*a, **kw): # noqa: ANN002, ANN003 - result = _original_collect(*a, **kw) - if isinstance(result, dict): - contract = (result.get("contract") - if isinstance(result.get("contract"), dict) else {}) - registry = (result.get("global_registry") - if isinstance(result.get("global_registry"), dict) else {}) - _statuses.append({ - "scan_roots": [str(x) for x in (kw.get("scan_roots") or [])], - "status_ok": result.get("ok"), - "contract_ok": contract.get("ok"), - "has_error_diagnostics": "error_diagnostics" in contract, - "contract_errors": json.dumps( - contract.get("error_diagnostics"), ensure_ascii=False - )[:600], - "global_registry_ok": registry.get("ok"), - "global_registry_error": json.dumps( - {k: v for k, v in registry.items() if k != "goals"}, - ensure_ascii=False, - )[:400], - }) - return result - - _turn_mod.collect_status = _spy_collect - - # Plan first, and keep the result whatever happens next. When the planner - # declines to call the host, `run-once` raises "LoopX Turn route is not - # host executable" and the receipt it leaves carries only ok and effects — - # the route, the selected Todo and the scheduler context all vanish. Eleven - # turns failed exactly that way, and reproducing the call on the host proved - # only that the *arguments* were fine, so every explanation stayed a guess. - # `turn plan` runs the same decision without invoking a host or spending - # quota, so recording it costs nothing and makes the next failure legible. - plan_payload: dict = {} - try: - # Parse `turn plan` argv rather than copying the run-once Namespace and - # renaming the subcommand. The two subparsers do not define the same - # options, so the copy was missing `include_transaction_detail` and the - # probe failed on its own AttributeError — producing five nulls that - # said nothing about the route it was meant to explain. - plan_parser = argparse.ArgumentParser() - plan_sub = plan_parser.add_subparsers(dest="command") - register_turn_commands( - plan_sub, lambda p: p.add_argument("--format", default="json") - ) - plan_args = plan_parser.parse_args([ - "turn", "plan", - "--goal-id", GOAL_ID, - "--agent-id", AGENT_ID, - "--host", "codex-cli", - "--execution-mode", "isolated-headless", - "--format", "json", - ]) - with redirect_stdout(io.StringIO()): - handle_turn_command( - plan_args, - registry_path=registry, - runtime_root_arg=str(runtime), - output_format=lambda *_a, **_k: "json", - print_payload=_print_payload, - ) - if captured: - p = captured.pop() - route = p.get("route") or {} - ctx = p.get("scheduler_execution_context") or {} - plan_payload = { - "route_kind": route.get("kind"), - "would_invoke_host": route.get("would_invoke_host"), - "selected_todo": (route.get("selected_todo") or {}).get("todo_id"), - "context_valid": ctx.get("valid"), - "context_errors": ctx.get("errors"), - } - # Keep the raw shape when the expected keys are absent. Reusing the - # run-once Namespace for `plan` produced a payload without `route` - # at all, and five nulls said only "not what I expected" — which is - # the same dead end as having no diagnostics. - if route or ctx: - pass - else: - plan_payload["raw_keys"] = sorted(p) - plan_payload["raw"] = json.dumps(p, ensure_ascii=False)[:1200] - except Exception as exc: - plan_payload = {"plan_error": f"{type(exc).__name__}: {exc}"} - - # Record the plan `run-once` builds and then rejects. "LoopX Turn route is - # not host executable" is raised by build_loopx_turn_host_request after - # reading route.would_invoke_host off a payload run-once assembled moments - # earlier and never prints, so the one run whose route matters is the one - # nobody can see — and `turn plan`, which does print it, keeps answering - # ready_for_host. Both go through build_loopx_turn_plan, so wrapping it - # captures run-once's own payload, including the `session` block that only - # run-once populates. Reproducing this on the host was not possible: there - # both subcommands succeed, so the difference lives in the container. - from loopx.cli_commands import turn as _turn_mod - _built: list[dict] = [] - _original_build = _turn_mod.build_loopx_turn_plan - - def _spy_build(*a, **kw): # noqa: ANN002, ANN003 - result = _original_build(*a, **kw) - _built.append({"session_binding": kw.get("session_binding"), - "payload": result}) - return result - - _turn_mod.build_loopx_turn_plan = _spy_build - - # The route came back `wait`, which `_typed_route` only returns when the - # envelope says should_run is false and a quiet no-op is allowed — a - # scheduling verdict, not a contract error. `turn plan`, run moments - # earlier in this same process against this same registry, said - # ready_for_host. So capture the decision the envelope is projected from: - # should_run is set by build_live_quota_should_run_decision, and its - # rationale is the only thing that can say why the two disagree. - _decisions: list[dict] = [] - _original_decision = _turn_mod.build_live_quota_should_run_decision - - def _spy_decision(*a, **kw): # noqa: ANN002, ANN003 - result = _original_decision(*a, **kw) - if isinstance(result, dict): - _decisions.append(result) - return result - - _turn_mod.build_live_quota_should_run_decision = _spy_decision - - buffer = io.StringIO() - try: - with redirect_stdout(buffer): - code = handle_turn_command( - args, - registry_path=registry, - runtime_root_arg=str(runtime), - output_format=lambda *_a, **_k: "json", - print_payload=_print_payload, - ) - finally: - _turn_mod.build_loopx_turn_plan = _original_build - _turn_mod.build_live_quota_should_run_decision = _original_decision - _turn_mod.collect_status = _original_collect - built_payload: dict = {} - if _built: - record = _built[-1] - built = record["payload"] - route = built.get("route") if isinstance(built.get("route"), dict) else {} - session = built.get("session") if isinstance(built.get("session"), dict) else {} - ctx = built.get("scheduler_execution_context") - ctx = ctx if isinstance(ctx, dict) else {} - transaction = (built.get("transaction") - if isinstance(built.get("transaction"), dict) else {}) - built_payload = { - "route_kind": route.get("kind"), - "would_invoke_host": route.get("would_invoke_host"), - "route_reasons": route.get("reasons") or route.get("errors"), - "session_action": session.get("action"), - "session_binding_status": session.get("binding_status"), - "session_binding_arg": record["session_binding"], - "context_valid": ctx.get("valid"), - "context_errors": ctx.get("errors"), - "turn_key": transaction.get("turn_key"), - "builds": len(_built), - } - envelope = (built.get("turn_envelope") - if isinstance(built.get("turn_envelope"), dict) else {}) - action = (envelope.get("action") - if isinstance(envelope.get("action"), dict) else {}) - built_payload["envelope"] = { - "should_run": envelope.get("should_run"), - "effective_action": envelope.get("effective_action"), - "delivery_allowed": action.get("delivery_allowed"), - "must_attempt": action.get("must_attempt"), - "quiet_noop_allowed": action.get("quiet_noop_allowed"), - } - if _decisions: - decision = _decisions[-1] - built_payload["decision"] = { - key: decision.get(key) - for key in ("should_run", "effective_action", "reason", "reasons", - "blocked_reason", "quota", "quota_state", "cadence", - "schedule", "gates") - if key in decision - } - built_payload["decision_keys"] = sorted(decision)[:40] - built_payload["decisions"] = len(_decisions) - # Two entries: the `plan` probe's status, then run-once's. Whatever differs - # between them is what flipped goal_status_health_ok. - built_payload["statuses"] = _statuses[:4] - if captured: - payload = captured[-1] - else: - raw = buffer.getvalue().strip() - try: - payload = json.loads(raw.splitlines()[-1]) if raw else {} - except Exception: - payload = {"unparsed": raw[-2000:]} - payload["_exit_code"] = code - payload["_plan"] = plan_payload - payload["_built"] = built_payload - # Carry the wrapper's log into the receipt. LoopX reports a failed host as - # `codex_cli_exit_nonzero` and keeps nothing else, and the container that - # holds the log is deleted as soon as the task ends, so the receipt is the - # only artifact that outlives the evidence. - codex_log = Path( - os.environ.get("MR_LOOPX_CODEX_LOG", "/tmp/loopx-goal/codex-wrapper.log") - ) - try: - payload["_codex_log"] = codex_log.read_text(encoding="utf-8")[-4000:] - codex_log.unlink() - except OSError: - payload["_codex_log"] = None - return payload - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--project", required=True) - parser.add_argument("--task-file", required=True) - parser.add_argument("--runtime-root", required=True) - parser.add_argument("--codex-bin", default="codex") - parser.add_argument("--adapter", default="loopx_codex_adapter.py", - help="generic-cli host adapter that executes one Turn " - "(loopx_codex_adapter.py | loopx_claude_adapter.py)") - parser.add_argument("--model", required=True) - parser.add_argument("--sandbox", default="danger-full-access") - parser.add_argument("--quota", type=int, default=DEFAULT_QUOTA) - args = parser.parse_args() - - project = Path(args.project).resolve() - runtime = Path(args.runtime_root) - runtime.mkdir(parents=True, exist_ok=True) - instruction = Path(args.task_file).read_text(encoding="utf-8").strip() - - # The host adapter needs the task text too, and cannot get it from the Turn - # envelope: the envelope carries only the selected Todo, and LoopX compacts - # that to an 8 KB budget, so the staged Todo's wording survives truncated - # ("Advance the task below by exactly one stage per Turn, and report which - # s...") while the `## Task` section of the goal document is never included - # at all. An adapter working from the envelope alone therefore sees a - # staging meta-instruction with no target behaviour attached, and a - # well-behaved model correctly refuses to invent one — four Turns of - # `validation_failed` with an empty patch, which reads like a model that - # could not do the work rather than a prompt that never described it. - # Handing the adapter the same file the goal document was built from keeps - # one source of truth for the task text. - os.environ["MR_LOOPX_TASK_FILE"] = str(Path(args.task_file).resolve()) - - hide_loopx_state_from_git(project) - state = stage_goal_state(project, instruction) - registry = stage_registry(project, runtime, state) - base_sha = head_sha(project) - - receipts = [] - for turn_index in range(1, args.quota + 1): - try: - payload = run_turn( - project=project, registry=registry, runtime=runtime, - codex_bin=args.codex_bin, model=args.model, - sandbox=args.sandbox, turn_index=turn_index, base_sha=base_sha, - - ) - except Exception as exc: # keep the receipt; a dead turn is evidence too - payload = {"error": f"{type(exc).__name__}: {exc}"} - receipts.append(payload) - # A Turn that never reached the host is a defect in this runner, not a - # result: repeating it burns the whole quota on the same traceback, as - # four identical TypeErrors once did. Stop and let the receipt show why. - if "error" in payload: - break - if payload.get("dry_run"): - payload["_fatal"] = "dry_run: --execute was not honoured" - break - status = str(payload.get("status") or payload.get("result_kind") or "") - # Stop early only when LoopX says the work is settled; a failed or - # repair-required Turn is exactly the case the next Turn exists for. - if status in {"completed", "goal_complete", "done"}: - break - - print(json.dumps({ - "schema": "deepswe_loopx_turn_log_v0", - "quota": args.quota, - "turns_run": len(receipts), - "receipts": receipts, - }, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmark/swe-marathon/skills/swe-marathon-five-arm/SKILL.md b/benchmark/swe-marathon/skills/swe-marathon-five-arm/SKILL.md index 65936205c7..cf680b08ec 100644 --- a/benchmark/swe-marathon/skills/swe-marathon-five-arm/SKILL.md +++ b/benchmark/swe-marathon/skills/swe-marathon-five-arm/SKILL.md @@ -1,305 +1,19 @@ --- name: swe-marathon-five-arm -description: 在 SWE-Marathon 上做 codex harness 五臂对照（裸 codex / 原生 /goal / LoopX 三模式）。包含 agent 适配、跑法、监控，以及一整天踩出来的坑——这些坑的共同特征是**退出码 0、日志干净、结果看着正常**，不看这份文档几乎必然重踩。 +description: Migrate historical SWE-Marathon agent configurations to the shared Codex runtime. --- -# SWE-Marathon 五臂对照 +# SWE-Marathon shared execution -对照维度是 **harness**，不是模型。五条臂的模型、effort、工具面、沙箱、容器完全一致，唯一变量是"怎么驱动 codex"。 +Read `../../runtime/RUNTIME.md` and `../../../runtime/RUNTIME.md` before preparing +a new study. Use the shared Harbor adapter and its explicit execution_mode and +iteration_context, preserving native tasks, verifier policy and scoring. -## 五条臂 +The historical five-arm WEN framework is retired. Do not run its assisted +unblock or sandbox-replacement wrappers, reinterpret withdrawn results, or +invent an independent validator from git status. Existing study data and +withdrawal notices remain tied to their original source revision. -| 臂 | agent 类 | 是什么 | runtime profile | -|---|---|---|---| -| `plain` | `codex_plain_appserver:CodexPlainAppServer` | 裸 codex，不挂 Goal、不装 LoopX | — | -| `goal` | `codex_goal_agent:CodexGoalAgent` | codex 原生 `/goal`，不装 LoopX | — | -| `ssh-goal` | `codex_loopx_agent:CodexLoopxAgent` | LoopX 模式二：Codex App over SSH | `codex_app_ssh_goal --begin-turn` | -| `codex-cli` | 同上，`WEN_MODE=codex-cli` | LoopX 模式三：Codex CLI 可见 `/goal` | `codex_cli` | -| `heartbeat` | 同上，`WEN_MODE=heartbeat` | LoopX 模式一：心跳自动化 | `generic_cli --turn-instance-id` | - -三个 LoopX 臂共用一个 agent 类，靠 `WEN_MODE` 选 `modes/profiles.py` 里的 Mode。**模式差异是数据不是分支**——body 文本、runtime profile、是否需要 turn instance 都写在 Mode 里。 - -`ssh-goal` 与 `codex-cli` 的 goal body 逐字只差一行（runtime profile 那行），这符合上游设计：分流发生在 LoopX CLI 的闸门里，不在 body 文本里。 - -## 必须披露的两处偏离 - -1. **`plain` 走 app-server，不是论文的 `codex exec`。** 原因见下面「只杀对照组」那节。 -2. **`heartbeat` 用 `generic_cli` 代替 `codex_app`。** 上游 `host_surface=codex_app` 是留给真 Codex App 的：它回一套 `stateful_backoff`，期待宿主调 App 的 `automation_update` 改 RRULE 再 ACK；没有真 App 就永远悬着。上游给自建定时器指定的对口就是 `generic_cli`（参考实现 `scripts/external_scheduler_worker.py` 的默认值）。这个替换写在 `profiles.py` 的 `substitution` 字段里，**但不会自动进产物，报表时要手工带上**。 - -## 跑法 - -```bash -# 环境验收（install-only，不烧 token） -./scripts/verify_envs.sh - -# 超时路径冒烟（~10 分钟跑完五臂，验证预算耗尽后能正常收尾评分） -./scripts/canary_timeout.sh - -# 全量 -MARATHON_CONCURRENCY=12 GOAL_TIMEOUT_SEC=21600 MARATHON_AGENT_TIMEOUT_MULT=1.0 \ - setsid nohup ./scripts/marathon_all.sh >> marathon-full.log 2>&1 < /dev/null & - -# 监控（进度 + 逐臂 reward + receipt 的续跑/解锁/错误事件 + 连续分） -./scripts/progress.sh -./scripts/health_check.sh [--fix] -``` - -常用旋钮（都可放进 `marathon-full/.driver_env`，`health_check --fix` 重新拉起时会读）： - -| 变量 | 作用 | -|---|---| -| `GOAL_TIMEOUT_SEC` | **我们自己的死线**，默认 5340（89 分钟）。多数任务是它先到，不是 harbor | -| `MARATHON_AGENT_TIMEOUT_MULT` | harbor 侧 = 任务声明预算 × 它 | -| `MARATHON_VERIFIER_MULT` | verifier 超时倍率，默认 4 | -| `MARATHON_CONCURRENCY` | 跨 (任务,臂) 的并发 | -| `MARATHON_ARM_MAJOR` | 1=按臂分组，0=按任务分组 | -| `MARATHON_TASKS` / `MARATHON_ARMS` | 覆盖任务集/臂集与顺序 | -| `MARATHON_NET_CAP` / `NET_MARGIN` | docker 网段池闸门 | -| `MARATHON_MAX_PASSES` | 多趟扫描次数，捡漏延迟失败 | - -## 环境准备（跑之前必须做完的三步） - -### 1. `stage_codex_offline.sh` —— 搬 codex 运行时 - -`@openai/codex-linux-x64` 这个 npm 包里 vendor 的是 **static-pie musl 静态可执行文件**，不需要 Node 运行时，断网空白容器里直接能跑。 - -**必须整棵拷,不能只抠 `codex` 和 `rg`**：0.151 的 `unified_exec` 需要 `codex-code-mode-host` 这个 sidecar。缺了它容器里**每次工具调用都失败,而且不报错**——表现是 47 轮什么都没干成、零错误上报。 - -### 2. `stage_local.sh` —— 把重资产搬到本地 NVMe - -`wen/` 在 NFS 上（`:/`，nfs v3）。docker 数据根在本地 NVMe，所以**镜像层不慢**，慢的是每个 trial 往容器里搬的东西： - -``` -codex 二进制 331M NFS 读 -LoopX 源码 178M NFS 读（整个 git checkout） -node（整个 nvm）721M 本地，但含 npm/lib/include，其实只要 bin/node -可移植 python 109M 本地 - ───── -LoopX 臂每次 ~1.3G，其中 ~509M 走 NFS —— 12 路并发时这是主要瓶颈 -``` - -预装到 `$HOME/wen-cache` 后全部本地，node 裁到只剩二进制（721M→119M），每 trial 降到约 400M。 - -**只搬 agent 自己的依赖，`swe-marathon/tasks` 一个字节不动**——那是 benchmark 数据，`dataset.toml` 里有 sha256 digest，改了就不是这个 benchmark。 - -### 3. `prebuild_images.sh` —— 预构建任务镜像 - -任务 Dockerfile 里有 `apt-get update && install`，构建要出网。本机出网必须过 ``，而 **dockerd 自己钉的代理 `` 是死的**，症状： - -``` -W: Failed to fetch http://deb.debian.org/... connection timed out -failed to solve: process "/bin/sh -c apt-get update ..." did not complete -``` - -**不能用 `~/.docker/config.json` 的 proxies 段**：它会把 `HTTP_PROXY` **同时注入运行时容器**，等于给 `no-network` 任务开了出口，公平性直接没了。 - -正确做法是只在**构建**时经 `--build-arg` 给代理 + `--network host`。BuildKit 层缓存按内容寻址，预构建产生的层 harbor 之后照样命中，而 harbor 自己那次构建不带代理、运行时容器干净。 - -`wen/.venv` 里改过 harbor 两个文件（自己的副本，允许改）： - -- `harbor/environments/docker/docker-compose-build.yaml` —— 加 `network: host` + proxy build-args。**两边 build-arg 必须一致**，不一致会导致缓存不命中、白构建一遍 -- `.../harbor-docker-egress-control-sidecar/Dockerfile` —— 去掉 digest pin - -## LoopX profile 怎么装进容器 - -布局**照抄 `benchmark_toolkit` 的 `_profile_paths()`**，这样 LoopX 自己的 `inspect` / `doctor` 逻辑才对得上： - -``` -/opt/loopx-src LoopX 源码（docker cp 进去） -/opt/loopx-py 可移植 python -/opt/loopx-node 可移植 node -/opt/lxprofile/ profile 根 - bin/loopx install-local.sh 生成的 wrapper - codex-home/ CODEX_HOME（config.toml、skills/） - registry.json - runtime/ -``` - -LoopX `dependencies = []`，零第三方依赖，`install-local.sh` 只拷文件 + 生成 wrapper，所以断网容器里装得起来。 - -**node 必须在 PATH 上**：`doctor` 用 `shutil.which("node")` 找它，TS control plane 要 node ≥ 22.6。装配和后续所有 CLI 调用都用这套环境： - -``` -PATH=/opt/loopx-node/bin:/opt/lxprofile/bin:/usr/local/bin:/usr/bin:/bin -HOME=/opt/lxprofile/home CODEX_HOME=/opt/lxprofile/codex-home -LOOPX_PYTHON=/opt/loopx-py/bin/python3 -``` - -（**查容器内状态时也要用这套**，否则 `loopx status` 会报 `status_collection_failed`，看着像 LoopX 坏了，其实是你没给 PATH。） - -### 三道门禁，任何一道不过就硬失败 - -1. **技能齐全**：`_REQUIRED_SKILLS` 从上游常量 `NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS` 读，**不要硬编码**——0.5.3 是 7 个，历史值是 6 个，硬编码会漏掉 `loopx-benchmark` -2. **`doctor` 通过**：`loopx --format json doctor --agent-type codex-app-ssh` -3. **`skills/list` 发现**：`native_codex_goal` 在 `thread/start` **之前**发 `skills/list`，codex 没真的发现那些 skill 就直接失败，**一个 token 都不花**——这是最省钱的门禁 - -装配成功的日志长这样：`LoopX profile 就绪（7 skills + CLI + doctor ok）` - -### goal body 渲染 - -`render_native_codex_goal_prompt` 产出的 body 里带 `$HOME/.codex/loopx/registry.global.json` 占位符，**必须替换成真实 registry 路径，替换后还要验证占位符确实消失**。 - -日志：`LoopX goal body 已渲染（2953 字符，goal_id=lhtb-goal，ungated=True）`。字符数按模式不同（heartbeat 1682 / codex-cli 2931 / ssh-goal 2953），这是模式确实分开渲染的证据。 - -### 运行期状态写在仓库里 - -`/app/.codex/goals//ACTIVE_GOAL_STATE.md`，Agent Todo 带完整元数据： - -```markdown -- [x] [P1] Identify the fastest validation command from Cargo.toml ... - -``` - -这是 LoopX "跨 turn 不丢上下文"的落地形式：自举出候选待办 → 带证据完成 → 派生后继。实测 LoopX CLI 调用占工具调用的 12–37%。 - -**注意**：`agent/` 目录保存了 sessions / goals sqlite / trajectory，但**没有保存 `registry.json`**，所以已完成 trial 的最终 todo 状态查不到，只能对在跑的容器采样。要在报表里给这个，得把 registry 加进 `_save_sessions` 的拷贝清单。 - -### 一个装配期的坑 - -`pip install -e` 会在源码树里留下 `loopx.egg-info`，`docker cp` 时被一起带进容器，污染 profile。改用非编辑安装 + 容器侧清理。 - -## 坑（按危险程度排） - -### 一、"只杀对照组"的偏向性 —— 会得出**反向结论** - -同一个故障如果只打某一条臂，最终会显示"另一条臂更强"，而那是基础设施差异。已发现四次： - -1. **TPM 限流**只杀 `plain`（裸 codex 上下文线性膨胀，单请求 116k token）→ 降并发 + 压预算 -2. **HTTP 400 `internal_chat_message_metadata_passthrough`** 只打 `plain`（只在 `codex exec` 路径出现）→ `scripts/param_strip_proxy.py` 剥字段 -3. **传输通道不同**：`plain` 走 SSE，其余四臂走 app-server，SSE 在 routify 上 2/2 全错 → 新增 `codex_plain_appserver.py`，plain 也走 app-server -4. **异常类分裂**（下条单列） - -**判据**：任何时候看到某条臂的失败率显著高于其他臂，先假设是基础设施，不要先假设是能力。 - -### 二、同名异常类分裂 —— 整晚三条臂颗粒无收 - -`NativeGoalProtocolError` 存在两份： - -``` -agents/native_codex_goal.py ← 符号链接到 wen/loopx 源码树 -loopx.capabilities.benchmark_toolkit.native_codex_goal ← 装在 .venv -``` - -互不为子类。`run()` 里按**类**捕获 `goal_timeout_before_terminal` 的后果：只有 `goal` 臂的超时被吞掉并正常评分，LoopX 三臂的超时逃到 harbor 被当基础设施故障 → 重试 → 记 errored，几小时的真实进度全丢。 - -**修法**：按**消息**判定（两个类都继承 `RuntimeError`）。 - -```python -except RuntimeError as exc: - if str(exc) != "goal_timeout_before_terminal": - raise -``` - -超时是长程任务的**正常预算耗尽**，五臂必须一视同仁按部分进度评分。 - -### 三、二值 reward 在紧预算下没有区分度 - -SWE-Marathon 的 `reward` 是二值的（全部测试通过才给 1）。实测：**撞死线的 23 条 trial 无一得分，自己收尾的 14 条中 11 条得分**——决定分数的是"任务能不能在预算内做完"，不是哪条臂。 - -连续分在 `metrics.json` 的 `partial_score`（任务官方定义，[0,1]）。用 `scripts/_partial.py` 提取。两个坑： - -- **字段名不统一**：有的任务只写 `pass_rate` -- **不是所有任务都真连续**：`s3-clone` 有 correctness+ux 两阶段，后一阶段按 `0.5×unit + 0.5×cua` 改写 `partial_score`，而两个子分各自二值 → 22 个 gate 过 16 个、pytest 87.7%，`partial_score` 仍是 0.0 -- **Rust 任务有构建门禁**：`BUILD FAILED` → `partial_score` 直接归零。同一条臂早一分钟或晚一分钟被切断，分数可能在 0 和 0.98 之间跳变 - -所以报表要**两个口径并排**（partial_score + 测试通过率），且把 `build_failed` 单独标注。 - -### 四、`BUILD FAILED` 不是环境故障 - -它发生在 **verifier 阶段**（agent 早跑完了），是 `cargo` 编译 **agent 写的代码**。典型报错是"缺东西"而不是"写错了"： - -``` -can't find lib `xxx` at path `src/lib.rs` Cargo.toml 声明了但没建文件 -error[E0432]: unresolved imports ... import 了还没定义的类型 -couldn't read `src/client.rs`: No such file 测试引用的源文件不存在 -``` - -排除环境嫌疑的三个交叉验证：同环境下别的臂构建成功；四条臂报错在四个不同 crate；网络类关键词（`failed to download` / `registry` / `no matching package`）命中 0 次。 - -harbor 记 `n_errored_trials=0`，**不作废**。对比真正的环境故障： - -``` -Docker compose command failed ... all predefined address pools have been fully subnetted -``` -那种记 `n_errored_trials=1`，**要作废重跑**。 - -### 五、docker 网段池 —— 一次废掉 12 条 - -默认地址池只支持约 32 个 bridge 网络（一个 RFC 1918 私网池切成 16 个子网，另一个私网池再切成 16 个子网）。**机器是共用的**，别人常年占约 15 个，每个 trial 一个 compose 网络。撑爆后新 trial 直接死在环境启动。 - -泄漏源常常是自己：**`docker rm -f` 只删容器，不删 compose 网络**。清理容器时必须连网络一起清。 - -`marathon_all.sh` 有起跑前闸门（`MARATHON_NET_CAP/NET_MARGIN`），等待时顺手回收空网络。扩池要改 `daemon.json` + 重启 dockerd，**会杀掉别人所有容器**，需要本人操作并先打招呼。 - -### 六、清理进程：绝不用模糊匹配（已犯五次） - -`pkill -f` / `pgrep -f` / `case` 匹配 `/proc/cmdline` 都会命中**调用者自己的 shell**——因为命令文本里就含那些字样。第五次的表现是 TERM 发给自己、shell 在清容器之前就死（退出码 144）。 - -正确写法：模式用字符串拼接构造，让命令行里不出现完整字面量，再显式排除 `$$` 和 `$PPID`： - -```bash -PAT=$'marathon''_all.sh'; ME=$$; PA=$PPID -for p in /proc/[0-9]*; do pid=${p#/proc/} - [ "$pid" = "$ME" ] && continue; [ "$pid" = "$PA" ] && continue - cl=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null) || continue - case "$cl" in *"$PAT"*) echo "$pid" >> /tmp/kill.txt;; esac -done -xargs -r kill -TERM < /tmp/kill.txt -``` - -另：`ps -eo` 里的 `-e` 会**覆盖** `-p`，`ps -eo pid= -p 123` 会列出全机进程。 - -### 七、监控工具本身出错比 trial 失败更危险 - -一天里监控工具错了七次，每次都表现为"看起来一切正常"： - -| 错误 | 后果 | -|---|---| -| `health_check.sh` 扫 `marathon-jobs*` 而产物在 `marathon-full` | 整晚报 OK，实际三条臂全挂 | -| `find -mindepth 4` 而 job 级 `result.json` 在深度 5 | 永远查不到错误跑次 | -| `_summarize.py` 用 `glob("*/*")` 太浅 + 目录 mtime 排序 | 「0 完成 0 错误」而实际已有失败 | -| `_partial.py` 用 `m.parts[1]`，绝对路径下是 `mnt` | 异常被 `2>/dev/null` 吞掉，整段连续分静默消失 | -| `grep -c` 计数为 0 时退出码 1，`\|\| echo 0` 追加第二个 0 | 变量含换行，监控输出被拆成三行 | -| 计数不按「本次跑次」切分 | 跨重启累加，显示 37/90 而当次只排了 10 | -| `_receipts.py` 读 `unblock_count`（真名 `_unblock_count`，带下划线） | 五臂全返回 None，会永远报"解锁 0 次" | - -**教训**：监控工具要和被测对象一样对待——先验证它真能看见东西。路径写错的监控比没有监控更坏，因为它每轮都报"正常"。 - -### 八、别拿环境不同的数据下结论 - -- **臂均值不可比**：各臂跑过的任务集不同。`ssh-goal` 均值低是因为它多跑了几个难任务。只用**双方都跑过的格**做配对比较 -- **遗留目录要排除**：root 属主的旧跑次目录（`marathon_run.sh` 写不进去会另起 `-`），里面的 receipt 是上一轮的。`_receipts.py` 按臂目录属主排除 -- **半成品 metrics 要排除**：正在重跑的 trial 也留 `metrics.json`，内容是 `phase: initialized`、partial 0.0，采信会**凭空造出一个 0 分** -- **查容器内状态要复现 agent 的环境**：`docker exec` 裸奔没有 PATH，`loopx doctor` 找不到 node 会报 `status_collection_failed`，看着像 LoopX 坏了。正确环境是 - `PATH=/opt/loopx-node/bin:/opt/lxprofile/bin:... HOME=/opt/lxprofile/home CODEX_HOME=/opt/lxprofile/codex-home` - -### 九、其他单点 - -- **codex 缺 `codex-code-mode-host` sidecar**：容器里每次工具调用都失败，不报错、只是零产出。`stage_codex_offline.sh` 要镜像整个 vendor 树 -- **app-server 需要 `sandbox_mode = "danger-full-access"` + `projects."/app" = { trust_level = "trusted" }"`**，否则 bwrap 缺用户命名空间，`initialize` 握手就失败 -- **`config.toml` 必须一次性写完**：TOML 里 table 头之后的裸键都归该 table，分两次 `cat >>` 会让 `web_search` 变成 `model_providers.harbor.web_search` -- **LoopX 自锁**：goal body 规定连续三轮相同阻塞就 `update_goal status=blocked`，只有 user `/goal resume` 能复活——benchmark 里没有 user。`LOOPX_UNGATED=1` 让 harness 扮演 operator，解锁次数记进 receipt 的 `_unblock_count` -- **NAS 输出盘的 mtime 比宿主机慢约 2 分钟**，别用宿主机时间 `find -newermt` 筛产物 -- **`$$` 在 bash 子 shell 里是父进程 PID**，认领文件要用 `$BASHPID` - -## 产物与判据 - -``` -marathon-full/////result.json job 级，含 stats - //verifier/ reward.txt / metrics.json / test-stdout.txt - /agent/ trajectory.json / goal_receipt.json / sessions/ -marathon-full/.claims/__.claim 在跑认领（内容是 BASHPID） -marathon-full/.driver_env 驱动参数，health_check --fix 会读 -marathon-voided/ 作废的 trial，带 README 说明原因 -``` - -"成功"的统一判据（`scripts/_is_done.py`，断点续跑和监控共用，避免两处判据打架）： - -``` -n_completed_trials >= 1 且 n_errored_trials == 0 -``` - -`errors > 0` **不算跑过**，必须重跑——那多半是基础设施故障。 +Validate the chosen native environment with a bounded task before a full run. +Model jobs, verifier access and publication require the caller's authorization; +loading this migration guide grants none of those actions. diff --git a/benchmark/tests/test_native_codex_goal.py b/benchmark/tests/test_native_codex_goal.py index 440243d8e6..7750d6343e 100644 --- a/benchmark/tests/test_native_codex_goal.py +++ b/benchmark/tests/test_native_codex_goal.py @@ -314,6 +314,7 @@ def test_goal_runtime_waits_for_automatic_continuation_until_terminal() -> None: def test_goal_runtime_exposes_typed_deadline_when_active_goal_never_continues() -> None: transport = ContinuationTransport(terminal_after_second_turn=False) transport.events = transport.events[:2] + observed = [] with pytest.raises( NativeGoalDeadlineExceeded, @@ -323,7 +324,11 @@ def test_goal_runtime_exposes_typed_deadline_when_active_goal_never_continues() transport, _config(), timeout_sec=0.01, + on_turn_started=observed.append, ) + assert len(observed) == 1 + assert observed[0].turn_completed_count == 1 + assert observed[0].goal_status == "active" def _write_fake_app_server(path: Path) -> None: diff --git a/benchmark/tests/test_shared_codex_runtime.py b/benchmark/tests/test_shared_codex_runtime.py new file mode 100644 index 0000000000..144797ff55 --- /dev/null +++ b/benchmark/tests/test_shared_codex_runtime.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +import json +import asyncio +import os +import shlex +import subprocess +import sys +import time +import tomllib +from types import SimpleNamespace +from pathlib import Path + +import pytest + +from benchmark.runtime.codex import Execution, prepare_codex_home +from benchmark.runtime.worker import run_once, turn_command + + +def settings(tmp_path, **changes): + skills = tmp_path / "installed-skills" + skills.mkdir(exist_ok=True) + return ( + dict( + home=tmp_path / "home", + execution=Execution(), + workspace=tmp_path, + model="fixture-model", + effort="high", + base_url="http://localhost:8123/v1", + api_key="fixture-key", + wire_api="responses", + skills=skills, + ) + | changes + ) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"mode": "unknown"}, + {"context": "resume-if-available"}, + {"mode": "turn"}, + {"timeout_seconds": float("nan")}, + {"validation_command": ("true",)}, + {"mode": "turn", "validation_command": "true"}, + {"mode": "turn", "validation_command": {"command": "true"}}, + {"sandbox": "unrecognised"}, + ], +) +def test_invalid_execution_rejected_before_host(kwargs): + with pytest.raises(ValueError): + Execution(**kwargs) + + +def test_trial_home_preserves_sessions_and_fixes_nonconversation_inputs(tmp_path): + args = settings(tmp_path) + prepare_codex_home(**args) + session = args["home"] / "sessions" / "old.jsonl" + session.parent.mkdir() + session.write_text("history\n") + prepare_codex_home(**args) + assert session.read_text() == "history\n" + config = tomllib.loads((args["home"] / "config.toml").read_text()) + assert config["features"]["goals"] is False + assert config["memories"] == {"generate_memories": False, "use_memories": False} + assert config["model_providers"]["harbor"]["wire_api"] == "responses" + assert not (args["home"] / "auth.json").exists() + with pytest.raises(ValueError, match="settings changed"): + prepare_codex_home(**(args | {"effort": "medium"})) + + +def test_baseline_isolated_from_loopx_skills(tmp_path): + args = settings(tmp_path, execution=Execution(mode="native-goal"), skills=None) + prepare_codex_home(**args) + assert not (args["home"] / "skills").exists() + assert tomllib.loads((args["home"] / "config.toml").read_text())["features"][ + "goals" + ] + (args["home"] / "skills").mkdir() + with pytest.raises(ValueError, match="baseline"): + prepare_codex_home(**args) + + +def worker_env(tmp_path): + task = tmp_path / "task.md" + task.write_text("Synthetic task.\n") + binary = tmp_path / "codex" + binary.write_text( + f"#!{sys.executable}\n" + + """ +import json, os, pathlib, sys +home = pathlib.Path(os.environ["CODEX_HOME"]) +sessions = home / "sessions" +sessions.mkdir(exist_ok=True) +number = len(list(sessions.iterdir())) +(sessions / f"session-{number}.jsonl").write_text("{}\\n") +print(json.dumps({"argv": sys.argv[1:], "home": str(home)})) +sys.stdin.read() +""" + ) + binary.chmod(0o755) + return dict(os.environ) | { + "LOOPX_EXECUTION_MODE": "plain", + "LOOPX_PROJECT": str(tmp_path), + "LOOPX_TASK_DOC": str(task), + "LOOPX_CODEX_HOME": str(tmp_path / "home"), + "LOOPX_WAKE_LOG_DIR": str(tmp_path / "logs" / "wakes"), + "CODEX_BIN": str(binary), + "MODEL_NAME": "fixture", + "REASONING_EFFORT": "high", + "OPENAI_BASE_URL": "http://localhost:8123/v1", + "OPENAI_API_KEY": "fixture-key", + } + + +def test_fresh_wakes_share_environment_without_resuming_or_duplicate_session_copies( + tmp_path, +): + env = worker_env(tmp_path) + for _ in range(2): + assert run_once(env)["ok"] + logs = tmp_path / "logs" + assert len(list((logs / "sessions").glob("*.jsonl"))) == 2 + calls = [json.loads(p.read_text()) for p in (logs / "wakes").glob("*/stdout.jsonl")] + assert len({p["home"] for p in calls}) == 1 + assert all("resume" not in p["argv"] for p in calls) + assert not list((logs / "wakes").glob("*/sessions")) + + +def test_process_failure_is_not_reported_as_success(tmp_path): + env = worker_env(tmp_path) + Path(env["CODEX_BIN"]).write_text(f"#!{sys.executable}\nraise SystemExit(7)\n") + receipt = run_once(env) + assert receipt["ok"] is False + assert receipt["return_code"] == 7 + + +def test_timeout_keeps_receipt_and_reaps_child(tmp_path): + env = worker_env(tmp_path) + env["LOOPX_CODEX_TURN_TIMEOUT_SEC"] = "1" + Path(env["CODEX_BIN"]).write_text( + f"#!{sys.executable}\n" + + """ +import os,pathlib,time +pathlib.Path("pid").write_text(str(os.getpid())) +time.sleep(60) +""" + ) + receipt = run_once(env) + assert receipt["timed_out"] and not receipt["ok"] + with pytest.raises(ProcessLookupError): + os.kill(int((tmp_path / "pid").read_text()), 0) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX scheduler cancellation") +def test_scheduler_cancel_reaps_detached_worker_host_and_keeps_receipt(tmp_path): + env = worker_env(tmp_path) + source = Path(__file__).resolve().parents[2] + env["PYTHONPATH"] = str(source) + Path(env["CODEX_BIN"]).write_text( + f"#!{sys.executable}\n" + "import os,pathlib,signal,time\n" + "signal.signal(signal.SIGINT, signal.SIG_IGN)\n" + "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "pathlib.Path('pid').write_text(str(os.getpid()))\n" + "time.sleep(60)\n" + ) + cli = tmp_path / "quota" + payload = { + "should_run": True, + "effective_action": "run_now", + "scheduler_hint": { + "action": "run_now", + "cadence_class": "active_work", + "reason": "fixture", + "reset_policy": {"reset_token": "fixture"}, + "cold_path_detail": { + "local_scheduler": { + "recommended_interval_minutes": 1, + "example_progression_minutes": [1], + "unchanged_poll_limit": None, + "after_limit": "continue", + } + }, + }, + } + cli.write_text(f"#!{sys.executable}\nprint({json.dumps(payload)!r})\n") + cli.chmod(0o755) + command = [ + sys.executable, + str(source / "scripts/external_scheduler_worker.py"), + "--cli-bin", + str(cli), + "--goal-id", + "fixture", + "--agent-id", + "fixture", + "--state-file", + str(tmp_path / "scheduler.json"), + "--wake-cmd", + "exec " + shlex.join([sys.executable, "-m", "benchmark.runtime.worker"]), + ] + process = subprocess.Popen( + command, + env=env, + cwd=tmp_path, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 10 + while not (tmp_path / "pid").exists() and time.monotonic() < deadline: + time.sleep(0.05) + assert (tmp_path / "pid").exists() + process.terminate() + process.wait(timeout=15) + with pytest.raises(ProcessLookupError): + os.kill(int((tmp_path / "pid").read_text()), 0) + receipt = json.loads( + next((tmp_path / "logs/wakes").glob("*/receipt.json")).read_text() + ) + assert receipt["error_kind"] == "KeyboardInterrupt" + assert not receipt["ok"] + finally: + if process.poll() is None: + process.kill() + process.wait() + + +def test_turn_uses_public_cli_and_core_session_policy(tmp_path): + env = worker_env(tmp_path) | { + "LOOPX_CLI": "loopx", + "LOOPX_REGISTRY": "registry.json", + "LOOPX_RUNTIME_ROOT": "runtime", + "LOOPX_GOAL_ID": "goal", + "LOOPX_AGENT_ID": "agent", + } + execution = Execution( + mode="turn", + context="resume-if-available", + validation_command=("python", "trusted-validator.py"), + ) + command = turn_command(env, execution, "wake-fixture") + from loopx.cli import build_parser + + parsed = build_parser().parse_args(command[1:]) + assert parsed.iteration_context == "resume-if-available" + assert parsed.validation_command_json == '["python", "trusted-validator.py"]' + assert parsed.codex_sandbox == "danger-full-access" + + +def test_failed_turn_restarts_same_transaction_until_core_recovers(tmp_path): + env = worker_env(tmp_path) + skills = tmp_path / "skills" + skills.mkdir() + cli = tmp_path / "loopx" + cli.write_text( + f"#!{sys.executable}\n" + + """ +import json, pathlib, sys +log = pathlib.Path("calls.jsonl") +first = not log.exists() +with log.open("a") as output: + output.write(json.dumps(sys.argv[1:]) + "\\n") +print(json.dumps({"ok": not first, "resume_turn_key": "sha256:" + "a" * 64})) +sys.exit(1 if first else 0) +""" + ) + cli.chmod(0o755) + env.update( + LOOPX_EXECUTION_MODE="turn", + LOOPX_CLI=str(cli), + LOOPX_SHARED_SKILLS=str(skills), + LOOPX_REGISTRY=str(tmp_path / "registry.json"), + LOOPX_RUNTIME_ROOT=str(tmp_path / "runtime"), + LOOPX_GOAL_ID="fixture-goal", + LOOPX_AGENT_ID="fixture-agent", + LOOPX_VALIDATION_COMMAND_JSON='["trusted-validator"]', + ) + assert not run_once(env)["ok"] + pending = tmp_path / "runtime/benchmark-pending-turn.json" + assert pending.exists() + assert run_once(env)["ok"] + assert not pending.exists() + assert run_once(env)["ok"] + calls = [ + json.loads(line) for line in (tmp_path / "calls.jsonl").read_text().splitlines() + ] + first, recovery, successor = calls + assert "--turn-instance-id" in first + assert "--turn-instance-id" not in recovery + assert "--retry-failed-turn" in recovery + assert recovery[recovery.index("--resume-turn-key") + 1] == "sha256:" + "a" * 64 + assert ( + successor[successor.index("--turn-instance-id") + 1] + != first[first.index("--turn-instance-id") + 1] + ) + + +def test_harbor_imports_and_keeps_native_sessions_separate(tmp_path, monkeypatch): + pytest.importorskip("harbor") + from benchmark.runtime.harbor import BenchmarkCodex + + agent = BenchmarkCodex(logs_dir=tmp_path, model_name="openai/fixture") + date = tmp_path / "sessions" / "2026" / "01" / "01" + date.mkdir(parents=True) + for name in ("first", "second"): + (date / f"{name}.jsonl").write_text("{}\n") + seen = [] + + def convert(directory): + seen.append([p.name for p in directory.glob("*.jsonl")]) + + monkeypatch.setattr(agent, "_convert_events_to_trajectory", convert) + agent._session_trajectories([tmp_path]) + assert sorted(seen) == [["first.jsonl"], ["second.jsonl"]] + + +def test_baseline_and_treatment_use_same_harbor_entry(tmp_path): + pytest.importorskip("harbor") + from benchmark.runtime.harbor import BenchmarkCodex + + modes = ("plain", "native-goal", "heartbeat", "loopx-goal", "turn") + for mode in modes: + kwargs = {"validation_command": ["trusted-validator"]} if mode == "turn" else {} + agent = BenchmarkCodex( + logs_dir=tmp_path / mode, + model_name="openai/fixture", + execution_mode=mode, + **kwargs, + ) + env = agent._worker_env(cwd="/workspace") + assert env["LOOPX_EXECUTION_MODE"] == mode + assert env["LOOPX_PROJECT"] == "/workspace" + assert env["MODEL_NAME"] == "fixture" + + +@pytest.mark.parametrize("existing", [False, True]) +def test_phase_bootstrap_uses_current_public_cli(tmp_path, monkeypatch, existing): + pytest.importorskip("harbor") + from benchmark.runtime.harbor import BenchmarkCodex + from loopx.cli import build_parser + + agent = BenchmarkCodex(logs_dir=tmp_path, model_name="openai/fixture") + calls = [] + + async def write_task(*args, **kwargs): + pass + + async def registry_exists(*args): + return existing + + async def cli(environment, args, **kwargs): + # Parse the actual adapter command, so retired flags fail without + # launching a model or mutating any active project. + build_parser().parse_args(args) + calls.append(args) + return {"todo_id": "todo_fixture", "after": {"execution_profile": {"replan_after_completed_todos": 3}}} + + monkeypatch.setattr(agent, "_write_task_document", write_task) + monkeypatch.setattr(agent, "_registry_exists", registry_exists) + monkeypatch.setattr(agent, "_loopx", cli) + async def no_pending(**kwargs): + return SimpleNamespace(return_code=1) + + asyncio.run(agent._prepare_phase(SimpleNamespace(exec=no_pending), "Synthetic task", cwd=str(tmp_path))) + assert any(args[:2] == ["todo", "add"] for args in calls) + assert any(args[0] == "bootstrap" for args in calls) is not existing + assert all("--clear-waiting-on" not in args for args in calls) + + +def test_staged_snapshot_keeps_observed_commit_when_branch_moves(tmp_path, monkeypatch): + pytest.importorskip("harbor") + import tarfile + from benchmark.runtime import harbor + + source = tmp_path / "source" + source.mkdir() + + def git(*args): + return subprocess.run( + ["git", "-C", str(source), *args], + check=True, + capture_output=True, + text=True, + ) + + git("init") + git("config", "user.name", "Fixture") + git("config", "user.email", "fixture@example.invalid") + marker = source / "revision.txt" + marker.write_text("original") + git("add", "revision.txt") + git("commit", "-m", "original") + original = git("rev-parse", "HEAD").stdout.strip() + monkeypatch.setattr(harbor, "__file__", str(source / "benchmark/runtime/harbor.py")) + monkeypatch.setenv("LOOPX_EXPECTED_COMMIT", original) + run = subprocess.run + + def moving_head(argv, **kwargs): + result = run(argv, **kwargs) + if argv[-2:] == ["rev-parse", "HEAD"]: + marker.write_text("successor") + git("add", "revision.txt") + git("commit", "-m", "successor") + return result + + monkeypatch.setattr(harbor.subprocess, "run", moving_head) + uploaded = [] + + class Environment: + async def upload_file(self, path, target): + with tarfile.open(path) as archive: + uploaded.append(archive.extractfile("revision.txt").read()) + + async def unpack(*args, **kwargs): + pass + + agent = harbor.BenchmarkCodex( + logs_dir=tmp_path / "logs", model_name="openai/fixture" + ) + monkeypatch.setattr(agent, "exec_as_root", unpack) + staged = asyncio.run(agent._stage_source(Environment(), source)) + assert staged == original + assert marker.read_text() == "successor" + assert uploaded == [b"original"] diff --git a/benchmark/tests/test_task_entry.py b/benchmark/tests/test_task_entry.py new file mode 100644 index 0000000000..72b073b061 --- /dev/null +++ b/benchmark/tests/test_task_entry.py @@ -0,0 +1,375 @@ +from __future__ import annotations + +import asyncio +import json +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from benchmark.runtime.codex import Execution +from benchmark.runtime.planning import validate_plan_readback +from benchmark.runtime.worker import run_once + + +@pytest.mark.parametrize( + "kwargs", + [ + {"task_entry": "unknown"}, + {"mode": "plain", "task_entry": "loopx-planned"}, + {"mode": "native-goal", "task_entry": "loopx-planned"}, + ], +) +def test_invalid_entry_rejected_before_model_call(kwargs): + with pytest.raises(ValueError): + Execution(**kwargs) + + +@pytest.fixture +def planning_env(tmp_path): + project = tmp_path / "project" + project.mkdir() + task = tmp_path / "task.md" + task.write_text("Repair the failure and preserve a regression test.\n") + state = tmp_path / "state.md" + state.write_text("# Active Goal State\n\n## Agent Todos\n\n## User Todos\n") + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "schema_version": "0.1", + "goals": [ + { + "id": "planning-goal", + "repo": str(project), + "state_file": str(state), + "status": "active", + "coordination": {"registered_agents": ["planner"]}, + } + ], + } + ) + ) + cli = tmp_path / "loopx" + cli.write_text( + f"#!{sys.executable}\nfrom loopx.cli import main\nraise SystemExit(main())\n" + ) + cli.chmod(0o755) + skills = tmp_path / "skills" + skills.mkdir() + binary = tmp_path / "codex" + binary.write_text( + f"#!{sys.executable}\n" + + """ +import json, os, pathlib, subprocess, sys +packet = json.loads(sys.stdin.read().split("Host-supplied planning checkpoint:\\n", 1)[1]) +command = [os.environ["LOOPX_CLI"], "--format", "json", "--registry", os.environ["LOOPX_REGISTRY"], + "--runtime-root", os.environ["LOOPX_RUNTIME_ROOT"], "todo", "add", "--goal-id", "planning-goal", + "--role", "agent", "--claimed-by", "planner", + "--text", "[P0] Reproduce and repair the failure, then pass the regression.", + "--task-class", "advancement_task", "--action-kind", "implement", "--execute"] +if not packet["runnable_todo_ids"]: + written = json.loads(subprocess.run(command, check=True, capture_output=True, text=True).stdout) + todo_id = written["todo_id"] +else: + todo_id = packet["runnable_todo_ids"][0] +result = {"input_digest": packet["input_digest"], "status": "ready", "todo_ids": [todo_id]} +pathlib.Path(sys.argv[sys.argv.index("--output-last-message") + 1]).write_text(json.dumps(result)) +pathlib.Path(os.environ["CODEX_HOME"], "seen-argv.json").write_text(json.dumps(sys.argv)) +""" + ) + binary.chmod(0o755) + return dict(os.environ) | { + "PYTHONPATH": str(Path(__file__).resolve().parents[2]), + "LOOPX_EXECUTION_MODE": "heartbeat", + "LOOPX_TASK_ENTRY": "loopx-planned", + "LOOPX_TASK_STAGE": "plan", + "LOOPX_PLANNING_TIMEOUT_SEC": "30", + "LOOPX_PLANNING_RESULT": str(tmp_path / "planning.json"), + "LOOPX_GOAL_ID": "planning-goal", + "LOOPX_AGENT_ID": "planner", + "LOOPX_REGISTRY": str(registry), + "LOOPX_RUNTIME_ROOT": str(tmp_path / "runtime"), + "LOOPX_CLI": str(cli), + "CODEX_BIN": str(binary), + "LOOPX_PROJECT": str(project), + "LOOPX_TASK_DOC": str(task), + "LOOPX_CODEX_HOME": str(tmp_path / "home"), + "LOOPX_SHARED_SKILLS": str(skills), + "LOOPX_WAKE_LOG_DIR": str(tmp_path / "logs" / "wakes"), + "MODEL_NAME": "fixture", + "REASONING_EFFORT": "high", + "OPENAI_BASE_URL": "http://localhost:8123/v1", + "OPENAI_API_KEY": "fixture-key", + } + + +def test_planning_writes_real_todo_then_reuses_it_without_executing_task(planning_env): + ids = [] + for _ in range(2): + receipt = run_once(planning_env) + assert receipt["ok"] and receipt["planning"]["state_readback_verified"] + ids.append(receipt["planning"]["todo_ids"]) + state = Path(planning_env["LOOPX_REGISTRY"]).with_name("state.md").read_text() + assert ids[0] == ids[1] and len(ids[0]) == 1 + assert state.count("todo_id=" + ids[0][0]) == 1 + assert not list(Path(planning_env["LOOPX_PROJECT"]).iterdir()) + argv = json.loads( + (Path(planning_env["LOOPX_CODEX_HOME"]) / "seen-argv.json").read_text() + ) + assert "features.goals=false" in argv and "resume" not in argv + assert not list( + Path(planning_env["LOOPX_RUNTIME_ROOT"]).rglob("benchmark-pending-turn.json") + ) + + +def test_prose_or_fabricated_todo_does_not_qualify_planning(planning_env): + binary = Path(planning_env["CODEX_BIN"]) + binary.write_text( + binary.read_text().replace( + '"todo_ids": [todo_id]', '"todo_ids": ["todo_fabricated"]' + ) + ) + with pytest.raises(ValueError, match="missing or unrelated"): + run_once(planning_env) + assert not Path(planning_env["LOOPX_PLANNING_RESULT"]).exists() + receipt = json.loads( + next( + Path(planning_env["LOOPX_WAKE_LOG_DIR"]).glob("*/receipt.json") + ).read_text() + ) + # A zero host exit must not be mistaken for qualified planning. + assert ( + not receipt["ok"] + and receipt.get("planning") is None + and receipt["error_kind"] == "ValueError" + ) + + +def test_plan_readback_rejects_stale_input_unclaimed_work_and_false_blockers(): + packet = { + "input_digest": "input-a", + "goal_id": "goal", + "agent_id": "agent", + "existing_todos": [ + {"todo_id": "todo_owned"}, + {"todo_id": "todo_unclaimed"}, + {"todo_id": "todo_gate"}, + ], + "runnable_todo_ids": ["todo_owned"], + "blocking_todo_ids": ["todo_gate"], + } + result = {"input_digest": "input-a", "status": "ready", "todo_ids": ["todo_owned"]} + assert validate_plan_readback(result, packet, packet)["status"] == "ready" + assert ( + validate_plan_readback( + result | {"status": "blocked", "todo_ids": ["todo_gate"]}, packet, packet + )["status"] + == "blocked" + ) + for invalid in [ + result | {"input_digest": "old"}, + result | {"todo_ids": ["todo_unclaimed"]}, + result | {"status": "blocked"}, + result | {"todo_ids": ["todo_owned", "todo_owned"]}, + ]: + with pytest.raises(ValueError): + validate_plan_readback(invalid, packet, packet) + + +def test_planned_phase_preserves_waits_and_does_not_prewrite_a_todo( + tmp_path, monkeypatch +): + pytest.importorskip("harbor") + from benchmark.runtime.harbor import BenchmarkCodex + + agent = BenchmarkCodex( + logs_dir=tmp_path, model_name="openai/fixture", task_entry="loopx-planned" + ) + calls = [] + + async def prepared(*args, **kwargs): + return True + + async def cli(environment, args, **kwargs): + calls.append(args) + return {"after": {"execution_profile": {"replan_after_completed_todos": 3}}} + + async def no_pending(**kwargs): + return SimpleNamespace(return_code=1) + + monkeypatch.setattr(agent, "_write_task_document", prepared) + monkeypatch.setattr(agent, "_registry_exists", prepared) + monkeypatch.setattr(agent, "_loopx", cli) + asyncio.run( + agent._prepare_phase( + SimpleNamespace(exec=no_pending), "new feedback", cwd=str(tmp_path) + ) + ) + assert all(args[:2] != ["todo", "add"] for args in calls) + assert all( + "--clear-waiting-on" not in args and "--agent-work-mode" not in args + for args in calls + ) + + +@pytest.mark.parametrize("status", ["ready", "blocked"]) +def test_planning_budget_and_blocked_handoff_use_the_real_adapter_run( + tmp_path, monkeypatch, status +): + pytest.importorskip("harbor") + from benchmark.runtime import harbor + + agent = harbor.BenchmarkCodex( + logs_dir=tmp_path, + model_name="openai/fixture", + task_entry="loopx-planned", + turn_timeout_sec=250, + scheduler_timeout_sec=500, + ) + clock = [0.0] + executions = [] + + async def prepare(*args, **kwargs): + pass + + async def execute(environment, *, command, env=None, **kwargs): + if command == "pwd": + return SimpleNamespace(stdout="/workspace", return_code=0) + if env.get("LOOPX_TASK_STAGE") == "plan": + clock[0] = 200 + else: + executions.append((command, env)) + return SimpleNamespace(stdout="", return_code=0) + + async def read_result(**kwargs): + return SimpleNamespace( + stdout=json.dumps({"status": status, "state_readback_verified": True}) + ) + + monkeypatch.setattr(harbor.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(agent, "_prepare_phase", prepare) + monkeypatch.setattr(agent, "exec_as_agent", execute) + monkeypatch.setattr(agent, "_populate_context", lambda *args: None) + environment = SimpleNamespace(exec=read_result, is_mounted=True) + asyncio.run(agent.run("Synthetic task", environment, SimpleNamespace())) + assert len(executions) == (1 if status == "ready" else 0) + if executions: + command, env = executions[0] + assert "--kill-after=30 300s" in command + assert float(env["LOOPX_CODEX_TURN_TIMEOUT_SEC"]) == 140 + assert "LOOPX_PHASE_DEADLINE_EPOCH=$(( $(date +%s) + 300 ))" in command + + +def test_pending_turn_prevents_phase_input_replacement(tmp_path, monkeypatch): + pytest.importorskip("harbor") + from benchmark.runtime.harbor import BenchmarkCodex + + agent = BenchmarkCodex(logs_dir=tmp_path, model_name="openai/fixture") + + async def pending(**kwargs): + return SimpleNamespace(return_code=0) + + async def unexpected_write(*args, **kwargs): + pytest.fail("pending transaction input must not be replaced") + + monkeypatch.setattr(agent, "_write_task_document", unexpected_write) + with pytest.raises(RuntimeError, match="pending Turn"): + asyncio.run( + agent._prepare_phase( + SimpleNamespace(exec=pending), "next task", cwd=str(tmp_path) + ) + ) + + +def test_late_scheduler_wake_does_not_open_an_unfinishable_turn(planning_env, monkeypatch): + from benchmark.runtime import worker + + env = planning_env | { + "LOOPX_EXECUTION_MODE": "turn", + "LOOPX_TASK_STAGE": "execute", + "LOOPX_VALIDATION_COMMAND_JSON": '["python", "check.py"]', + "LOOPX_CODEX_TURN_TIMEOUT_SEC": "60", + "LOOPX_PHASE_DEADLINE_EPOCH": "260", + } + monkeypatch.setattr(worker.time, "time", lambda: 100) + monkeypatch.setattr(worker, "prepare_codex_home", lambda *a, **kw: pytest.fail("late wake must not launch a host")) + for entry in ("seeded-todo", "loopx-planned"): + receipt = run_once(env | {"LOOPX_TASK_ENTRY": entry}) + assert receipt["budget_exhausted"] and receipt["host_invoked"] is False + assert receipt.get("turn_execution") is None + assert not (Path(env["LOOPX_RUNTIME_ROOT"]) / "benchmark-pending-turn.json").exists() + + +def test_remaining_phase_time_caps_later_host_windows(planning_env, monkeypatch): + from benchmark.runtime import worker + + class CapturedWindow(Exception): + pass + + def capture(home, *, execution, **kwargs): + assert execution.timeout_seconds == 40 + raise CapturedWindow + + monkeypatch.setattr(worker.time, "time", lambda: 100) + monkeypatch.setattr(worker, "prepare_codex_home", capture) + with pytest.raises(CapturedWindow): + run_once(planning_env | { + "LOOPX_TASK_STAGE": "execute", + "LOOPX_CODEX_TURN_TIMEOUT_SEC": "60", + "LOOPX_PHASE_DEADLINE_EPOCH": "300", + }) + + +@pytest.mark.parametrize("status", ["open", "blocked", "done", "deferred"]) +def test_seeded_followup_uses_real_todo_delta_without_reviving_terminal_work( + planning_env, tmp_path, monkeypatch, status +): + import contextlib + import io + pytest.importorskip("harbor") + from benchmark.runtime import harbor + from loopx.cli import main + + monkeypatch.setattr(harbor, "_GOAL_ID", "planning-goal") + monkeypatch.setattr(harbor, "_AGENT_ID", "planner") + agent = harbor.BenchmarkCodex(logs_dir=tmp_path, model_name="openai/fixture") + + async def cli(environment, args, **kwargs): + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = main([ + "--format", "json", "--registry", planning_env["LOOPX_REGISTRY"], + "--runtime-root", planning_env["LOOPX_RUNTIME_ROOT"], *args, + ]) + assert code == 0, output.getvalue() + return json.loads(output.getvalue()) + + monkeypatch.setattr(agent, "_loopx", cli) + + async def scenario(): + agent._phase_number = 1 + await agent._seed_phase(None, cwd=planning_env["LOOPX_PROJECT"]) + original = agent._seeded_todo_id + transition = (["complete", "--no-follow-up", "--note", "Synthetic task independently validated; no remaining work"] + if status == "done" else ["update", "--status", status]) + if status == "deferred": + transition += ["--resume-when", "capacity_available:fixture_pool"] + await cli(None, ["todo", *transition, "--goal-id", "planning-goal", + "--todo-id", original, "--agent-id", "planner", "--execute"]) + agent._phase_number = 2 + await agent._seed_phase(None, cwd=planning_env["LOOPX_PROJECT"]) + listed = await cli(None, ["todo", "list", "--goal-id", "planning-goal", "--role", "agent"]) + todos = {t["todo_id"]: t for t in listed["todos"]} + if status in {"open", "blocked"}: + assert agent._seeded_todo_id == original and len(todos) == 1 + assert todos[original]["status"] == status + assert "task-phase-002.md" in todos[original]["text"] + else: + assert agent._seeded_todo_id != original and len(todos) == 2 + assert todos[original]["status"] == status + assert "task-phase-001.md" in todos[original]["text"] + + asyncio.run(scenario()) diff --git a/docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.md b/docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.md index e016c7ef72..5ab0dedd29 100644 --- a/docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.md +++ b/docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.md @@ -946,6 +946,20 @@ research target; E4 remains a future promotion gate. A benchmark adapter may be technically E1-ready while its study is still C0, and a C2 result is not valid when its E2 runtime evidence is incomplete. +The [shared Codex research runtime](../../../benchmark/runtime/RUNTIME.md) is +the LHTB/SWE-Marathon native bridge checkpoint: it shares trial setup and uses +product heartbeat, Turn and Goal execution while retaining native verification. +Synthetic Harbor conformance qualifies this engineering seam only; E3 matched +studies and E4 cross-benchmark claims remain separate acceptance. + +Task entry is a separate ablation axis: a runner-seeded execution Todo versus +model planning through the product's `todo plan` checkpoint. Planning runs +before the selected driver, uses the shared Goal planner/Todo-delta contract, +and consumes the phase budget without counting as advancement. Qualification +must read back task identity and actual Todos, preserve blocked state and +disclose the separate planning session; synthetic task success alone does not +prove equivalence to interactive `$loopx` startup or planning effectiveness. + ### 11.3 Required delivery slice Every benchmark engineering PR or contributor task should identify a bounded diff --git a/docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.zh-CN.md b/docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.zh-CN.md index c9314fc4ab..21083a51b2 100644 --- a/docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.zh-CN.md +++ b/docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.zh-CN.md @@ -804,6 +804,17 @@ release promise： 一个 adapter 可以在工程上达到 E1，但其 study 仍然只有 C0；如果 E2 runtime evidence 不完整，C2 result 也不成立。 +[共享 Codex 研究 runtime](../../../benchmark/runtime/RUNTIME.md) 是 +LHTB/SWE-Marathon 原生桥接的工程检查点：共享 trial 初始化，复用产品的 +heartbeat、Turn 和 Goal 执行，并保留原生验证。合成 Harbor conformance +只验证这条工程路径；E3 matched study 和 E4 跨 benchmark 结论仍需独立验收。 + +任务入口是独立的消融轴：runner 预写执行 Todo，或模型通过产品 `todo plan` +检查点进行规划。规划先于所选执行驱动，复用 Goal planner 和 Todo 增量契约， +消耗 phase 总预算但不计作 advancement。验收须读回任务身份和真实 Todo，保留 +阻塞状态，并披露独立的规划会话；合成任务通过不能证明与交互式 `$loopx` 启动 +等价，也不能证明规划效果提升。 + ### 11.3 必需的 delivery slice 每个 benchmark engineering PR 或 contributor task 都应声明一个有界 slice，包含： diff --git a/docs/project-agent-todo-contract.md b/docs/project-agent-todo-contract.md index 227592edac..18f12cf246 100644 --- a/docs/project-agent-todo-contract.md +++ b/docs/project-agent-todo-contract.md @@ -29,6 +29,29 @@ sync catch up. ## Write Contract +For a caller-owned runtime that already registered its Goal/Agent, generate the +model planning checkpoint before writing executable task Todos: + +```bash +loopx --format json todo plan --goal-id --agent-id \ + --text '' +``` + +This read-only command reuses `/loopx`'s planner and Todo-delta contract. It +returns the current frontier, typed result schema and an explicit caller-owned +execution handoff; it does not run a model, create a Goal, write a Todo, activate +a host loop or spend quota. A model consumes the packet and uses the existing +Todo CLI to plan actual task work. No planning/setup Todo is required. Read back +the returned ids and current state before the caller activates its driver and +enters the normal quota guard. A planning result does not authorize execution. +Follow-up input preserves the Goal/Agent and existing waits; reconcile the plan +instead of restarting the Goal. Unrelated peers and their claims remain intact. + +The installed `$loopx` skill recognizes this explicit packet as a bounded +planning checkpoint. Without one, its existing startup/continuation behavior is +unchanged. Omit `todo plan` to use that ordinary interactive entry; callers must +not simulate a planning checkpoint by prewriting an advancement Todo. + When read-only analysis, a review packet, a gate checklist, or P0/P1 steering finds a concrete user or owner action, write it immediately with the todo CLI. Use `user_gate` only when the item blocks an agent or the whole goal: diff --git a/loopx/capabilities/benchmark_toolkit/native_codex_goal.py b/loopx/capabilities/benchmark_toolkit/native_codex_goal.py index 7033cf29ff..677465fd5d 100644 --- a/loopx/capabilities/benchmark_toolkit/native_codex_goal.py +++ b/loopx/capabilities/benchmark_toolkit/native_codex_goal.py @@ -15,7 +15,7 @@ import threading import time from collections import Counter, deque -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from hashlib import sha256 from typing import Any, Protocol, Self, TextIO @@ -414,6 +414,7 @@ def run_native_goal_until_terminal( config: NativeGoalConfig, *, timeout_sec: float, + on_turn_started: Callable[[NativeGoalTurn], None] | None = None, ) -> NativeGoalTurn: """Run one native Goal until its status leaves ``active``. @@ -425,6 +426,8 @@ def run_native_goal_until_terminal( if timeout_sec <= 0: raise ValueError("timeout_sec must be positive") turn = start_native_goal_turn(transport, config) + if on_turn_started is not None: + on_turn_started(turn) deadline = time.monotonic() + timeout_sec completed_before = turn.turn_completed_count while True: diff --git a/loopx/cli_commands/todo.py b/loopx/cli_commands/todo.py index 322b66ecc1..a0453ccf3b 100644 --- a/loopx/cli_commands/todo.py +++ b/loopx/cli_commands/todo.py @@ -30,6 +30,10 @@ build_todo_suggestion_prompt_packet, render_todo_suggestion_prompt_markdown, ) +from ..control_plane.goals.task_planning import ( + build_task_planning_packet, + render_task_planning_packet, +) from ..todos import ( add_goal_todo, archive_completed_todos, @@ -50,6 +54,7 @@ validate_todo_list_options, validate_todo_project_markdown_options, validate_todo_suggest_options, + validate_todo_plan_options, validate_todo_supersede_options, validate_todo_update_options, ) @@ -195,6 +200,9 @@ def handle_todo_command( post_writeback_projection_builder: PostWritebackProjectionBuilder | None = None, ) -> int: renderer = ( + render_task_planning_packet + if args.todo_command == "plan" + else render_todo_suggestion_prompt_markdown if args.todo_command == "suggest" else render_todo_markdown @@ -208,7 +216,14 @@ def handle_todo_command( ) validate_shared_todo_options(args) validate_capability_gap_options(args) - if args.todo_command == "list": + if args.todo_command == "plan": + validate_todo_plan_options(args) + payload = build_task_planning_packet( + registry_path=registry_path, runtime_root_arg=runtime_root_arg, + goal_id=args.goal_id, agent_id=args.agent_id, text=args.text, + project=Path(args.project).expanduser() if args.project else None, + ) + elif args.todo_command == "list": validate_todo_list_options(args) payload = list_goal_todos( registry_path=registry_path, diff --git a/loopx/cli_commands/todo_argument_validation.py b/loopx/cli_commands/todo_argument_validation.py index b980ecc073..6753f36b88 100644 --- a/loopx/cli_commands/todo_argument_validation.py +++ b/loopx/cli_commands/todo_argument_validation.py @@ -303,6 +303,15 @@ def validate_todo_list_options(args: argparse.Namespace) -> None: ) +def validate_todo_plan_options(args: argparse.Namespace) -> None: + _validate_todo_option_subset( + args, {"text", "agent_id"}, + "todo plan only accepts --goal-id, --agent-id, --text, --project and --format; unsupported: ", + ) + if not args.text or not args.agent_id: + raise ValueError("todo plan requires --text and a registered --agent-id") + + def validate_todo_project_markdown_options(args: argparse.Namespace) -> None: if not getattr(args, "provider_revision", None): raise ValueError("todo project-markdown requires --provider-revision") @@ -551,7 +560,7 @@ def validate_shared_todo_options(args: argparse.Namespace) -> None: "--authority-reason is supported only by todo update/complete/supersede" ) if ( - args.todo_command not in {"suggest", "capture-followups"} + args.todo_command not in {"suggest", "plan", "capture-followups"} and args.agent_id and not agent_id_allowed_for_user_authoring and not agent_id_allowed_for_read diff --git a/loopx/cli_commands/todo_registration.py b/loopx/cli_commands/todo_registration.py index 77b7d3c2c9..34023b9984 100644 --- a/loopx/cli_commands/todo_registration.py +++ b/loopx/cli_commands/todo_registration.py @@ -41,6 +41,7 @@ def register_todo_command( "supersede", "archive-completed", "suggest", + "plan", "capture-followups", "project-markdown", ], @@ -50,6 +51,7 @@ def register_todo_command( "agent id, list to read projected todos, update/complete/supersede to transition by todo_id, or " "archive-completed to move older completed todos into Completed Work Archive. " "Use suggest to generate an agent-facing candidate todo analysis prompt without writing state. " + "Use plan with --text and --agent-id for the existing Goal's model planning checkpoint; the caller owns subsequent execution. " "Use capture-followups to record a capped public-safe unclaimed follow-up batch." ), ) diff --git a/loopx/cli_commands/turn_registration.py b/loopx/cli_commands/turn_registration.py index efc009d931..a787f708a5 100644 --- a/loopx/cli_commands/turn_registration.py +++ b/loopx/cli_commands/turn_registration.py @@ -209,9 +209,11 @@ def register_turn_commands( run_once.add_argument("--codex-model") run_once.add_argument( "--codex-sandbox", - choices=["read-only", "workspace-write"], + choices=["read-only", "workspace-write", "danger-full-access"], default="read-only", - help="Sandbox for a new Codex CLI session; resume preserves its original session policy.", + help=("Codex CLI sandbox (default: read-only). danger-full-access explicitly " + "disables the inner sandbox; callers must provide their own isolation. " + "The setting is passed explicitly for both new and resumed sessions."), ) run_once.add_argument( "--dsh-provider", diff --git a/loopx/control_plane/goals/start_contract.py b/loopx/control_plane/goals/start_contract.py index ce3579f427..1a395a6323 100644 --- a/loopx/control_plane/goals/start_contract.py +++ b/loopx/control_plane/goals/start_contract.py @@ -5,6 +5,50 @@ GOAL_START_SCHEMA_VERSION = "loopx_goal_start_command_v0" +def goal_planner_contract(*, fine_grained: bool = False) -> dict[str, Any]: + planner = { + "required_before_todo_write": True, + "default_profile": "open_ended_product_direction", + "profile_selection": ( + "Use open_ended_product_direction when the user's goal is a broad, " + "fuzzy product direction or new initiative. Use clear_bounded_problem " + "when the target is a concrete task with a clear success condition. " + "In both cases, let the model produce a real ordered plan before writes." + ), + "profiles": { + "open_ended_product_direction": { + "suggested_items_min": 2, + "suggested_items_max": 5, + "intent": ( + "turn an ambiguous product direction into public-safe, ranked " + "todo options before execution" + ), + }, + "clear_bounded_problem": { + "item_count_policy": "planner_sized", + "may_reuse_current_todo_when_it_already_represents_the_plan": True, + "intent": ( + "make the approach explicit with enough concise ordered todos, " + "without arbitrary caps or management-only filler" + ), + }, + }, + "allowed_priorities": ["P0", "P1", "P2"], + "default_role": "agent", + "default_task_class": "advancement_task", + "required_fields": ["priority", "text", "task_class", "action_kind"], + "public_safe_only": True, + "budget_policy": "minimum sufficient plan; no fixed-count filler", + } + if fine_grained: + planner["fine_grained_plan_horizon"] = ( + "write one current runnable checkpoint; keep later options as evidence-linked " + "planning notes until the existing replan path qualifies the successor" + ) + planner["maximum_runnable_todos_written_ahead"] = 1 + return planner + + def build_goal_start_contract( *, goal_text: str | None, @@ -26,40 +70,7 @@ def build_goal_start_contract( "explicit_invocation_confirms_project_local_state_writes": True, "connect_if_needed": True, "bootstrap_policy": "create project-local LoopX state only when no matching registry goal exists", - "planner": { - "required_before_todo_write": True, - "default_profile": "open_ended_product_direction", - "profile_selection": ( - "Use open_ended_product_direction when the user's goal is a broad, " - "fuzzy product direction or new initiative. Use clear_bounded_problem " - "when the target is a concrete task with a clear success condition. " - "In both cases, let the model produce a real ordered plan before writes." - ), - "profiles": { - "open_ended_product_direction": { - "suggested_items_min": 2, - "suggested_items_max": 5, - "intent": ( - "turn an ambiguous product direction into public-safe, ranked " - "todo options before execution" - ), - }, - "clear_bounded_problem": { - "item_count_policy": "planner_sized", - "may_reuse_current_todo_when_it_already_represents_the_plan": True, - "intent": ( - "make the approach explicit with enough concise ordered todos, " - "without arbitrary caps or management-only filler" - ), - }, - }, - "allowed_priorities": ["P0", "P1", "P2"], - "default_role": "agent", - "default_task_class": "advancement_task", - "required_fields": ["priority", "text", "task_class", "action_kind"], - "public_safe_only": True, - "budget_policy": "minimum sufficient plan; no fixed-count filler", - }, + "planner": goal_planner_contract(fine_grained=fine_grained), "priority_ordering": { "bucket_order": ["P0", "P1", "P2"], "same_priority_tie_breaker": "planner_order_then_todo_write_order", @@ -149,12 +160,6 @@ def build_goal_start_contract( "replan": "direction_change_or_bounded_chain", "checkpoint_accounting": "advancement_only", } - planner = contract["planner"] - planner["fine_grained_plan_horizon"] = ( - "write one current runnable checkpoint; keep later options as evidence-linked " - "planning notes until the existing replan path qualifies the successor" - ) - planner["maximum_runnable_todos_written_ahead"] = 1 return contract diff --git a/loopx/control_plane/goals/start_goal_todo_delta.py b/loopx/control_plane/goals/start_goal_todo_delta.py index 8152323c03..01ff4ea770 100644 --- a/loopx/control_plane/goals/start_goal_todo_delta.py +++ b/loopx/control_plane/goals/start_goal_todo_delta.py @@ -110,10 +110,12 @@ def _todo_add_command_template( runtime_root: str | Path | None, goal_id: str, agent_id: str | None, + registry_path: Path | None = None, ) -> str: return ( f"{render_cli_command_prefix(cli_bin=cli_bin, runtime_root=runtime_root)} " - f"todo add --goal-id " + + (f"--registry {shell_arg(str(registry_path))} " if registry_path is not None else "") + + "todo add --goal-id " f"{shell_arg(str(goal_id or ''))} " "--project . " "--role agent " @@ -136,6 +138,7 @@ def todo_authoring_steps( runtime_root: str | Path | None, goal_id: str, agent_id: str | None, + registry_path: Path | None = None, ) -> list[dict[str, Any]]: """Ordered Todo-authoring steps, conditional on the runnable frontier.""" add_template = _todo_add_command_template( @@ -143,6 +146,7 @@ def todo_authoring_steps( runtime_root=runtime_root, goal_id=goal_id, agent_id=agent_id, + registry_path=registry_path, ) if not existing_runnable_frontier: return [ diff --git a/loopx/control_plane/goals/task_planning.py b/loopx/control_plane/goals/task_planning.py new file mode 100644 index 0000000000..6a354b133e --- /dev/null +++ b/loopx/control_plane/goals/task_planning.py @@ -0,0 +1,151 @@ +"""Model-owned task planning for an existing Goal, before its caller starts work.""" + +from __future__ import annotations + +import hashlib +import json +import shlex +from pathlib import Path +from typing import Any + +from ...agent_registry import require_registered_agent_id +from ...execution_profile import execution_profile_is_fine_grained +from ...history import load_registry +from ...registry import find_registry_goal +from ...todos import list_goal_todos +from ..todos.todo_semantics import todo_item_is_actionable_open +from ..todos.contract import ( + TODO_STATUS_BLOCKED, + TODO_TERMINAL_STATUS_VALUES, + TODO_TASK_CLASS_ADVANCEMENT, + TODO_TASK_CLASS_BLOCKER, + TODO_TASK_CLASS_USER_GATE, +) +from .start_contract import goal_planner_contract +from .start_goal_todo_delta import todo_authoring_steps + + +TASK_PLAN_SCHEMA = "loopx_task_planning_v0" +TASK_PLAN_RESULT_SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": ["input_digest", "status", "todo_ids"], + "properties": { + "input_digest": {"type": "string"}, + "status": {"type": "string", "enum": ["ready", "blocked"]}, + "todo_ids": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + }, +} + + +def build_task_planning_packet( + *, + registry_path: Path, + goal_id: str, + agent_id: str, + text: str, + project: Path | None = None, + runtime_root_arg: str | None = None, +) -> dict[str, Any]: + """Read canonical planning inputs; create no Goal, Todo, Turn or host loop.""" + if not text.strip(): + raise ValueError("todo plan requires non-empty --text") + agent_id = require_registered_agent_id( + registry_path=registry_path, + goal_id=goal_id, + agent_id=agent_id, + field="agent_id", + ) + goal = find_registry_goal(load_registry(registry_path), goal_id) + if goal is None: + raise ValueError("todo plan requires an existing Goal") + # Read both roles explicitly: the compact lane display is not a complete frontier. + todos = [] + for role in ("agent", "user"): + listed = list_goal_todos( + registry_path=registry_path, + goal_id=goal_id, + agent_id=agent_id, + role=role, + project=project, + runtime_root_arg=runtime_root_arg, + ) + todos.extend(listed["todos"]) + runnable = [ + t + for t in todos + if t.get("role") == "agent" + and t.get("task_class") == TODO_TASK_CLASS_ADVANCEMENT + and todo_item_is_actionable_open(t) + ] + fine = execution_profile_is_fine_grained(goal.get("execution_profile")) + prefix = ["loopx", "--format", "json", "--registry", str(registry_path)] + if runtime_root_arg: + prefix += ["--runtime-root", runtime_root_arg] + cli = shlex.join(prefix) + steps = todo_authoring_steps( + existing_runnable_frontier=runnable, + plan_prompt=None, + fine_grained=fine, + cli_bin="loopx", + runtime_root=runtime_root_arg, + goal_id=goal_id, + agent_id=agent_id, + registry_path=registry_path, + ) + identity = {"goal_id": goal_id, "agent_id": agent_id, "text": text} + digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() + return { + "ok": True, + "read_only": True, + "dry_run": True, + "command": "plan", + "schema_version": TASK_PLAN_SCHEMA, + **identity, + "input_digest": digest, + "planner": goal_planner_contract(fine_grained=fine), + "ordered_steps": steps, + "existing_todos": todos, + "runnable_todo_ids": [ + t["todo_id"] for t in runnable if t.get("claimed_by") == agent_id + ], + "blocking_todo_ids": [ + t["todo_id"] + for t in todos + if t.get("status") not in TODO_TERMINAL_STATUS_VALUES + and ( + t.get("status") == TODO_STATUS_BLOCKED + or t.get("task_class") + in {TODO_TASK_CLASS_BLOCKER, TODO_TASK_CLASS_USER_GATE} + ) + ], + "goal_waiting_on": goal.get("waiting_on"), + "result_schema": TASK_PLAN_RESULT_SCHEMA, + "execution_handoff": { + "owner": "caller", + "requires_quota_guard": True, + "starts_host_loop": False, + "spends_quota": False, + "planning_is_advancement": False, + }, + "task_body": ( + "Execute the LoopX task-planning checkpoint for this already registered Goal/Agent. " + "Use the attached planner and ordered_steps, shared with /loopx. Read the exact text " + "and inspect the workspace as needed; make the approach and acceptance explicit " + "before writing task Todos through the routed public CLI. Compare all existing " + "work and waits; reuse/update covered work and add only uncovered work. " + "Do not create a planning/setup Todo, restart the Goal, complete task Todos, " + "change task files, clear waits, activate a loop, execute task work or spend quota. " + "The caller owns the execution_handoff and must enter its quota guard after readback. " + "This is a planning-stage boundary, not task completion. Planning does not grant " + "additional permissions. For ready, return the actual open advancement Todo ids " + "claimed by this agent that cover this input. For blocked, persist/reference the " + "relevant blocker or User gate and return its Todo ids. Return input_digest exactly. " + "Do not claim success from prose or fabricate Todo ids. Command prefix: " + + cli + ), + } + + +def render_task_planning_packet(payload: dict[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=False, indent=2) diff --git a/loopx/control_plane/quota/turn_envelope.ts b/loopx/control_plane/quota/turn_envelope.ts index 6756491cc7..5a2133ab2b 100644 --- a/loopx/control_plane/quota/turn_envelope.ts +++ b/loopx/control_plane/quota/turn_envelope.ts @@ -327,6 +327,22 @@ function boundary(payload: JsonObject): JsonObject { const values = textList(source[field], 16, 180); if (values.length > 0) result[field] = values; } + // Carry the already resolved approval from quota; dropping it leaves only + // the bootstrap approval requirement and strands authorized execution. + const approved = object(source.checkpointed_boundary_authority); + const approvedScopes = Array.isArray(approved.active_write_scope) + ? approved.active_write_scope.filter((scope): scope is string => + typeof scope === "string" && scope.length > 0 && scope.length <= 180).slice(0, 16) + : []; + if (approved.schema_version === "checkpointed_boundary_authority_v0" + && Number.isInteger(approved.active_count) && Number(approved.active_count) > 0 + && approvedScopes.length > 0) { + result.checkpointed_boundary_authority = { + schema_version: approved.schema_version, + active_count: approved.active_count, + active_write_scope: approvedScopes, + }; + } const guards = textList(source.guards, 8, 280); if (guards.length > 0) result.guards = guards; const stopCondition = text(source.stop_condition, 320); diff --git a/loopx/control_plane/turn_driver/codex_cli.py b/loopx/control_plane/turn_driver/codex_cli.py index 0ba85c65ed..cc19372af0 100644 --- a/loopx/control_plane/turn_driver/codex_cli.py +++ b/loopx/control_plane/turn_driver/codex_cli.py @@ -38,7 +38,7 @@ "wait", "iteration_failed", ) -CODEX_CLI_SANDBOXES = ("read-only", "workspace-write") +CODEX_CLI_SANDBOXES = ("read-only", "workspace-write", "danger-full-access") SESSION_ID_MAX_CHARS = 256 OUTPUT_DRAIN_TIMEOUT_SECONDS = 2.0 SESSION_INVALIDATING_FAILURE_CATEGORIES = frozenset( @@ -345,6 +345,13 @@ def _prompt(request: Mapping[str, Any]) -> str: "Turn request:", request_json, ] + boundary = _mapping(_mapping(request.get("turn_envelope")).get("boundary")) + if boundary.get("checkpointed_boundary_authority"): + instructions.append( + "The boundary's checkpointed_boundary_authority records existing write approval " + "only within its active_write_scope. It satisfies the write approval requirement " + "for those scopes; other scopes, publish, and production actions retain their gates." + ) if _has_subagent_topology(request): instructions[7:7] = [ "When subagent_execution_topology is present, return one compact child_execution_receipts item for each observed child, including the actual context_mode. Never copy prompts, transcripts, tool output, credentials, private links, or local absolute paths into a receipt. If no child was observed, return an empty list.", @@ -712,7 +719,7 @@ def run_codex_cli_host( if request.get("schema_version") != LOOPX_TURN_HOST_REQUEST_SCHEMA_VERSION: raise ValueError("unsupported LoopX Turn host request schema") if sandbox not in CODEX_CLI_SANDBOXES: - raise ValueError("Codex CLI sandbox must be read-only or workspace-write") + raise ValueError(f"Codex CLI sandbox must be one of {CODEX_CLI_SANDBOXES}") resolved = shutil.which(codex_bin) if os.path.sep not in codex_bin else codex_bin if not resolved or not Path(resolved).exists(): raise ValueError("Codex CLI executable is unavailable") @@ -807,6 +814,9 @@ def discard_stderr() -> None: _terminate_process(proc) timed_out = True returncode = proc.returncode + except BaseException: + _terminate_process(proc) + raise finally: reader.join(timeout=OUTPUT_DRAIN_TIMEOUT_SECONDS) stderr_reader.join(timeout=OUTPUT_DRAIN_TIMEOUT_SECONDS) diff --git a/loopx/extensions/process_runtime.py b/loopx/extensions/process_runtime.py index 6915393f8b..7f6f118cdc 100644 --- a/loopx/extensions/process_runtime.py +++ b/loopx/extensions/process_runtime.py @@ -30,14 +30,16 @@ def _wait_for_process(process: subprocess.Popen[bytes], timeout: float) -> bool: return True -def _terminate_posix_process_group(process: subprocess.Popen[bytes]) -> None: +def _terminate_posix_process_group( + process: subprocess.Popen[bytes], grace_seconds: float +) -> None: process_group_id = process.pid try: os.killpg(process_group_id, signal.SIGTERM) except ProcessLookupError: process.wait() return - _wait_for_process(process, _PROCESS_TERMINATE_GRACE_SECONDS) + _wait_for_process(process, grace_seconds) try: os.killpg(process_group_id, signal.SIGKILL) except ProcessLookupError: @@ -47,7 +49,9 @@ def _terminate_posix_process_group(process: subprocess.Popen[bytes]) -> None: process.wait() -def _terminate_windows_process_tree(process: subprocess.Popen[bytes]) -> None: +def _terminate_windows_process_tree( + process: subprocess.Popen[bytes], grace_seconds: float +) -> None: if process.poll() is not None: return subprocess.run( @@ -56,7 +60,7 @@ def _terminate_windows_process_tree(process: subprocess.Popen[bytes]) -> None: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) - if _wait_for_process(process, _PROCESS_TERMINATE_GRACE_SECONDS): + if _wait_for_process(process, grace_seconds): return subprocess.run( ["taskkill", "/PID", str(process.pid), "/T", "/F"], @@ -67,17 +71,19 @@ def _terminate_windows_process_tree(process: subprocess.Popen[bytes]) -> None: process.wait() -def _terminate_process_tree(process: subprocess.Popen[bytes]) -> None: +def _terminate_process_tree( + process: subprocess.Popen[bytes], grace_seconds: float +) -> None: if os.name == "posix": - _terminate_posix_process_group(process) + _terminate_posix_process_group(process, grace_seconds) return if os.name == "nt": # pragma: no cover - exercised on Windows hosts. - _terminate_windows_process_tree(process) + _terminate_windows_process_tree(process, grace_seconds) return if process.poll() is not None: # pragma: no cover - unsupported platform fallback. return process.terminate() - if not _wait_for_process(process, _PROCESS_TERMINATE_GRACE_SECONDS): + if not _wait_for_process(process, grace_seconds): process.kill() process.wait() @@ -90,6 +96,7 @@ def run_capped_process( output_limit_bytes: int, env: Mapping[str, str] | None = None, cwd: str | Path | None = None, + termination_grace_seconds: float = _PROCESS_TERMINATE_GRACE_SECONDS, ) -> CappedProcessResult: """Run a provider while bounding both output streams during execution.""" @@ -179,24 +186,29 @@ def write_stdin() -> None: deadline = time.monotonic() + timeout_seconds timed_out = False - while process.poll() is None: - remaining = deadline - time.monotonic() - if remaining <= 0: - timed_out = True - _terminate_process_tree(process) - break - if limit_event.wait(timeout=min(0.05, remaining)): - _terminate_process_tree(process) - break + try: + while process.poll() is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + timed_out = True + _terminate_process_tree(process, termination_grace_seconds) + break + if limit_event.wait(timeout=min(0.05, remaining)): + _terminate_process_tree(process, termination_grace_seconds) + break + except BaseException: + _terminate_process_tree(process, termination_grace_seconds) + raise + finally: + for thread in threads: + thread.join(timeout=_PROCESS_TERMINATE_GRACE_SECONDS) + for stream in (process.stdout, process.stderr): + try: + stream.close() + except (OSError, ValueError): + pass returncode = process.wait() - for thread in threads: - thread.join(timeout=_PROCESS_TERMINATE_GRACE_SECONDS) - for stream in (process.stdout, process.stderr): - try: - stream.close() - except (OSError, ValueError): - pass return CappedProcessResult( returncode=returncode, stdout=bytes(stdout), diff --git a/loopx/slash_command_install.py b/loopx/slash_command_install.py index c9dca36605..c36b124d43 100644 --- a/loopx/slash_command_install.py +++ b/loopx/slash_command_install.py @@ -177,6 +177,7 @@ def _command_prompt_specs(*, cli_bin: str, include_legacy_aliases: bool) -> list host_surface=None, ), "Treat the returned `ordered_steps` and `goal_start_contract` as authoritative. Follow their identity, capability-route, Todo, writeback, host-loop, quota, and stop/gate rules before substantive work; do not reconstruct those rules from skill memory.", + "When the host explicitly supplies a `loopx_task_planning_v0` packet from `loopx todo plan` for a registered Goal/Agent, execute that bounded planning checkpoint instead of starting another Goal. Follow its shared planner and Todo delta, then return actual Todo ids for readback. Its caller-owned execution_handoff retains host activation and quota; do not create a planning Todo, execute task work, or claim delivery during the checkpoint.", "For a Codex App heartbeat, run the returned activation command, require ok=true, and save its `LoopX managed heartbeat bootstrap v2` task_body through automation_update. The saved loader fetches the current thin contract on every wake; do not persist a raw thin/compact/full execution body. Preserve the current goal, registered agent, task binding and existing schedule; read back the automation through the same App.", "If the packet exposes a goal-selection gate, rerun one exact choice before any mutation.", "When authoring task Todos, treat `--action-kind` as the documented extensible public-safe token: choose a short task-relevant value such as `implement`, `test`, or `review`; do not search the LoopX source for an allowlist.", diff --git a/scripts/external_scheduler_worker.py b/scripts/external_scheduler_worker.py index 084810e941..9533b84b9c 100644 --- a/scripts/external_scheduler_worker.py +++ b/scripts/external_scheduler_worker.py @@ -25,6 +25,7 @@ import json import os import shlex +import signal import sys import time from dataclasses import dataclass @@ -268,6 +269,7 @@ def _run_wake(command: str, *, timeout_seconds: float) -> CappedProcessResult: stdin=b"", timeout_seconds=max(0.01, timeout_seconds), output_limit_bytes=PROCESS_OUTPUT_LIMIT_BYTES, + termination_grace_seconds=10, ) @@ -558,4 +560,8 @@ def main(argv: Sequence[str] | None = None) -> int: if __name__ == "__main__": + def cancelled(signum, frame): + raise KeyboardInterrupt("scheduler cancelled") + + signal.signal(signal.SIGTERM, cancelled) raise SystemExit(main()) diff --git a/tests/control_plane/test_task_planning.py b/tests/control_plane/test_task_planning.py new file mode 100644 index 0000000000..826a78746a --- /dev/null +++ b/tests/control_plane/test_task_planning.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import contextlib +import io +import json +from pathlib import Path + +import pytest + +from loopx.cli import main +from loopx.control_plane.goals.task_planning import build_task_planning_packet + + +@pytest.fixture +def bound_goal(tmp_path): + state = tmp_path / "state.md" + state.write_text("# Active Goal State\n\n## Agent Todos\n\n## User Todos\n") + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "schema_version": "0.1", + "goals": [ + { + "id": "planning-goal", + "repo": str(tmp_path), + "state_file": str(state), + "status": "active", + "waiting_on": "protected approval", + "coordination": {"registered_agents": ["planner", "peer"]}, + } + ], + } + ) + ) + return dict( + registry_path=registry, + goal_id="planning-goal", + agent_id="planner", + text="Investigate the failure, then repair and validate it.\nKeep the API stable.", + project=tmp_path, + runtime_root_arg=str(tmp_path / "runtime"), + ) + + +def test_empty_frontier_plans_before_writes_without_starting_a_loop(bound_goal): + registry = bound_goal["registry_path"] + state = bound_goal["project"] / "state.md" + before = registry.read_bytes(), state.read_bytes() + packet = build_task_planning_packet(**bound_goal) + assert packet["text"] == bound_goal["text"] + assert [step["id"] for step in packet["ordered_steps"]] == [ + "plan_ranked_todos", + "write_ordered_todos", + ] + assert packet["planner"]["required_before_todo_write"] is True + assert packet["execution_handoff"] == { + "owner": "caller", + "requires_quota_guard": True, + "starts_host_loop": False, + "spends_quota": False, + "planning_is_advancement": False, + } + assert packet["goal_waiting_on"] == "protected approval" + assert (registry.read_bytes(), state.read_bytes()) == before + + +def test_existing_plan_is_an_incremental_frontier_not_a_new_goal(bound_goal): + state = bound_goal["project"] / "state.md" + state.write_text("""# Active Goal State + +## Agent Todos +- [ ] [P0] Reproduce the API failure and retain a regression. + +- [ ] [P0] Another agent's task. + + +## User Todos +""") + packet = build_task_planning_packet(**bound_goal) + assert packet["runnable_todo_ids"] == ["todo_existing"] + assert [step["id"] for step in packet["ordered_steps"]] == [ + "compare_planned_todos_with_frontier", + "apply_todo_delta", + ] + assert all(item["todo_id"] != "todo_peer" for item in packet["existing_todos"]) + + +def test_unknown_identity_rejected_without_creating_it(bound_goal): + with pytest.raises(ValueError, match="not registered"): + build_task_planning_packet(**(bound_goal | {"agent_id": "unknown"})) + + +def test_full_frontier_and_live_blockers_ignore_display_order_and_terminal_items(bound_goal): + state = bound_goal["project"] / "state.md" + work = [ + f"- [ ] [P0] Task {i}.\n" + f" " + for i in range(40) + ] + gates = [ + f"- [{'x' if status == 'done' else ' '}] [P0] Gate {status}.\n" + f" " + for status in ("open", "blocked", "done", "deferred") + ] + for ordered in (work, list(reversed(work))): + state.write_text("# Active Goal State\n\n## Agent Todos\n" + "\n".join(ordered) + + "\n\n## User Todos\n" + "\n".join(gates) + "\n") + packet = build_task_planning_packet(**bound_goal) + assert set(packet["runnable_todo_ids"]) == {f"todo_work_{i}" for i in range(40)} + assert set(packet["blocking_todo_ids"]) == {"todo_gate_open", "todo_gate_blocked"} + + +def test_public_cli_returns_a_read_only_checkpoint_and_rejects_execution(bound_goal): + command = [ + "--format", + "json", + "--registry", + str(bound_goal["registry_path"]), + "--runtime-root", + bound_goal["runtime_root_arg"], + "todo", + "plan", + "--goal-id", + bound_goal["goal_id"], + "--agent-id", + bound_goal["agent_id"], + "--project", + str(bound_goal["project"]), + "--text", + bound_goal["text"], + ] + output = io.StringIO() + with contextlib.redirect_stdout(output): + assert main(command) == 0 + packet = json.loads(output.getvalue()) + assert packet["read_only"] and packet["dry_run"] + assert packet["runnable_todo_ids"] == [] + output = io.StringIO() + with contextlib.redirect_stdout(output): + assert main(command + ["--execute"]) == 1 + assert "unsupported" in json.loads(output.getvalue())["error"] + assert not list(Path(bound_goal["runtime_root_arg"]).rglob("*rollout*")) diff --git a/tests/control_plane_ts/turn_envelope.test.ts b/tests/control_plane_ts/turn_envelope.test.ts index a55fff3edc..0cdaafd091 100644 --- a/tests/control_plane_ts/turn_envelope.test.ts +++ b/tests/control_plane_ts/turn_envelope.test.ts @@ -72,6 +72,36 @@ const protocolActionFields = { agent_action: "advance one bounded segment", }; +test("Turn preserves checkpointed scope approval without lifting other gates", () => { + const source = payload(); + const scope = source.goal_boundary as Record; + scope.requires_parent_approval = ["write", "publish", "production-action"]; + const render = () => buildTurnEnvelope({payload: source, + protocol_action_fields: protocolActionFields, scheduler_execution_args: ""}); + const baseline = render(); + scope.checkpointed_boundary_authority = { + schema_version: "checkpointed_boundary_authority_v0", active_count: 1, + active_write_scope: ["src/**"], entries: [{source: "operator-decision"}], + }; + const approved = render(); + const boundary = approved.boundary as Record; + assert.deepEqual(boundary.checkpointed_boundary_authority, { + schema_version: "checkpointed_boundary_authority_v0", active_count: 1, + active_write_scope: ["src/**"], + }); + assert.deepEqual(boundary.requires_parent_approval, ["write", "publish", "production-action"]); + for (const inactive of [ + {schema_version: "checkpointed_boundary_authority_v0", active_count: 0, active_write_scope: []}, + {schema_version: "unknown", active_count: 1, active_write_scope: ["**"]}, + {schema_version: "checkpointed_boundary_authority_v0", active_count: 1, active_write_scope: ["x".repeat(181)]}, + ]) { + scope.checkpointed_boundary_authority = inactive; + assert.deepEqual(render().boundary, baseline.boundary); + } + delete scope.checkpointed_boundary_authority; + assert.deepEqual(render(), baseline); +}); + test("only active hook reads carry additive prompt budget through the envelope", () => { const source = payload(); const baseline = buildTurnEnvelope({ payload: source, protocol_action_fields: protocolActionFields, scheduler_execution_args: "" }); diff --git a/tests/test_loopx_turn_codex_cli.py b/tests/test_loopx_turn_codex_cli.py index 30ab25a05c..5fed62efc0 100644 --- a/tests/test_loopx_turn_codex_cli.py +++ b/tests/test_loopx_turn_codex_cli.py @@ -349,9 +349,11 @@ def test_codex_cli_prompt_isolates_subagent_instructions_to_enabled_request() -> assert "opaque evidence_refs such as artifact:child-result" in prompt +@pytest.mark.parametrize("sandbox", ["read-only", "workspace-write", "danger-full-access"]) def test_codex_cli_host_starts_then_resumes_opaque_session( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + sandbox: str, ) -> None: executable, log_path = _fake_codex(tmp_path) monkeypatch.setenv("FAKE_CODEX_LOG", str(log_path)) @@ -365,7 +367,7 @@ def test_codex_cli_host_starts_then_resumes_opaque_session( runtime_root=runtime_root, project=project, codex_bin=str(executable), - sandbox="workspace-write", + sandbox=sandbox, timeout_seconds=5, ) with pytest.raises(RuntimeError, match="binding changed after planning"): @@ -385,7 +387,7 @@ def test_codex_cli_host_starts_then_resumes_opaque_session( runtime_root=runtime_root, project=project, codex_bin=str(executable), - sandbox="workspace-write", + sandbox=sandbox, timeout_seconds=5, ) @@ -399,7 +401,7 @@ def test_codex_cli_host_starts_then_resumes_opaque_session( assert "session-fixture-0001" in argv_rows[1] resume_argv = argv_rows[1] assert resume_argv[resume_argv.index("-c") + 1] == ( - 'sandbox_mode="workspace-write"' + f'sandbox_mode="{sandbox}"' ) assert resume_argv[resume_argv.index("-C") + 1] == str(project) assert resume_argv.index("-C") < resume_argv.index("resume") @@ -822,3 +824,19 @@ def test_public_e2e_smoke_runs_n_transactions_on_one_session() -> None: "scheduler_acknowledged": False, "state_written": False, } + + +def test_checkpointed_write_approval_is_scoped_and_absent_by_default(): + request = _request() + assert "It satisfies the write approval requirement" not in _prompt(request) + request["turn_envelope"]["boundary"] = { + "requires_parent_approval": ["write", "publish", "production-action"], + "checkpointed_boundary_authority": { + "schema_version": "checkpointed_boundary_authority_v0", + "active_count": 1, + "active_write_scope": ["src/**"], + }, + } + prompt = _prompt(request) + assert "only within its active_write_scope" in prompt + assert "publish, and production actions retain their gates" in prompt