Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions benchmark/runtime/RUNTIME.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,30 @@ agents:
replan_after_turns: 6
```

### Official-style Codex Stop hook

Set `execution_mode: plain` and `official_stop_hook: true` in
`BenchmarkCodex` kwargs for a native Codex baseline. One
`codex exec` owns the conversation. On attempted completion, the native Stop
hook asks Codex to continue working, using the same instruction as EdgeBench's
SForge official profile. The worker uses the remaining phase clock through
native execution. The task controller may stop it earlier when the task's
original final submission is accepted.

The runner stages only its own hook in the trial's Codex home and trusts the
exact handler identity discovered by that Codex binary. Both discovery requests
share the worker deadline when one is declared. With no worker or phase deadline,
discovery retains its bounded RPC timeouts while model work has no independent
cutoff; the task controller still owns submission and cancellation. If the phase expires during discovery, the wake
records budget exhaustion and no model launch. Earlier discovery failures
still fail the trial before inference. Install and trajectory receipts record
the opt-in. The wake receipt records successful hook discovery.
Use the same pinned task, model, reasoning effort, time and evaluator in
comparison arms. The benchmark adapter still owns feedback, final submission,
and scoring. EdgeBench's SForge auto-resume remains a provider-specific part of
its official profile. Use a fresh trial with the option omitted for a
single-pass Codex baseline.

For an explicitly selected **heartbeat-only** context experiment, add
`turn_envelope: true` to `kwargs` (default `false`). Each wake creates a private
capture root and asks the product renderer for a short TurnEnvelope dispatcher.
Expand Down
4 changes: 4 additions & 0 deletions benchmark/runtime/codex.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ class Execution:
context_delivery: str | None = None
native_child_limit: int | None = None
complete_turn: bool = False
official_stop_hook: bool = False

def __post_init__(self) -> None:
if self.native_child_limit is not None and (
Expand All @@ -48,6 +49,8 @@ def __post_init__(self) -> None:
raise ValueError("turn_envelope requires heartbeat execution and a boolean opt-in")
if type(self.complete_turn) is not bool or (self.complete_turn and (self.mode != "heartbeat" or self.turn_envelope)):
raise ValueError("complete_turn requires heartbeat execution without legacy turn_envelope")
if type(self.official_stop_hook) is not bool or (self.official_stop_hook and self.mode != "plain"):
raise ValueError("official_stop_hook requires plain execution and a boolean opt-in")
if self.mode not in MODES or self.context not in CONTEXTS:
raise ValueError("unsupported execution mode or iteration context")
if self.task_entry is None:
Expand Down Expand Up @@ -208,6 +211,7 @@ def codex_settings(
"[features]",
f"goals = {str(execution.native_goal).lower()}",
"unified_exec = true",
*(["hooks = true"] if execution.official_stop_hook else []),
*(["[agents]", f"max_concurrent_threads_per_session = {execution.native_child_limit}"]
if execution.native_child_limit is not None else []),
"[memories]",
Expand Down
56 changes: 4 additions & 52 deletions benchmark/runtime/context_delivery.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import shlex
from pathlib import Path

from .native_hooks import prepare_native_hooks


def preflight_context_instruction(env: dict[str, str], path: Path, digest: str, turn_id: str, *,
guide_in_host: bool = False) -> str:
Expand Down Expand Up @@ -44,7 +46,6 @@ def preflight_context_instruction(env: dict[str, str], path: Path, digest: str,

def prepare_context_delivery(env: dict[str, str], home: Path) -> None:
from loopx.control_plane.turn_driver.codex_context import codex_context_configuration
from loopx.control_plane.turn_driver.codex_stdio import StdioNativeGoalTransport

root = home / "loopx-context"
root.mkdir(mode=0o700, exist_ok=True)
Expand All @@ -54,59 +55,10 @@ def prepare_context_delivery(env: dict[str, str], home: Path) -> None:
env["LOOPX_CODEX_CONTEXT_DIR"] = str(root)
configuration = codex_context_configuration(str(root), goal_id=env["LOOPX_GOAL_ID"],
agent_id=env["LOOPX_AGENT_ID"], scenario="codex_benchmark_heartbeat", cli_argv=[env["LOOPX_CLI"], "--cli-bin", env["LOOPX_CLI"]])
content = json.dumps({"hooks": configuration["hooks"]})
path = home / "hooks.json"
if path.is_symlink() or (path.exists() and path.read_text() != content):
raise ValueError("native context hooks changed within a trial; use a new trial")
if not path.exists():
path.write_text(content)
path.chmod(0o600)
# The native host is the hash/key owner. Do not duplicate its hashing or
# bypass trust for unrelated workspace hooks. Discovery launches no model.
with (home / "context-hooks-stderr.log").open("a") as stderr:
with StdioNativeGoalTransport.spawn(
# This process only reads native hook identities. Catalog/plugin
# startup can clone repositories into an otherwise private home;
# it is unrelated to discovery and must not affect solver config.
[env["CODEX_BIN"], "-c", "features.plugins=false",
"-c", "features.remote_plugin=false", "-c", "features.plugin_sharing=false",
"app-server", "--listen", "stdio://"],
cwd=env["LOOPX_PROJECT"], env=env, stderr=stderr, response_timeout_sec=10,
) as native:
native.request("initialize", {"clientInfo": {"name": "loopx_context_transport", "version": "0.1.0"},
"capabilities": {"experimentalApi": True}})
native.notify("initialized", {})
listed = native.request("hooks/list", {"cwds": [env["LOOPX_PROJECT"]]})
hooks = [hook for entry in listed["data"] for hook in entry["hooks"]
if hook.get("sourcePath") == str(path.resolve())]
# Match the generated handler multiset, not just its size. Native discovery
# can reorder it; duplicates, disabled handlers and substitutions cannot
# stand in for an expected event/command/settings combination.
expected = [{"eventName": event[0].lower() + event[1:],
"handlerType": handler["type"], "command": handler["command"],
"matcher": group.get("matcher"), "timeoutSec": handler.get("timeout"),
"async": handler.get("async", False)}
for event, groups in configuration["hooks"].items()
for group in groups for handler in group["hooks"]]
if not expected or len(hooks) != len(expected) or path.read_text() != content:
raise ValueError("native host did not discover the exact trial context hooks")
entries, keys = [], set()
for hook in hooks:
actual = {field: hook.get(field) for field in expected[0]}
if hook.get("enabled") is not True or type(hook.get("async")) is not bool or actual not in expected:
raise ValueError("native host did not discover the exact trial context hooks")
expected.remove(actual)
key, digest = hook.get("key"), hook.get("currentHash")
if not isinstance(key, str) or not key or key in keys or not isinstance(digest, str) or not digest:
raise ValueError("native host returned no unique context-hook trust identity")
keys.add(key)
entries.append(f"{json.dumps(key)} = {{ trusted_hash = {json.dumps(digest)} }}")
# CLI override keys are dot-separated; native keys contain paths/dots.
# Carry the entire table as TOML so native keys remain literal strings.
native_args = prepare_native_hooks(env, home, configuration["hooks"], purpose="context")
# This isolated trial's native developer instructions survive compaction;
# a failed SessionStart handler cannot erase the read-only recovery route.
native_args = ["-c", "hooks.state={" + ",".join(entries) + "}",
"-c", "developer_instructions=" + json.dumps(configuration["developer_instructions"])]
native_args += ["-c", "developer_instructions=" + json.dumps(configuration["developer_instructions"])]
native_args += [argument for key, value in configuration["config_overrides"].items()
for argument in ("-c", key + "=" + json.dumps(value))]
env["LOOPX_CODEX_CONTEXT_TRUST_ARGS"] = json.dumps(native_args)
12 changes: 9 additions & 3 deletions benchmark/runtime/harbor.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ def __init__(
agent_id=_AGENT_ID,
defer_codex_home=False,
complete_turn=False,
official_stop_hook=False,
**kwargs,
):
if "research_loop" in kwargs:
Expand All @@ -105,6 +106,7 @@ def __init__(
context_delivery,
native_child_limit,
complete_turn=complete_turn,
official_stop_hook=official_stop_hook,
)
if type(provider_request_admission) is not bool:
raise ValueError("provider_request_admission must be a boolean opt-in")
Expand Down Expand Up @@ -414,6 +416,7 @@ async def install(self, environment: BaseEnvironment) -> None:
if self.execution_observation is not None else {}),
**self._turn_envelope_receipt(),
**self._planning_receipt(),
**({"official_stop_hook": True} if self.execution.official_stop_hook else {}),
**({"provider_request_admission": "on"} if self.provider_request_admission else {}),
"home_scope": "trial",
"loopx_usage_ping_enabled": False,
Expand Down Expand Up @@ -695,6 +698,7 @@ def _worker_env(self, *, cwd: str) -> dict[str, str]:
**({"LOOPX_TURN_ENVELOPE": "1"} if self.execution.turn_envelope else {}),
"LOOPX_COMPLETE_TURN": "1" if self.execution.complete_turn else "0",
"LOOPX_CONTEXT_DELIVERY": self.execution.context_delivery,
"LOOPX_OFFICIAL_STOP_HOOK": "1" if self.execution.official_stop_hook else "0",
# A trial's explicit None must not inherit a container opt-in.
"LOOPX_NATIVE_CHILD_LIMIT": (
str(self.execution.native_child_limit)
Expand Down Expand Up @@ -831,6 +835,7 @@ def token_delta(key: str) -> int | None:
"task_entry": self.execution.task_entry,
**self._turn_envelope_receipt(),
**self._planning_receipt(),
**({"official_stop_hook": True} if self.execution.official_stop_hook else {}),
**({"provider_request_admission": "on"} if self.provider_request_admission else {}),
"home_scope": "trial",
**self._replan_receipt(),
Expand Down Expand Up @@ -897,7 +902,7 @@ async def run(
if entry["status"] == "blocked":
return
remaining = int(deadline - time.monotonic())
reserve = phase_work_reserve(self.execution.mode)
reserve = phase_work_reserve(self.execution.mode, official_stop_hook=self.execution.official_stop_hook)
if remaining <= reserve:
raise TimeoutError("Task budget exhausted before execution handoff")
# Planning, admission and work share one absolute phase clock.
Expand All @@ -913,11 +918,12 @@ async def run(
resume_budget_stop=self._phase_number > 1,
)
phase_log = f"/logs/agent/worker-phase-{self._phase_number:03d}.log"
supervisor_timeout = remaining + (30 if self.execution.official_stop_hook else 0)
shell = (
"set +e; "
# Use the task environment's clock, including remote backends.
f"export LOOPX_PHASE_DEADLINE_EPOCH=$(( $(date +%s) + {remaining} )); "
f"timeout --signal=TERM --kill-after=30 {remaining}s "
f"timeout --signal=TERM --kill-after=30 {supervisor_timeout}s "
f"{shlex.join(command)} >> {shlex.quote(phase_log)} 2>&1; "
"rc=$?; "
# Budget exhaustion retains partial artifacts for native scoring.
Expand All @@ -928,7 +934,7 @@ async def run(
command=shell,
env=env,
cwd=cwd,
timeout_sec=remaining + 60,
timeout_sec=supervisor_timeout + 60,
)
finally:
# Remote Harbor backends download logs after run(). Read them now
Expand Down
75 changes: 75 additions & 0 deletions benchmark/runtime/native_hooks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
"""Stage and trust only the benchmark's generated native Codex hooks."""

from __future__ import annotations

import json
import shlex
from functools import partial
from pathlib import Path


OFFICIAL_STOP_HOOK_REASON = "Do not stop. Continue working on the implementation."


def prepare_official_stop_hook(env: dict[str, str], home: Path, *, deadline: float | None = None) -> list[str]:
# Put the entire handler in the declaration so native trust covers the
# command body as well as the path to the hook declaration.
response = shlex.quote(json.dumps({"decision": "block", "reason": OFFICIAL_STOP_HOOK_REASON}))
command = rf"cat >/dev/null && printf '%s\n' {response}"
return prepare_native_hooks(env, home, {
"Stop": [{"hooks": [{"type": "command", "command": command, "timeout": 30}]}],
}, purpose="official-stop", deadline=deadline)


def prepare_native_hooks(env: dict[str, str], home: Path, configuration: dict, *,
purpose: str, deadline: float | None = None) -> list[str]:
from loopx.control_plane.turn_driver.codex_stdio import StdioNativeGoalTransport

content = json.dumps({"hooks": configuration})
path = home / "hooks.json"
if path.is_symlink() or (path.exists() and path.read_text() != content):
raise ValueError(f"native {purpose} hooks changed within a trial; use a new trial")
if not path.exists():
path.write_text(content)
path.chmod(0o600)
# The native host owns keys and hashes. Discovery launches no model, and
# discovery-only plugin overrides leave the solver configuration intact.
with (home / f"{purpose}-hooks-stderr.log").open("a") as stderr:
with StdioNativeGoalTransport.spawn(
[env["CODEX_BIN"], "-c", "features.plugins=false",
"-c", "features.remote_plugin=false", "-c", "features.plugin_sharing=false",
"app-server", "--listen", "stdio://"],
cwd=env["LOOPX_PROJECT"], env=env, stderr=stderr, response_timeout_sec=10,

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Stop hook 的初始化会越过阶段截止时间并返回普通失败。 阶段只剩 1.5 秒时,真实 worker 的 initialize 等约 10 秒后失败。请把同一个阶段截止时刻传给 initialize 与 hooks/list,并保留阶段耗尽的退出和回执。

) as native:
request = native.request if deadline is None else partial(native.request, deadline=deadline)

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 分开锁定版本时 Stop hook 在 initialize 前失败。 用实际 BenchmarkCodex._stage_source 把产品锁在 main、runner 锁在此提交后,产品 transport 仍是旧签名。开启 official_stop_hook 的 worker 在 initialize 调用报 unexpected keyword argument,退出 1,模型请求为零。关闭同一组合正常。请在保持两端 pin 独立的前提下提供兼容的限时入口,或在安装时清楚拒绝不兼容版本,并加分锁 staging 回归。

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 在产品与 runner 独立固定版本时保留可运行的发现接口。按已记录的 Harbor staging:产品锁当前 main,runner 锁本 head,plain 开启 official_stop_hook。实际仅覆盖 benchmark/runtime,旧产品 request 不接受 deadline,因此第一次 initialize 前即 TypeError,零 RPC/exec;同锁四对照中的 on 才成功。请在可随 runner 交付的兼容接口处理同一绝对预算,或在安装时明确检查产品能力,保留产品 pin;用 runtime-only 归档覆盖的 split-on/off 和同锁 on/off 复验。

request("initialize", {"clientInfo": {"name": "loopx_context_transport", "version": "0.1.0"},
"capabilities": {"experimentalApi": True}})
native.notify("initialized", {})
listed = request("hooks/list", {"cwds": [env["LOOPX_PROJECT"]]})
hooks = [hook for entry in listed["data"] for hook in entry["hooks"]
if hook.get("sourcePath") == str(path.resolve())]
# Discovery can reorder handlers. Counts alone cannot rule out
# duplicates, disabled handlers or substitutions.
expected = [{"eventName": event[0].lower() + event[1:],
"handlerType": handler["type"], "command": handler["command"],
"matcher": group.get("matcher"), "timeoutSec": handler.get("timeout"),
"async": handler.get("async", False)}
for event, groups in configuration.items()
for group in groups for handler in group["hooks"]]
mismatch = f"native host did not discover the exact trial {purpose} hooks"
if not expected or len(hooks) != len(expected) or path.read_text() != content:
raise ValueError(mismatch)
entries, keys = [], set()
for hook in hooks:
actual = {field: hook.get(field) for field in expected[0]}
if hook.get("enabled") is not True or type(hook.get("async")) is not bool or actual not in expected:
raise ValueError(mismatch)
expected.remove(actual)
key, digest = hook.get("key"), hook.get("currentHash")
if not isinstance(key, str) or not key or key in keys or not isinstance(digest, str) or not digest:
raise ValueError(f"native host returned no unique {purpose}-hook trust identity")
keys.add(key)
entries.append(f"{json.dumps(key)} = {{ trusted_hash = {json.dumps(digest)} }}")
# Native keys contain dots and paths. Pass the whole table with literal
# keys instead of nesting keys via CLI dot notation.
return ["-c", "hooks.state={" + ",".join(entries) + "}"]
6 changes: 3 additions & 3 deletions benchmark/runtime/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
PHASE_WORK_RESERVE_SECONDS = 160


def phase_work_reserve(mode: str) -> int:
"""Heartbeat cleanup must not consume the remaining solver allowance."""
return 0 if mode == "heartbeat" else PHASE_WORK_RESERVE_SECONDS
def phase_work_reserve(mode: str, *, official_stop_hook: bool = False) -> int:
"""Give deadline-driven workers the full phase; bound cleanup separately."""
return 0 if mode == "heartbeat" or official_stop_hook else PHASE_WORK_RESERVE_SECONDS


def read_phase_deadline(raw: bytes) -> int | float:
Expand Down
Loading
Loading