Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
aa16c47
fix(turn): reject changed task declarations before native launch
loopx-agent Nov 14, 2026
21a925c
Keep bound Turn dispatch inside the work authority lock
loopx-agent Nov 15, 2026
6028bb9
Retain saved Host sessions across pre-dispatch refusals
loopx-agent Nov 15, 2026
497a97f
Fence bound Host dispatch at the active lease deadline
loopx-agent Nov 15, 2026
ae407f2
Merge commit 'f38fae05bb9a1c2cc54a2641359224cc253d32ac' into codex/be…
loopx-agent Nov 16, 2026
e65b074
Honor the live Host lease at Benchmark dispatch
loopx-agent Nov 16, 2026
0f99d80
Integrate current main without dropping live-work or lease recovery c…
loopx-agent Nov 16, 2026
79de0cd
Carry the admitted task lease deadline through native Chat dispatch
loopx-agent Nov 16, 2026
0243ed7
Hold native Turn recovery when lease-bound dispatch timing is unconfi…
loopx-agent Nov 16, 2026
954ec7c
Keep reviewer protocol links readable by the docs publisher
loopx-agent Nov 16, 2026
25f49f6
Merge commit 'f599a60ce64fd78db80d6318c84e3af58bfc1df9' into codex/na…
loopx-agent Nov 16, 2026
b589d5d
Reconcile a durable native Host attempt before starting another Turn
loopx-agent Nov 16, 2026
8e6f980
Keep Benchmark provider dispatch bound to its original lease
loopx-agent Nov 16, 2026
ea9b0b6
Keep Benchmark recovery on its original replan and soft claim
loopx-agent Nov 16, 2026
9d58445
Merge current main into native Host admission
loopx-agent Nov 16, 2026
58c114b
Serialize Benchmark worker spawn with canonical lease release
loopx-agent Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions benchmark/runtime/RUNTIME.md
Original file line number Diff line number Diff line change
Expand Up @@ -599,9 +599,11 @@ process or admission identity. The request check reuses the existing Stop, quota
and canonical admission owners; it does not grant additional work or invent a
second quota counter. Inbound parsing, upstream connection setup and waiting for
the response do not hold the Goal/registry lock. The admitted POST dispatch has
an absolute two-second deadline, shortened by a smaller remaining phase budget.
An uncertain or partially sent request remains unknown and is not retried by
this transport.
an absolute two-second deadline, shortened by the remaining phase budget or the
active hard lease. A bound checkpoint request holds its original work lease
fence during dispatch. A final write returning after the deadline leaves the
original attempt unknown even if the provider received the body. A request with
an expired or replaced lease for the original work is refused.

Per-invocation receipts distinguish requests started, bytes fully sent, pending
responses and unknown partial sends. `sent` does not certify provider acceptance
Expand Down
19 changes: 16 additions & 3 deletions benchmark/runtime/provider_request_admission.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import uuid
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import urlsplit
from urllib.request import proxy_bypass_environment
Expand Down Expand Up @@ -59,6 +60,11 @@ def __init__(self, env, session_wake, receipt, wake, *, timeout=None, original_q
captured_session = SimpleNamespace(goal_ref=copy.deepcopy(session_wake.goal_ref))
check = heartbeat_launch_check(captured_env, captured_session,
original_quota=copy.deepcopy(original_quota))
preparation = original_quota.get("execution_preparation") if original_quota is not None else None
launch_scope = {}
if isinstance(preparation, dict) and preparation.get("handoff_mode") == "hard_lease":
# Keep the stored work's canonical lease fence through actual POST dispatch.
launch_scope["work_runtime_root"] = Path(captured_env["LOOPX_RUNTIME_ROOT"])
deadline = math.inf
if timeout is not None:
if not math.isfinite(timeout) or timeout < 0:
Expand All @@ -72,7 +78,7 @@ def __init__(self, env, session_wake, receipt, wake, *, timeout=None, original_q
deadline = min(deadline,
time.monotonic() + epoch - reserve - time.time())
self._invocation = _Invocation(
session_wake.admission.with_launch_decision(check), captured_env["LOOPX_TURN"],
session_wake.admission.with_launch_decision(check, **launch_scope), captured_env["LOOPX_TURN"],
session_wake.session_id, deadline)
self._env = captured_env
self._process = None
Expand Down Expand Up @@ -149,13 +155,20 @@ def lifetime(self):
self._require_live()
entered = False
try:
with self._invocation.admission.launch_lifetime(operation="provider_request_dispatch"):
with self._invocation.admission.launch_lifetime(operation="provider_request_dispatch") as expires_at_ms:
entered = True
with self._lock:
# Authority/registry contention can outlive the process or
# phase. Revalidate after every wait and before any POST.
self._require_live()
yield min(time.monotonic() + 2, self._invocation.deadline)
now = time.monotonic()
deadline = min(now + 2, self._invocation.deadline)
if expires_at_ms is not None:
remaining = expires_at_ms / 1000 - time.time()
if remaining <= 0:
self._reject("goal_launch_not_allowed")
deadline = min(deadline, now + remaining)

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 最后一段 POST 过期后仍记为正常发送 Benchmark 的 provider admission 已读到有效的 hard lease。网络发送线程在为最后一个 POST body 分片设置 socket timeout 后暂停,等原租约过期才调用真实 send。 ProviderRequestAdmission.lifetime 会把现有 deadline 收紧到租约。_dispatch_bounded 在最后一次 send 前设置相对 socket timeout,真实调用返回后缺少绝对 deadline 核对,外层把 forwarded 设为 true。 File 和 SQLite 的当前提交分别把 socket timeout 收紧至 1.254 秒和 1.314 秒,loopback 上游仍在租约到期约 34 ms 和 31 ms 后收到完整 body 并返回 HTTP 200。两个后端都记录 sent=1、unknown=0,基线同样正常转发。夹具额度扣减为零。 在现有 gateway 检查最后一次真实写入是否在绝对 deadline 内完成。可能已经送达时保留原 invocation 的 unknown 观察,避免返回正常成功或自动重发。 租约截止时间 · 实际发送 · 发送观察

yield deadline
except ProviderGatewayRequestRejected:
raise
except FirstPartyHostRuntimeRejected as error:
Expand Down
12 changes: 10 additions & 2 deletions benchmark/runtime/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,13 +145,21 @@ def child_process(command: list[str], *, env: dict[str, str], stdout, stderr,
if check_current_launch is not None and goal_admission is None:
raise ValueError("current Host launch check requires its original Goal admission")
if check_current_launch is not None:
goal_admission = goal_admission.with_launch_decision(check_current_launch)
# The live heartbeat can select hard-lease work even on a new Turn.
# Use its canonical writer fence for the decision and actual spawn.
goal_admission = goal_admission.with_launch_decision(
check_current_launch, work_runtime_root=Path(env["LOOPX_RUNTIME_ROOT"]),
)
# Session selection may precede rendering/capture. Serialize only actual
# launch with the existing Goal lifetime owner, never the whole model run.
with (goal_admission.launch_lifetime(operation="codex_agent_host_launch")
if goal_admission is not None else nullcontext()):
if goal_admission is not None else nullcontext()) as expires_at_ms:

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 在已绑定 checkpoint 启动时持有租约 writer fence 恢复已绑定 hard lease 的 Benchmark checkpoint。Gateway 保持默认关闭,在 live admission 和 expiry 检查已经通过后暂停 Popen,同一 Todo 的 canonical owner 从另一线程经 CLI 释放租约。 checkpoint_wake.py 把原 quota 传给 native_child_process。child_process 在绑定 live callback 时省略 work_runtime_root。launch_lifetime 因此只取 registry 锁,task-lease release 使用的 canonical 工作锁仍可进入。 File 和 SQLite 均观察到 canonical lease released → native Popen → checkpoint input received。CLI 已返回 released,真实脚本子进程仍启动并读到这次 continuation 的完整输入。原回执各保留一份,测试扣额为零。 通过 native_child_process 把已绑定 hard lease 的 canonical work runtime root 传给 launcher 的 with_launch_decision,复用同一个短期 writer fence 直到 Popen。再读取原 work/lease,覆盖 release 先到时拒绝、在 live 检查后到时等到派发完成的两种时序。 checkpoint 传递 · writer fence

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 为已绑定 checkpoint 的 native spawn 保留原租约写锁 恢复已绑定 hard lease 的 Benchmark checkpoint。Gateway 保持默认关闭,在 live admission 和时钟检查通过、启动进程之前暂停 Popen,另一线程用真实 task-lease release 释放同一 Todo 租约。 CheckpointWake.run 给 native_child_process 传原 quota。child_process 调用 with_launch_decision 时省略 work_runtime_root。launch_lifetime 因此只持 registry 锁,canonical lease release 可在同一 live check 与 Popen 之间提交。 File 和 SQLite 均观察到先 released,后真实 Popen,再收到完整 checkpoint stdin。只给同一个 guard 传原 work root 的对照恢复了 writer 排序,两种后端均通过。 为已经绑定的 native_child_process 传入原 canonical work runtime root,使 live recheck 和 Popen 使用同一 lease writer fence。派发输入前还须处理已经跨期的 spawn。 原 checkpoint 路由 · 现有写锁

# The child can write a native event before Popen returns.
launch_started_at_unix = time.time()
if expires_at_ms is not None and launch_started_at_unix * 1000 >= expires_at_ms:

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 在启动结果跨过硬租约期限时拒绝投递 checkpoint 输入 保留同一个 hard lease,在 worker.child_process 的最后时钟检查通过、真正调用 Popen 之前暂停到到期。Gateway 使用默认关闭。 目前唯一的 spawn deadline 比较在 Popen 前。CheckpointWake._write_input 接到子进程后只核对 wake timeout,沿实际 pipe 传入原内容。 在 File 与 SQLite 中,真实 Popen 分别在原期限后约 31 ms 和 34 ms 执行。脚本进程退出成功并写下收到的完整 checkpoint stdin。未产生额外 quota debit。 在实际 Popen 返回、交出 child 或给 stdin 写入之前核对同一个绝对 deadline。期限已过时沿现有清理路径退出。预检合法窗口与临界后拒绝都需覆盖。 checkpoint pipe

from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostLaunchRejected

raise FirstPartyHostLaunchRejected("goal_launch_not_allowed")
process = subprocess.Popen(

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 在子进程启动跨过硬租约期限后拦住原工作输入 在已绑定 hard lease 的 checkpoint 上打开默认 native 派发。保留 live check 通过,将同一个 lease 续到可控的短期限,在真正 Popen 前暂停至期限后。 child_process 在 Popen 前比较一次硬租约期限。锁保护显式 writer 的顺序。Popen 返回后直接交出 child,CheckpointWake._write_input 核对 wake timeout 后写入原 stdin。 File 和 SQLite 的实际 Popen 调用分别在原期限后约 30 ms 和 32 ms,真实 child 收到完整 synthetic checkpoint input 并成功退出。两个后端都保有一份原 heartbeat receipt,额外 quota debit 为零。 在真实 Popen 返回后、交出 child 之前检查原 deadline。已知尚未交付原输入的情况沿现有资源清理退出。若 child 已发生不可确认效果,沿现有 unknown 路径核对。 原输入边界

command,
cwd=env["LOOPX_PROJECT"],
Expand Down
230 changes: 228 additions & 2 deletions benchmark/tests/test_provider_request_admission.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,18 @@

import pytest

from benchmark.runtime import worker
from benchmark.runtime.provider_request_admission import (
ProviderRequestAdmission,
request_admission_enabled,
)
from loopx.capabilities.benchmark_toolkit import provider_gateway
from loopx.control_plane.coordination.shadow_management import shadow_maintenance_lock_target
from loopx.control_plane.goals import activation_service, first_party_host_admission
from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission
from loopx.control_plane.projects.registry_codec import project_registry_transaction
from loopx.control_plane.work_items.task_lease import lease_expires_at
from loopx.file_lock import LockAcquireTimeoutError, exclusive_mutation_file_lock
from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime
from tests.control_plane import test_quota_settlement_cli as quota_fixture

Expand All @@ -44,7 +48,7 @@ class _Authority:
api_key: str = "fixture-provider-key"

def invocation(self, upstream, name, *, timeout=None, receipt=None, wake=None, turn_id=None,
extra_env=None):
extra_env=None, original_quota=None):
if wake is None:
wake = self.root / name
wake.mkdir()
Expand All @@ -68,7 +72,9 @@ def invocation(self, upstream, name, *, timeout=None, receipt=None, wake=None, t
)
if receipt is None:
receipt = {}
guard = ProviderRequestAdmission(env, session, receipt, wake, timeout=timeout)
guard = ProviderRequestAdmission(
env, session, receipt, wake, timeout=timeout, original_quota=original_quota,
)
return guard, receipt, wake

def stop(self):
Expand Down Expand Up @@ -463,6 +469,226 @@ def test_real_backpressure_bounds_writer_wait_and_records_unknown_acceptance(aut
assert len(upstream.rows) == 1, "unknown acceptance must not retry the POST"


@pytest.fixture(params=["file", "sqlite"])
def held_provider_task(tmp_path, monkeypatch, request):
isolate_sqlite_runtime(tmp_path, monkeypatch)
project, runtime, registry = quota_fixture._write_fixture(tmp_path)
quota_fixture._initialize_fixture_authority(
project, runtime, provider=request.param, handoff_mode="hard_lease",
)
authority = _Authority(project, runtime, registry, tmp_path)
goal, agent, todo, turn = quota_fixture.GOAL_ID, quota_fixture.AGENT_ID, quota_fixture.TODO_ID, quota_fixture.TURN_ID
lease_scope = ["--goal-id", goal, "--todo-id", todo, "--owner", agent]

def lease(action, version, ttl=None, *, key="provider-original-lease"):
ttl_args = [] if ttl is None else ["--ttl-seconds", str(ttl)]
code, result = quota_fixture._run_cli(
registry, runtime, "task-lease", action, *lease_scope,
"--idempotency-key", key, "--expected-version", str(version), *ttl_args, cwd=project,
)
assert code == 0 and result["ok"], result
return result["lease"]

original = lease("acquire", 0, 60)
code, quota = quota_fixture._run_cli(
registry, runtime, "quota", "should-run", "--goal-id", goal, "--agent-id", agent,
"--todo-id", todo, "--turn-instance-id", turn, "--runtime-profile", "generic_cli",
"--complete-turn", cwd=project,
)
assert code == 0 and quota["should_run"], quota
assert quota["interaction_contract"]["cli_channel"]["execution_preparation"]["handoff_mode"] == "hard_lease"
return authority, quota, original, lease


def test_provider_request_keeps_original_soft_claim_turn(authority):
turn = quota_fixture.TURN_ID
with _upstream() as upstream, _process() as process:
guard, _, wake = authority.invocation(upstream.base_url, "soft-claim", turn_id=turn)
with guard.serve(["native", "-"]):
guard.started(process)
# The native session first binds work in its own Turn. The gateway
# was already created and must consume that same live authority.
code, quota = quota_fixture._run_cli(
authority.registry, authority.runtime, "quota", "should-run",
"--goal-id", quota_fixture.GOAL_ID, "--agent-id", quota_fixture.AGENT_ID,
"--todo-id", quota_fixture.TODO_ID, "--turn-instance-id", turn,
"--runtime-profile", "generic_cli", "--complete-turn", cwd=authority.project,
)
identity = quota["heartbeat_receipt"]["settlement_identity"]
assert code == 0 and quota["normal_delivery_allowed"] is True
assert identity["todo_id"] == quota_fixture.TODO_ID
assert "execution_preparation" not in quota

status, body = _post(guard.base_url)
assert status == 200, body
event, = _finished(wake)
assert event["turn_id"] == turn and event["forwarded"] is True
assert len(upstream.rows) == 1
assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1
assert quota_fixture._spend_run_count(authority.runtime) == 0


@pytest.mark.parametrize("released_before_recheck", [False, True])
def test_bound_native_worker_serializes_original_lease_release(held_provider_task, monkeypatch, released_before_recheck):
authority, quota, original, lease = held_provider_task
turn = quota_fixture.TURN_ID
marker = authority.root / "checkpoint-input"
command = [sys.executable, "-c",
f"import sys; from pathlib import Path; Path({str(marker)!r}).write_text(sys.stdin.read())"]
env = {
"LOOPX_PROJECT": str(authority.project), "LOOPX_RUNTIME_ROOT": str(authority.runtime),
"LOOPX_GOAL_ID": quota_fixture.GOAL_ID, "LOOPX_AGENT_ID": quota_fixture.AGENT_ID,
"LOOPX_TURN": turn,
}
session = SimpleNamespace(
admission=FirstPartyHostGoalAdmission.for_plan(
registry_path=authority.registry, goal_id=quota_fixture.GOAL_ID, planned_goal_ref=None,
),
goal_ref=None,
session_id="fixture-original",
)
launch = {
"env": env, "stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL,
"session_wake": session, "receipt": {}, "wake": authority.root, "original_quota": quota,
}
real_popen = subprocess.Popen
mutations = []

def start_with_release(args, *extra, **kwargs):
if args == command:
target = shadow_maintenance_lock_target(authority.runtime, quota_fixture.GOAL_ID)
with pytest.raises(LockAcquireTimeoutError):
with exclusive_mutation_file_lock(target, timeout_seconds=0):
pytest.fail("the original lease could be released before the child starts")
mutations.append(_async(lambda: lease("release", original["version"])))
return real_popen(args, *extra, **kwargs)

monkeypatch.setattr(worker.subprocess, "Popen", start_with_release)
if released_before_recheck:
assert lease("release", original["version"])["status"] == "released"
with pytest.raises(first_party_host_admission.FirstPartyHostRuntimeRejected,
match="goal_launch_not_allowed"):
with worker.native_child_process(command, **launch):
pytest.fail("the released lease started another worker")
assert not marker.exists() and not mutations
else:
with worker.native_child_process(command, **launch) as process:
process.communicate(input="original checkpoint input", timeout=5)
assert process.returncode == 0
assert marker.read_text() == "original checkpoint input"
released, = mutations
assert released.result(timeout=10)["status"] == "released"
assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1
assert quota_fixture._spend_run_count(authority.runtime) == 0


@pytest.mark.parametrize("retry_lease", ["expired", "replacement"])
def test_final_provider_write_crossing_hard_lease_preserves_unknown_attempt(
held_provider_task, monkeypatch, retry_lease,
):
authority, quota, original, lease = held_provider_task
turn = quota_fixture.TURN_ID
suffix = b"final-provider-body"
body = b"x" * (64 * 1024) + suffix
accepted_write_at = []
original_connection = provider_gateway._upstream_connection

def delay_final_write(*args, **kwargs):
connection = original_connection(*args, **kwargs)
send = connection.send

def send_after_preemption(data):
if accepted_write_at or not isinstance(data, memoryview) or data != suffix:
return send(data)
assert 0 < connection.sock.gettimeout() < 2, "the hard lease must shorten the gateway send limit"
now = time.time()
assert now < expires_at
time.sleep(expires_at - now + .03)
send(data)
accepted_write_at.append(time.time())

connection.send = send_after_preemption
return connection

monkeypatch.setattr(provider_gateway, "_upstream_connection", delay_final_write)
with _upstream() as upstream, _process() as process:
guard, receipt, wake = authority.invocation(
upstream.base_url, "lease-expiry", turn_id=turn, original_quota=quota,
)
with guard.serve(["native", "-"]):
guard.started(process)
renewed = lease("renew", original["version"], 2)
assert renewed["lease_epoch"] == original["lease_epoch"]
expires_at = lease_expires_at(renewed).timestamp()
status, error = _post(guard.base_url, body)
assert upstream.body.wait(5) and upstream.rows[0]["body"] == body
assert len(accepted_write_at) == 1 and accepted_write_at[0] >= expires_at
assert status == 502 and error["error"]["code"] == "provider_gateway_upstream_unavailable", error
event, = _finished(wake)
summary, = receipt["provider_request_invocations"]
assert event["turn_id"] == turn and event["invocation_id"] == summary["invocation_id"]
assert event["forwarded"] is None and "response_status" not in event and "refusal_code" not in event
assert summary["requests"] == summary["unknown"] == 1 and summary["pending"] == summary["sent"] == 0
if retry_lease == "replacement":
replacement = lease("acquire", renewed["version"], 60, key="provider-replacement-lease")
assert replacement["lease_epoch"] > renewed["lease_epoch"]
status, denied = _post(guard.base_url, body)
assert status == 409 and denied["error"]["code"] == "goal_launch_not_allowed", denied
_, rejected = _finished(wake, count=2)
assert rejected["invocation_id"] == event["invocation_id"] and rejected["forwarded"] is False
assert len(upstream.rows) == 1
assert summary["unknown"] == summary["refused"] == 1 and summary["sent"] == 0
assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1
assert quota_fixture._spend_run_count(authority.runtime) == 0


def test_bound_provider_send_holds_lease_writer_until_dispatch(held_provider_task, monkeypatch):
authority, quota, original, lease = held_provider_task
waiting, resume = threading.Event(), threading.Event()
original_connection = provider_gateway._upstream_connection

def connection_at_send(*args, **kwargs):
connection = original_connection(*args, **kwargs)
send = connection.send

def pause_body(data):
if isinstance(data, memoryview):
waiting.set()
assert resume.wait(4), "test did not release the admitted POST"
return send(data)

connection.send = pause_body
return connection

monkeypatch.setattr(provider_gateway, "_upstream_connection", connection_at_send)
with _upstream(hold_response=True) as upstream, _process() as process:
guard, _, wake = authority.invocation(
upstream.base_url, "lease-release", turn_id=quota_fixture.TURN_ID, original_quota=quota,
)
with guard.serve(["native", "-"]):
guard.started(process)
response = _async(lambda: _post(guard.base_url))
try:
assert waiting.wait(5) and not upstream.body.is_set()
target = shadow_maintenance_lock_target(authority.runtime, quota_fixture.GOAL_ID)
with pytest.raises(LockAcquireTimeoutError):
with exclusive_mutation_file_lock(target, timeout_seconds=0):
pass
mutation = _async(lambda: lease("release", original["version"]))
resume.set()
released = mutation.result(timeout=5)
assert released["status"] == "released"
assert upstream.body.wait(5) and upstream.rows[0]["body"] == b"{}"
assert not response.done()
finally:
resume.set()
upstream.respond.set()
assert response.result(timeout=5)[0] == 200
assert _finished(wake)[0]["forwarded"] is True
assert _post(guard.base_url)[0] == 409
assert len(upstream.rows) == 1 and quota_fixture._spend_run_count(authority.runtime) == 0


def test_slow_response_does_not_hold_goal_writer(authority):
with _upstream(hold_response=True) as upstream, _process() as process:
guard, receipt, wake = authority.invocation(upstream.base_url, "slow-response")
Expand Down
Loading
Loading