diff --git a/benchmark/runtime/RUNTIME.md b/benchmark/runtime/RUNTIME.md index 91ae2b658d..bae1b45f6b 100644 --- a/benchmark/runtime/RUNTIME.md +++ b/benchmark/runtime/RUNTIME.md @@ -599,9 +599,11 @@ process or admission identity. The request check reuses the existing Stop, quota and canonical admission owners; it does not grant additional work or invent a second quota counter. Inbound parsing, upstream connection setup and waiting for the response do not hold the Goal/registry lock. The admitted POST dispatch has -an absolute two-second deadline, shortened by a smaller remaining phase budget. -An uncertain or partially sent request remains unknown and is not retried by -this transport. +an absolute two-second deadline, shortened by the remaining phase budget or the +active hard lease. A bound checkpoint request holds its original work lease +fence during dispatch. A final write returning after the deadline leaves the +original attempt unknown even if the provider received the body. A request with +an expired or replaced lease for the original work is refused. Per-invocation receipts distinguish requests started, bytes fully sent, pending responses and unknown partial sends. `sent` does not certify provider acceptance diff --git a/benchmark/runtime/provider_request_admission.py b/benchmark/runtime/provider_request_admission.py index de03bb9961..b3250758cf 100644 --- a/benchmark/runtime/provider_request_admission.py +++ b/benchmark/runtime/provider_request_admission.py @@ -15,6 +15,7 @@ import uuid from contextlib import contextmanager from dataclasses import dataclass +from pathlib import Path from types import SimpleNamespace from urllib.parse import urlsplit from urllib.request import proxy_bypass_environment @@ -59,6 +60,11 @@ def __init__(self, env, session_wake, receipt, wake, *, timeout=None, original_q captured_session = SimpleNamespace(goal_ref=copy.deepcopy(session_wake.goal_ref)) check = heartbeat_launch_check(captured_env, captured_session, original_quota=copy.deepcopy(original_quota)) + preparation = original_quota.get("execution_preparation") if original_quota is not None else None + launch_scope = {} + if isinstance(preparation, dict) and preparation.get("handoff_mode") == "hard_lease": + # Keep the stored work's canonical lease fence through actual POST dispatch. + launch_scope["work_runtime_root"] = Path(captured_env["LOOPX_RUNTIME_ROOT"]) deadline = math.inf if timeout is not None: if not math.isfinite(timeout) or timeout < 0: @@ -72,7 +78,7 @@ def __init__(self, env, session_wake, receipt, wake, *, timeout=None, original_q deadline = min(deadline, time.monotonic() + epoch - reserve - time.time()) self._invocation = _Invocation( - session_wake.admission.with_launch_decision(check), captured_env["LOOPX_TURN"], + session_wake.admission.with_launch_decision(check, **launch_scope), captured_env["LOOPX_TURN"], session_wake.session_id, deadline) self._env = captured_env self._process = None @@ -149,13 +155,20 @@ def lifetime(self): self._require_live() entered = False try: - with self._invocation.admission.launch_lifetime(operation="provider_request_dispatch"): + with self._invocation.admission.launch_lifetime(operation="provider_request_dispatch") as expires_at_ms: entered = True with self._lock: # Authority/registry contention can outlive the process or # phase. Revalidate after every wait and before any POST. self._require_live() - yield min(time.monotonic() + 2, self._invocation.deadline) + now = time.monotonic() + deadline = min(now + 2, self._invocation.deadline) + if expires_at_ms is not None: + remaining = expires_at_ms / 1000 - time.time() + if remaining <= 0: + self._reject("goal_launch_not_allowed") + deadline = min(deadline, now + remaining) + yield deadline except ProviderGatewayRequestRejected: raise except FirstPartyHostRuntimeRejected as error: diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index 98e33f6109..69865f6aef 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -145,13 +145,21 @@ def child_process(command: list[str], *, env: dict[str, str], stdout, stderr, if check_current_launch is not None and goal_admission is None: raise ValueError("current Host launch check requires its original Goal admission") if check_current_launch is not None: - goal_admission = goal_admission.with_launch_decision(check_current_launch) + # The live heartbeat can select hard-lease work even on a new Turn. + # Use its canonical writer fence for the decision and actual spawn. + goal_admission = goal_admission.with_launch_decision( + check_current_launch, work_runtime_root=Path(env["LOOPX_RUNTIME_ROOT"]), + ) # Session selection may precede rendering/capture. Serialize only actual # launch with the existing Goal lifetime owner, never the whole model run. with (goal_admission.launch_lifetime(operation="codex_agent_host_launch") - if goal_admission is not None else nullcontext()): + if goal_admission is not None else nullcontext()) as expires_at_ms: # The child can write a native event before Popen returns. launch_started_at_unix = time.time() + if expires_at_ms is not None and launch_started_at_unix * 1000 >= expires_at_ms: + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostLaunchRejected + + raise FirstPartyHostLaunchRejected("goal_launch_not_allowed") process = subprocess.Popen( command, cwd=env["LOOPX_PROJECT"], diff --git a/benchmark/tests/test_provider_request_admission.py b/benchmark/tests/test_provider_request_admission.py index 5325dd23bd..942f1565d0 100644 --- a/benchmark/tests/test_provider_request_admission.py +++ b/benchmark/tests/test_provider_request_admission.py @@ -23,14 +23,18 @@ import pytest +from benchmark.runtime import worker from benchmark.runtime.provider_request_admission import ( ProviderRequestAdmission, request_admission_enabled, ) from loopx.capabilities.benchmark_toolkit import provider_gateway +from loopx.control_plane.coordination.shadow_management import shadow_maintenance_lock_target from loopx.control_plane.goals import activation_service, first_party_host_admission from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission from loopx.control_plane.projects.registry_codec import project_registry_transaction +from loopx.control_plane.work_items.task_lease import lease_expires_at +from loopx.file_lock import LockAcquireTimeoutError, exclusive_mutation_file_lock from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime from tests.control_plane import test_quota_settlement_cli as quota_fixture @@ -44,7 +48,7 @@ class _Authority: api_key: str = "fixture-provider-key" def invocation(self, upstream, name, *, timeout=None, receipt=None, wake=None, turn_id=None, - extra_env=None): + extra_env=None, original_quota=None): if wake is None: wake = self.root / name wake.mkdir() @@ -68,7 +72,9 @@ def invocation(self, upstream, name, *, timeout=None, receipt=None, wake=None, t ) if receipt is None: receipt = {} - guard = ProviderRequestAdmission(env, session, receipt, wake, timeout=timeout) + guard = ProviderRequestAdmission( + env, session, receipt, wake, timeout=timeout, original_quota=original_quota, + ) return guard, receipt, wake def stop(self): @@ -463,6 +469,226 @@ def test_real_backpressure_bounds_writer_wait_and_records_unknown_acceptance(aut assert len(upstream.rows) == 1, "unknown acceptance must not retry the POST" +@pytest.fixture(params=["file", "sqlite"]) +def held_provider_task(tmp_path, monkeypatch, request): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = quota_fixture._write_fixture(tmp_path) + quota_fixture._initialize_fixture_authority( + project, runtime, provider=request.param, handoff_mode="hard_lease", + ) + authority = _Authority(project, runtime, registry, tmp_path) + goal, agent, todo, turn = quota_fixture.GOAL_ID, quota_fixture.AGENT_ID, quota_fixture.TODO_ID, quota_fixture.TURN_ID + lease_scope = ["--goal-id", goal, "--todo-id", todo, "--owner", agent] + + def lease(action, version, ttl=None, *, key="provider-original-lease"): + ttl_args = [] if ttl is None else ["--ttl-seconds", str(ttl)] + code, result = quota_fixture._run_cli( + registry, runtime, "task-lease", action, *lease_scope, + "--idempotency-key", key, "--expected-version", str(version), *ttl_args, cwd=project, + ) + assert code == 0 and result["ok"], result + return result["lease"] + + original = lease("acquire", 0, 60) + code, quota = quota_fixture._run_cli( + registry, runtime, "quota", "should-run", "--goal-id", goal, "--agent-id", agent, + "--todo-id", todo, "--turn-instance-id", turn, "--runtime-profile", "generic_cli", + "--complete-turn", cwd=project, + ) + assert code == 0 and quota["should_run"], quota + assert quota["interaction_contract"]["cli_channel"]["execution_preparation"]["handoff_mode"] == "hard_lease" + return authority, quota, original, lease + + +def test_provider_request_keeps_original_soft_claim_turn(authority): + turn = quota_fixture.TURN_ID + with _upstream() as upstream, _process() as process: + guard, _, wake = authority.invocation(upstream.base_url, "soft-claim", turn_id=turn) + with guard.serve(["native", "-"]): + guard.started(process) + # The native session first binds work in its own Turn. The gateway + # was already created and must consume that same live authority. + code, quota = quota_fixture._run_cli( + authority.registry, authority.runtime, "quota", "should-run", + "--goal-id", quota_fixture.GOAL_ID, "--agent-id", quota_fixture.AGENT_ID, + "--todo-id", quota_fixture.TODO_ID, "--turn-instance-id", turn, + "--runtime-profile", "generic_cli", "--complete-turn", cwd=authority.project, + ) + identity = quota["heartbeat_receipt"]["settlement_identity"] + assert code == 0 and quota["normal_delivery_allowed"] is True + assert identity["todo_id"] == quota_fixture.TODO_ID + assert "execution_preparation" not in quota + + status, body = _post(guard.base_url) + assert status == 200, body + event, = _finished(wake) + assert event["turn_id"] == turn and event["forwarded"] is True + assert len(upstream.rows) == 1 + assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1 + assert quota_fixture._spend_run_count(authority.runtime) == 0 + + +@pytest.mark.parametrize("released_before_recheck", [False, True]) +def test_bound_native_worker_serializes_original_lease_release(held_provider_task, monkeypatch, released_before_recheck): + authority, quota, original, lease = held_provider_task + turn = quota_fixture.TURN_ID + marker = authority.root / "checkpoint-input" + command = [sys.executable, "-c", + f"import sys; from pathlib import Path; Path({str(marker)!r}).write_text(sys.stdin.read())"] + env = { + "LOOPX_PROJECT": str(authority.project), "LOOPX_RUNTIME_ROOT": str(authority.runtime), + "LOOPX_GOAL_ID": quota_fixture.GOAL_ID, "LOOPX_AGENT_ID": quota_fixture.AGENT_ID, + "LOOPX_TURN": turn, + } + session = SimpleNamespace( + admission=FirstPartyHostGoalAdmission.for_plan( + registry_path=authority.registry, goal_id=quota_fixture.GOAL_ID, planned_goal_ref=None, + ), + goal_ref=None, + session_id="fixture-original", + ) + launch = { + "env": env, "stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL, + "session_wake": session, "receipt": {}, "wake": authority.root, "original_quota": quota, + } + real_popen = subprocess.Popen + mutations = [] + + def start_with_release(args, *extra, **kwargs): + if args == command: + target = shadow_maintenance_lock_target(authority.runtime, quota_fixture.GOAL_ID) + with pytest.raises(LockAcquireTimeoutError): + with exclusive_mutation_file_lock(target, timeout_seconds=0): + pytest.fail("the original lease could be released before the child starts") + mutations.append(_async(lambda: lease("release", original["version"]))) + return real_popen(args, *extra, **kwargs) + + monkeypatch.setattr(worker.subprocess, "Popen", start_with_release) + if released_before_recheck: + assert lease("release", original["version"])["status"] == "released" + with pytest.raises(first_party_host_admission.FirstPartyHostRuntimeRejected, + match="goal_launch_not_allowed"): + with worker.native_child_process(command, **launch): + pytest.fail("the released lease started another worker") + assert not marker.exists() and not mutations + else: + with worker.native_child_process(command, **launch) as process: + process.communicate(input="original checkpoint input", timeout=5) + assert process.returncode == 0 + assert marker.read_text() == "original checkpoint input" + released, = mutations + assert released.result(timeout=10)["status"] == "released" + assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1 + assert quota_fixture._spend_run_count(authority.runtime) == 0 + + +@pytest.mark.parametrize("retry_lease", ["expired", "replacement"]) +def test_final_provider_write_crossing_hard_lease_preserves_unknown_attempt( + held_provider_task, monkeypatch, retry_lease, +): + authority, quota, original, lease = held_provider_task + turn = quota_fixture.TURN_ID + suffix = b"final-provider-body" + body = b"x" * (64 * 1024) + suffix + accepted_write_at = [] + original_connection = provider_gateway._upstream_connection + + def delay_final_write(*args, **kwargs): + connection = original_connection(*args, **kwargs) + send = connection.send + + def send_after_preemption(data): + if accepted_write_at or not isinstance(data, memoryview) or data != suffix: + return send(data) + assert 0 < connection.sock.gettimeout() < 2, "the hard lease must shorten the gateway send limit" + now = time.time() + assert now < expires_at + time.sleep(expires_at - now + .03) + send(data) + accepted_write_at.append(time.time()) + + connection.send = send_after_preemption + return connection + + monkeypatch.setattr(provider_gateway, "_upstream_connection", delay_final_write) + with _upstream() as upstream, _process() as process: + guard, receipt, wake = authority.invocation( + upstream.base_url, "lease-expiry", turn_id=turn, original_quota=quota, + ) + with guard.serve(["native", "-"]): + guard.started(process) + renewed = lease("renew", original["version"], 2) + assert renewed["lease_epoch"] == original["lease_epoch"] + expires_at = lease_expires_at(renewed).timestamp() + status, error = _post(guard.base_url, body) + assert upstream.body.wait(5) and upstream.rows[0]["body"] == body + assert len(accepted_write_at) == 1 and accepted_write_at[0] >= expires_at + assert status == 502 and error["error"]["code"] == "provider_gateway_upstream_unavailable", error + event, = _finished(wake) + summary, = receipt["provider_request_invocations"] + assert event["turn_id"] == turn and event["invocation_id"] == summary["invocation_id"] + assert event["forwarded"] is None and "response_status" not in event and "refusal_code" not in event + assert summary["requests"] == summary["unknown"] == 1 and summary["pending"] == summary["sent"] == 0 + if retry_lease == "replacement": + replacement = lease("acquire", renewed["version"], 60, key="provider-replacement-lease") + assert replacement["lease_epoch"] > renewed["lease_epoch"] + status, denied = _post(guard.base_url, body) + assert status == 409 and denied["error"]["code"] == "goal_launch_not_allowed", denied + _, rejected = _finished(wake, count=2) + assert rejected["invocation_id"] == event["invocation_id"] and rejected["forwarded"] is False + assert len(upstream.rows) == 1 + assert summary["unknown"] == summary["refused"] == 1 and summary["sent"] == 0 + assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1 + assert quota_fixture._spend_run_count(authority.runtime) == 0 + + +def test_bound_provider_send_holds_lease_writer_until_dispatch(held_provider_task, monkeypatch): + authority, quota, original, lease = held_provider_task + waiting, resume = threading.Event(), threading.Event() + original_connection = provider_gateway._upstream_connection + + def connection_at_send(*args, **kwargs): + connection = original_connection(*args, **kwargs) + send = connection.send + + def pause_body(data): + if isinstance(data, memoryview): + waiting.set() + assert resume.wait(4), "test did not release the admitted POST" + return send(data) + + connection.send = pause_body + return connection + + monkeypatch.setattr(provider_gateway, "_upstream_connection", connection_at_send) + with _upstream(hold_response=True) as upstream, _process() as process: + guard, _, wake = authority.invocation( + upstream.base_url, "lease-release", turn_id=quota_fixture.TURN_ID, original_quota=quota, + ) + with guard.serve(["native", "-"]): + guard.started(process) + response = _async(lambda: _post(guard.base_url)) + try: + assert waiting.wait(5) and not upstream.body.is_set() + target = shadow_maintenance_lock_target(authority.runtime, quota_fixture.GOAL_ID) + with pytest.raises(LockAcquireTimeoutError): + with exclusive_mutation_file_lock(target, timeout_seconds=0): + pass + mutation = _async(lambda: lease("release", original["version"])) + resume.set() + released = mutation.result(timeout=5) + assert released["status"] == "released" + assert upstream.body.wait(5) and upstream.rows[0]["body"] == b"{}" + assert not response.done() + finally: + resume.set() + upstream.respond.set() + assert response.result(timeout=5)[0] == 200 + assert _finished(wake)[0]["forwarded"] is True + assert _post(guard.base_url)[0] == 409 + assert len(upstream.rows) == 1 and quota_fixture._spend_run_count(authority.runtime) == 0 + + def test_slow_response_does_not_hold_goal_writer(authority): with _upstream(hold_response=True) as upstream, _process() as process: guard, receipt, wake = authority.invocation(upstream.base_url, "slow-response") diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index f64b42a9bf..fc7fdafe68 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -483,16 +483,42 @@ an existing observation. They can contain private Goal context and must not be included in public logs or artifacts. Host retry validates and reuses the original journal's capture. Before native -Codex or an external stdio Host launches with a verified selected Todo, the -current complete contract must still match the original task body, required write scopes and -required capabilities. Progress notes do not invalidate that work. A mismatch -returns `current_turn_work_changed` before dispatching the model. For stdio, -the shared process supervisor holds the launch guard through spawn. The -journal records the refused attempt and its original capture remains readable. -Use a fresh eligible Turn for the changed task. Keep original captures and receipts for -readback. Reading detail does not invoke quota or spend a slot. Current source -admission, cancellation, workspace/lease and mutation checks retain their -existing owners; the observation grants no fresh authority. +Codex or an external stdio Host dispatches a bound Todo, the current complete +contract must match the original task body, required write scopes, required +capabilities and execution lease key when present. Progress notes and renewal +of the same lease preserve that match. A changed declaration or lease key +returns `current_turn_work_changed` before dispatch. + +For local canonical work, the Todo and lease writers share the dispatch lock +from the live check through process spawn or the native `turn/start` send. A +writer that commits first changes the inputs to the live check. A writer that +arrives after the check can continue once dispatch completes, while the Host +runs. The same live hard-lease read supplies the dispatch deadline. The process +supervisor checks it at spawn. The native sender checks the deadline before +each nonblocking write and after the complete `turn/start` frame is accepted +by the pipe. A proven pre-dispatch expiry returns `goal_launch_not_allowed` +with no Host invocation, writeback or debit. If a complete frame may have been +delivered after expiry, the sender retires the receiver and the original +attempt stays in progress with `host_outcome_unknown`. Recovery holds retry +pending reconciliation of that attempt. A Todo-bound native admission links +its original receipt to the journal after the Host attempt is durable and +before dispatch. Fresh admission returns the original Turn key for reconciliation +if that attempt is unconfirmed or its linked journal is missing. An admission +that stops before recording an attempt has no journal link. Registry contention +can refuse dispatch with `goal_authority_unavailable`. +The journal retains the original capture, identity and saved session recovery +for retry or readback. + +硬租约的派发截止时间来自同一次权威读。进程启动前检查到期时间。原生 +`turn/start` 每次非阻塞写入前都检查时限,管道接受完整帧后再检查一次。 +确认未派发即到期时,返回 `goal_launch_not_allowed`,不启动模型工作、不写回、 +不扣额。完整帧可能在到期后送达时,发送方终止接收进程,原尝试保留在执行中并 +标为 `host_outcome_unknown`。绑定 Todo 的原生执行在尝试落盘后、派发前,将日志索引 +附到原准入回执。结果未确认或已绑定日志缺失时,新准入返回原 Turn 索引供核对,先核查 +原尝试再派发。止于尝试落盘前的准入没有绑定日志索引。 +Use a fresh eligible Turn for changed work. Reading detail consumes no quota. +Source admission, cancellation, workspace and mutation authority remain with +their existing owners. Missing, corrupt, wrong-identity or source-hash-mismatched captures refuse Host reinvocation. Restore the original evidence rather than reconstructing it from a new guard. A capture published before a journal write failure is retained; @@ -514,10 +540,16 @@ blocks resume. Journal inspection can report the invalid work contract without performing recovery or a debit. Retain the historical reader until the last supported journal and its recovery window are reconciled. -原生 Codex 和外部 stdio Host 启动前会用当前完整契约复核所选 Todo 的已验证正文、 -所需写入范围和能力。只改进度备注可以继续。改变工作声明会在模型调用前返回 -`current_turn_work_changed`。stdio 进程由共用的进程管理器在准入锁内启动。 -journal 记录已拒绝的尝试。变更后的工作使用新的合格 Turn,原捕获与回执仍可读回。 +原生 Codex 和外部 stdio Host 派发已绑定 Todo 前,使用当前完整契约复核原正文、 +所需写入范围、能力和已有执行租约键。只改进度备注或续期同一租约仍可继续。 +工作声明或租约键改变时,在派发前返回 `current_turn_work_changed`。 + +对本地权威工作,从实时检查到进程启动或发送原生 `turn/start`,派发与 Todo 和 +租约写入共用同一锁。写入先提交时,实时检查读取变更后的事实。写入在检查后到达时, +可以在派发完成、Host 仍运行时继续。遇到 registry 写入竞争,派发可以返回 +`goal_authority_unavailable`。journal 保留原捕获、身份和已保存的会话恢复信息,用于重试或读回。 +变更后的工作通过新的合格 Turn 执行。读取详情不扣额。来源准入、取消、工作区和 +写入权限继续由原模块维护。 恢复预览和 journal 解释共用同一个 typed plan reader。受支持的历史记录保留原版本和 字节。显式声明不支持的 plan、envelope 或动作签名版本及覆盖范围时,恢复会停止。 diff --git a/loopx/capabilities/benchmark_toolkit/provider_gateway.py b/loopx/capabilities/benchmark_toolkit/provider_gateway.py index 2c8b5d14cf..a0a2247c6e 100644 --- a/loopx/capabilities/benchmark_toolkit/provider_gateway.py +++ b/loopx/capabilities/benchmark_toolkit/provider_gateway.py @@ -126,20 +126,23 @@ def _dispatch_bounded(connection: http.client.HTTPConnection, path: str, body: b for key, value in headers.items(): connection.putheader(key, value) - def remaining() -> None: + def remaining_budget() -> float: budget = deadline - time.monotonic() if not math.isfinite(budget) or budget <= 0: raise TimeoutError("provider request dispatch deadline exceeded") - connected_socket.settimeout(budget) + return budget - remaining() + connected_socket.settimeout(remaining_budget()) connection.endheaders() # Recompute the absolute budget between writes; repeated successful writes # must not renew a socket timeout while a Goal writer is waiting. view = memoryview(body) for start in range(0, len(view), 64 * 1024): - remaining() + connected_socket.settimeout(remaining_budget()) connection.send(view[start:start + 64 * 1024]) + # The sender can be preempted after setting the last relative timeout. + # A late return may have delivered bytes, so leave the attempt unconfirmed. + remaining_budget() @contextmanager diff --git a/loopx/capabilities/issue_fix/README.md b/loopx/capabilities/issue_fix/README.md index 3338cccdbe..04f1ce376e 100644 --- a/loopx/capabilities/issue_fix/README.md +++ b/loopx/capabilities/issue_fix/README.md @@ -345,14 +345,13 @@ candidate so a maintainer can audit why that person was selected. `CODEOWNERS` remains the strongest repository-native signal. Commit volume is only evidence of familiarity; it is not proof of maintainership, availability, -or review authority. See the [reviewer recommendation -contract](docs/protocols/issue-fix-reviewer-recommendation-v0.md) for scoring, -identity, and future-signal details, and the [reviewer request -contract](docs/protocols/issue-fix-reviewer-request-v0.md) for the external-write, +or review authority. See the +[reviewer recommendation contract](docs/protocols/issue-fix-reviewer-recommendation-v0.md) for +scoring, identity, and future-signal details, and the +[reviewer request contract](docs/protocols/issue-fix-reviewer-request-v0.md) for the external-write, idempotency, and verification rules. Secondary delivery, dedicated-bot isolation, local-private identity mapping, and readback are defined by the -[reviewer notification sink -contract](docs/protocols/issue-fix-reviewer-notification-sinks-v0.md). +[reviewer notification sink contract](docs/protocols/issue-fix-reviewer-notification-sinks-v0.md). ### 6. PR publication and public-write boundary diff --git a/loopx/capabilities/native_chat/codex_adapter.py b/loopx/capabilities/native_chat/codex_adapter.py index 7fd5dd5d58..161adba9d7 100644 --- a/loopx/capabilities/native_chat/codex_adapter.py +++ b/loopx/capabilities/native_chat/codex_adapter.py @@ -1020,9 +1020,9 @@ def read_turn(): if previous_guard is None: yield else: - with previous_guard(): + with previous_guard() as expires_at_ms: check() - yield + yield expires_at_ms # Native sandboxPolicy is sticky across subsequent Turns. Reassert the # current managed Chat read boundary without changing Session defaults. return {"sandbox_policy": {"type": "readOnly", "networkAccess": False}, diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index 96083a9160..1378cb339b 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -801,9 +801,10 @@ def launch(): observed = self._observe(actor) with ExitStack() as stack: try: - if previous_guard is not None: - stack.enter_context(previous_guard()) - stack.enter_context(launch_scope()) + previous_expiry = ( + stack.enter_context(previous_guard()) if previous_guard is not None else None + ) + expires_at_ms = stack.enter_context(launch_scope()) stack.enter_context(nullcontext() if actor.get("schema_version") == "loopx_managed_session_actor_v0" else exclusive_file_lock(self.bindings.path, operation="native_steward_launch")) current(observed) @@ -815,7 +816,9 @@ def launch(): # failure; exceptions after yield remain unknown. from ...control_plane.turn_driver.host_failure import BuiltInHostError raise BuiltInHostError(str(exc), failure_kind="contract_rejected") from exc - yield + if previous_expiry is not None and (expires_at_ms is None or previous_expiry < expires_at_ms): + expires_at_ms = previous_expiry + yield expires_at_ms adapter.session._launch_guard = launch try: diff --git a/loopx/chat_agent.py b/loopx/chat_agent.py index 033b57975a..70b100ccd4 100644 --- a/loopx/chat_agent.py +++ b/loopx/chat_agent.py @@ -717,7 +717,7 @@ class CodexChatAgentSession: read_only_review: bool = False answer_review: bool = False metadata_review: bool = False - _launch_guard: Callable[[], AbstractContextManager[None]] | None = field( + _launch_guard: Callable[[], AbstractContextManager[int | None]] | None = field( default=None, repr=False ) _native_task_policy_used: bool = field(default=False, repr=False) @@ -751,7 +751,7 @@ def start( reasoning_effort: str | None = None, dynamic_tools: list[dict[str, Any]] | None = None, host_config: dict[str, Any] | None = None, - launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, account_runtime_root: Path | None = None, account_consumer: str = "worker", _compatibility_catalog_path: Path | None = None, @@ -853,7 +853,11 @@ def start( try: # The admitting caller owns this short launch fence. Native startup # RPCs and model execution must not hold the Goal writer lock. - with launch_guard() if launch_guard is not None else nullcontext(): + with launch_guard() if launch_guard is not None else nullcontext() as expires_at_ms: + if expires_at_ms is not None and time.time() * 1000 >= expires_at_ms: + from .control_plane.goals.first_party_host_admission import FirstPartyHostLaunchRejected + + raise FirstPartyHostLaunchRejected("goal_launch_not_allowed") try: process = subprocess.Popen( command, @@ -1174,7 +1178,7 @@ def _runtime_error(self, summary: str) -> CodexChatAgentError: def _write( self, payload: dict[str, Any], *, - write_guard: Callable[[], AbstractContextManager[None]] | None = None, + write_guard: Callable[[], AbstractContextManager[int | None]] | None = None, deadline: float | None = None, ) -> None: if self.process.stdin is None: @@ -1193,10 +1197,14 @@ def _write( ) from exc def _guarded_write( - self, frame: str, write_guard: Callable[[], AbstractContextManager[None]], *, + self, frame: str, write_guard: Callable[[], AbstractContextManager[int | None]], *, deadline: float | None, ) -> None: - from .extensions.process_runtime import force_terminate_process, write_process_stdin_before_deadline + from .extensions.process_runtime import ( + ProcessStdinLeaseExpired, + force_terminate_process, + write_process_stdin_before_deadline, + ) # Serialization, delivery and response share the original RPC/Turn # deadline. No shorter send budget may reject an otherwise valid frame. @@ -1204,24 +1212,20 @@ def _guarded_write( if not self._write_lock.acquire(timeout=max(0, dispatch_deadline - time.monotonic())): raise self._timeout_error("response_timeout", "Codex input dispatch timed out.") try: - with write_guard(): + with write_guard() as expires_at_ms: process = self.process try: - if time.monotonic() >= dispatch_deadline: - # Canonical admission itself can consume the remaining - # budget. No writer started, but this Session still - # retires its peer before releasing the launch fence. - try: - force_terminate_process(process, process_tree_owned=self.process_tree_owned) - except Exception as error: - raise RuntimeError("native input receiver cleanup unconfirmed") from error - raise TimeoutError("native input dispatch deadline expired") write_process_stdin_before_deadline( process, frame, deadline=dispatch_deadline, cancel_and_reap=lambda: force_terminate_process( process, process_tree_owned=self.process_tree_owned, ), + lease_expires_at_ms=expires_at_ms, retire_on_expired_deadline=True, ) + except ProcessStdinLeaseExpired as exc: + from .control_plane.goals.first_party_host_admission import FirstPartyHostLaunchRejected + + raise FirstPartyHostLaunchRejected("goal_launch_not_allowed") from exc except TimeoutError as exc: raise self._timeout_error( "response_timeout", "Codex input dispatch timed out." @@ -1389,7 +1393,7 @@ def _request( *, request_id: int | None = None, deadline: float | None = None, - write_guard: Callable[[], AbstractContextManager[None]] | None = None, + write_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> dict[str, Any]: if request_id is None: with self._request_id_lock: @@ -1734,7 +1738,7 @@ def send( output_schema: dict[str, Any] | None = None, project_collaboration: bool = False, on_native_item: Callable[[dict[str, Any]], None] | None = None, - turn_start_guard: Callable[[], AbstractContextManager[None]] | None = None, + turn_start_guard: Callable[[], AbstractContextManager[int | None]] | None = None, check_running: Callable[[], None] | None = None, _native_task: bool = False, sandbox_policy: dict[str, object] | None = None, diff --git a/loopx/cli_commands/turn_decision.py b/loopx/cli_commands/turn_decision.py index edc067f02b..f637a8cf30 100644 --- a/loopx/cli_commands/turn_decision.py +++ b/loopx/cli_commands/turn_decision.py @@ -390,6 +390,7 @@ def current_turn_host_launch_decision( turn_instance_id=identity.turn_instance_id, turn_start_hook_dispatch=turn_start_hook_dispatch, workspace_path=Path(args.project).expanduser().resolve(), goal_ref=goal_ref, + include_launch_lease_deadline=True, interaction_projection_hooks=(periodic_report_pending_intent_interaction_hook( registry_path=registry_path, runtime_root=runtime_root, goal_id=identity.goal_id, agent_id=identity.agent_id, @@ -472,7 +473,9 @@ def current_heartbeat_host_launch_decision( selection.receipt, turn_instance_id=original_turn, status="replayed", ) if (not held.get("event_id") or current_receipt.get("event_id") != held.get("event_id") - or current_receipt.get("settlement_identity") != held.get("settlement_identity")): + or current_receipt.get("settlement_identity") != held.get("settlement_identity") + or current_receipt.get("semantic_replan_obligation_id") + != held.get("semantic_replan_obligation_id")): raise FirstPartyHostRuntimeRejected("heartbeat_launch_original_binding_changed") current = build_live_quota_should_run_decision( collect_turn_status_payload(args, registry_path=registry_path, runtime_root_arg=str(runtime_root)), @@ -486,6 +489,7 @@ def current_heartbeat_host_launch_decision( receipt_bound_replan_obligation_id=selection.receipt_bound_replan_obligation_id, retained_action_selection_todo_id=selection.retained_todo_id_for_decision, turn_instance_id=original_turn, workspace_path=project, goal_ref=goal_ref, + include_launch_lease_deadline=True, interaction_projection_hooks=(periodic_report_pending_intent_interaction_hook( registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id, agent_id=agent_id, ),), @@ -493,9 +497,16 @@ def current_heartbeat_host_launch_decision( # Initial heartbeat/planning has not chosen an action yet. Its generic # prompt asks the Host to consume the current decision after launch. A # held continuation already has an action and cannot substitute another. - return {"current": current, "planned_effective_action": ( + launch = {"current": current, "planned_effective_action": ( original_quota.get("effective_action") if original_quota is not None else current.get("effective_action"))} + if selection.receipt_bound_todo_id is not None: + launch["receipt_bound_todo_id"] = selection.receipt_bound_todo_id + if selection.receipt_bound_replan_obligation_id is not None: + launch["receipt_bound_replan_obligation_id"] = selection.receipt_bound_replan_obligation_id + if original_quota is not None: + launch["planned_execution_preparation"] = original_quota.get("execution_preparation") + return launch def build_fresh_envelope_for_managed_step( diff --git a/loopx/cli_commands/turn_host_request.py b/loopx/cli_commands/turn_host_request.py index 34852557a3..8bcb027ba1 100644 --- a/loopx/cli_commands/turn_host_request.py +++ b/loopx/cli_commands/turn_host_request.py @@ -95,9 +95,7 @@ def prepare_turn_host_request( def build_codex_cli_host_runner( args: argparse.Namespace, *, runtime_root: Path, project: Path, - registry_path: Path, goal_ref: Mapping[str, object] | None, - goal_admission: FirstPartyHostGoalAdmission | None, - current_launch_decision: Callable[[Path], Mapping[str, Any]], + registry_path: Path, goal_admission: FirstPartyHostGoalAdmission | None, run_codex_cli_host: Callable[..., dict[str, Any]], ) -> Callable[[Mapping[str, Any]], dict[str, Any]]: """Bind the existing Codex adapter; resolve options/admission only at launch.""" @@ -113,13 +111,8 @@ def run_built_in_host( "reasoning_effort": args.codex_reasoning_effort, "mcp_server": args.codex_mcp_server_json, "timeout_seconds": None if args.timeout_seconds is None else max(1.0, args.timeout_seconds - 5.0), + "goal_admission": goal_admission, } - options["goal_admission"] = ( - goal_admission or FirstPartyHostGoalAdmission.for_plan( - registry_path=registry_path, goal_id=args.goal_id, - planned_goal_ref=goal_ref, agent_id=args.agent_id, - ) - ).with_launch_decision(current_launch_decision) if getattr(args, "codex_operation_tools", False): from ..control_plane.turn_driver.codex_operation_host import ( run_codex_operation_host, diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index 80e46f402f..558d0d1407 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -804,13 +804,14 @@ def current_launch_decision(current_registry: Path) -> Mapping[str, Any]: turn_start_hook_dispatch=turn_start_hook_dispatch, goal_ref=goal_ref, ) - if args.host == "codex-app-server": - native_admission = (goal_admission or FirstPartyHostGoalAdmission.for_plan( - registry_path=registry_path, goal_id=args.goal_id, planned_goal_ref=goal_ref, - )).with_launch_decision(current_launch_decision) + host_admission = (goal_admission.with_launch_decision( + current_launch_decision, work_runtime_root=runtime_root if settlement_identity.todo_id else None, + ) if goal_admission is not None else None) + if args.host == "codex-app-server": def run_native_host(request: Mapping[str, Any]) -> dict[str, Any]: - return native_host_runner(request, lambda: native_admission.launch_lifetime( + assert host_admission is not None + return native_host_runner(request, lambda: host_admission.launch_lifetime( operation="native_conversation_turn_launch")) host_runner = run_native_host @@ -818,9 +819,7 @@ def run_native_host(request: Mapping[str, Any]) -> dict[str, Any]: elif args.host == "codex-cli": host_runner = build_codex_cli_host_runner( args, runtime_root=runtime_root, project=project, - registry_path=registry_path, goal_ref=goal_ref, - goal_admission=goal_admission, - current_launch_decision=current_launch_decision, + registry_path=registry_path, goal_admission=host_admission, run_codex_cli_host=run_codex_cli_host, ) @@ -882,7 +881,7 @@ def codex_settlement_admission(): settlement_admission = codex_settlement_admission - def on_managed_start_admitted() -> None: + def ensure_start_receipt(turn_key: str | None = None) -> None: if settlement_identity.request_subject is not None: require_current_request_operation(request_locator, settlement_identity, registry_path=registry_path, runtime_root=runtime_root, workspace=project) @@ -904,6 +903,7 @@ def on_managed_start_admitted() -> None: else None ), goal_ref=goal_ref, + turn_key=turn_key, ) managed_cadence = managed_cadence_start( @@ -912,7 +912,7 @@ def on_managed_start_admitted() -> None: agent_id=args.agent_id, automation_id=args.automation_id, manual_reason=args.manual_interval_bypass_reason, - on_admitted=on_managed_start_admitted, + on_admitted=ensure_start_receipt, request_id=settlement_identity.effect_id if args.host == "codex-cli" else None, ) @@ -950,11 +950,14 @@ def on_managed_start_admitted() -> None: ) if args.execute else None), admit_start=managed_cadence.admit if args.execute else None, confirm_start=managed_cadence.confirm if args.execute else None, + record_host_attempt=ensure_start_receipt if args.execute and settlement_identity.todo_id and ( + args.host == "codex-app-server" or getattr(args, "codex_operation_tools", False) + ) else None, goal_admission=goal_admission, host_launch_guard=(partial( - goal_admission.with_launch_decision(current_launch_decision).launch_lifetime, + host_admission.launch_lifetime, operation="turn_external_host_launch", - ) if raw_argv is not None and goal_admission is not None else None), + ) if raw_argv is not None and host_admission is not None else None), ) if args.execute: result.update(drain_committed_turn_post_writeback_hooks( diff --git a/loopx/control_plane/goals/first_party_host_admission.py b/loopx/control_plane/goals/first_party_host_admission.py index d463b25c2f..f15f9fe55f 100644 --- a/loopx/control_plane/goals/first_party_host_admission.py +++ b/loopx/control_plane/goals/first_party_host_admission.py @@ -11,11 +11,12 @@ from ...file_lock import ( cross_runtime_lock_witness, exclusive_cross_runtime_file_lock, + exclusive_mutation_file_lock, ) from .. import compact_control_plane_policy from ..agents.runtime_model import agent_runtime_model_for_goal from ..agents.work_mode import agent_work_mode_for_goal -from ..coordination.shadow_management import runtime_artifact_lock_target +from ..coordination.shadow_management import runtime_artifact_lock_target, shadow_maintenance_lock_target from ..effect_runtime import ( EffectRuntimeRemoteError, EffectRuntimeStartupError, @@ -55,6 +56,10 @@ def __init__(self, code: str) -> None: self.code = code +class FirstPartyHostLaunchRejected(FirstPartyHostRuntimeRejected): + """Refusal observed before the Host dispatched model work.""" + + @dataclass(frozen=True, slots=True) class FirstPartyHostTurnEffectAdmission: goal_admission: FirstPartyHostGoalAdmission @@ -343,6 +348,7 @@ class FirstPartyHostGoalAdmission: launch_decision_reader: Callable[[Path], Mapping[str, Any]] | None = None legacy_running_authority_sha256: str | None = None agent_id: str | None = None + launch_work_runtime_root: Path | None = None @classmethod def for_plan( @@ -415,7 +421,7 @@ def for_plan( def launch_scope_for_plan( cls, *, admission: FirstPartyHostGoalAdmission | None, registry_path: Path | None, goal_id: str, planned_goal_ref: object | None, - ) -> Callable[[str], AbstractContextManager[None]]: + ) -> Callable[[str], AbstractContextManager[int | None]]: """Capture once, then reuse the same admission at each native boundary. A provided Turn admission keeps its quota/health reader. Legacy callers @@ -426,7 +432,7 @@ def launch_scope_for_plan( admission = cls.for_plan(registry_path=registry_path, goal_id=goal_id, planned_goal_ref=planned_goal_ref) - def scope(operation: str) -> AbstractContextManager[None]: + def scope(operation: str) -> AbstractContextManager[int | None]: return admission.launch_lifetime(operation=operation) if admission is not None else nullcontext() return scope @@ -436,7 +442,7 @@ def enabled(self) -> bool: return self.source_profile def with_launch_decision( - self, reader: Callable[[Path], Mapping[str, Any]], + self, reader: Callable[[Path], Mapping[str, Any]], *, work_runtime_root: Path | None = None, ) -> FirstPartyHostGoalAdmission: """Bind a legacy Turn's existing read-only quota owner to launch. @@ -451,7 +457,7 @@ def with_launch_decision( # nested inside that lifetime's source guard. return self return replace( - self, launch_decision_reader=reader, + self, launch_decision_reader=reader, launch_work_runtime_root=work_runtime_root, legacy_source_admission=(self.legacy_source_admission.with_launch_decision(reader) if self.legacy_source_admission is not None else None), ) @@ -547,21 +553,42 @@ def current_lifetime(self, *, operation: str) -> Iterator[None]: yield @contextmanager - def launch_lifetime(self, *, operation: str) -> Iterator[None]: + def launch_lifetime(self, *, operation: str) -> Iterator[int | None]: """Recheck the original launch authority under its existing writer lock. Legacy registries have no exact Goal instance identity. Their original byte witness can reject stale admission without promoting an alias to one; a changed registry requires a fresh plan. A caller with the original Turn binding supplies its full current admission check. This lifecycle - witness alone does not certify quota/health. Hold only through dispatch. + witness alone does not certify quota/health. A bound hard lease also + supplies its epoch-ms expiry for the transport to check at dispatch. """ if self.source_profile: with self.current_lifetime(operation=operation): yield return - with self._legacy_lifetime(operation=operation, launch=True): - yield + work_root = self.launch_work_runtime_root + if work_root is None: + with self._legacy_lifetime(operation=operation, launch=True) as expires_at_ms: + yield expires_at_ms + return + + with ExitStack() as scope: + try: + # Canonical Todo and lease writes share this marker. Keep the + # live decision and the transport dispatch inside its lifetime. + scope.enter_context(exclusive_mutation_file_lock( + shadow_maintenance_lock_target(work_root, self.goal_id), + operation=operation, + )) + expires_at_ms = scope.enter_context(self._legacy_lifetime( + operation=operation, launch=True, registry_lock_timeout=0.0, + )) + except FirstPartyHostRuntimeRejected as exc: + raise FirstPartyHostLaunchRejected(exc.code) from exc + except OSError as exc: + raise FirstPartyHostLaunchRejected("goal_authority_unavailable") from exc + yield expires_at_ms def require_running(self, *, operation: str) -> None: """Recheck the admitted Goal for an already-dispatched native Turn.""" @@ -576,31 +603,41 @@ def require_running(self, *, operation: str) -> None: raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") from exc @contextmanager - def _legacy_lifetime(self, *, operation: str, launch: bool) -> Iterator[None]: + def _legacy_lifetime( + self, *, operation: str, launch: bool, registry_lock_timeout: float | None = None, + ) -> Iterator[int | None]: with ExitStack() as scope: + expires_at_ms = None if self.legacy_source_admission is not None: # Activation writes source then global projection under these # same locks. A failed mirror sync must not hide a source stop. - scope.enter_context(self.legacy_source_admission._legacy_lifetime( - operation=operation, launch=launch, + expires_at_ms = scope.enter_context(self.legacy_source_admission._legacy_lifetime( + operation=operation, launch=launch, registry_lock_timeout=registry_lock_timeout, )) try: + # Registry administration may already hold this lock while + # requesting the canonical work marker. Avoid waiting on the + # inverse lock order from a bound Host launch. transaction = scope.enter_context(project_registry_transaction( - self.registry_path, operation=operation, + self.registry_path, operation=operation, timeout_seconds=registry_lock_timeout, )) except (OSError, TypeError, ValueError) as exc: raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") from exc authority = _legacy_goal_authority(transaction.payload_copy(), self.goal_id, self.agent_id) if launch: - self._decision( - "launch", legacy_authority=authority, - registry_witness=transaction.source_read_witness(), - launch_decision=(self.launch_decision_reader(self.registry_path) - if self.launch_decision_reader is not None else None), - ) + live = self.launch_decision_reader(self.registry_path) if self.launch_decision_reader is not None else None + self._decision("launch", legacy_authority=authority, + registry_witness=transaction.source_read_witness(), launch_decision=live) + if live is not None: + preparation = live["current"].get("execution_preparation") or {} + if preparation.get("state") == "ready" and preparation.get("handoff_mode") == "hard_lease": + deadline = preparation.get("lease_expires_at_ms") + if type(deadline) is not int: + raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") + expires_at_ms = deadline if expires_at_ms is None else min(expires_at_ms, deadline) else: self._decision("require_current", legacy_authority=authority) - yield + yield expires_at_ms @contextmanager def source_journal_admission( diff --git a/loopx/control_plane/goals/first_party_host_runtime.ts b/loopx/control_plane/goals/first_party_host_runtime.ts index e2b655ccad..04bd84ee37 100644 --- a/loopx/control_plane/goals/first_party_host_runtime.ts +++ b/loopx/control_plane/goals/first_party_host_runtime.ts @@ -1,5 +1,8 @@ import type { JsonObject } from "../effect_program.ts"; +import { decodeTaskLeaseProof } from "../coordination/task_lease_proof.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; +import { EffectiveAction } from "../quota/effective_action.generated.ts"; +import { normalizeHeartbeatReplanObligationId } from "../quota/heartbeat_receipt_identity.ts"; import { assertNever, jsonObject } from "../runtime_decode.ts"; import { BARE_SHA256_PATTERN } from "../content_digest.ts"; import { @@ -185,10 +188,34 @@ function launchDecisionRejection(facts: JsonObject): FirstPartyHostRuntimeDecisi // health, repair, replan or scheduler policy in the Host adapter. // A newly selected repair/replan is not permission to launch the old action. // An already-admitted health repair keeps the quota owner's exception. - return decision?.should_run === true && typeof decision.status_health_ok === "boolean" - && typeof launch?.planned_effective_action === "string" && !!launch.planned_effective_action - && decision.effective_action === launch.planned_effective_action - ? null : {kind: "reject", code: "goal_launch_not_allowed"}; + const rejected = {kind: "reject", code: "goal_launch_not_allowed"} as const; + if (decision?.should_run !== true || typeof decision.status_health_ok !== "boolean" + || typeof launch?.planned_effective_action !== "string" || !launch.planned_effective_action + || decision.effective_action !== launch.planned_effective_action) return rejected; + if (launch.receipt_bound_todo_id === undefined) return null; + const todoId = launch.receipt_bound_todo_id; + if (typeof todoId !== "string") return rejected; + + if (decision.effective_action === EffectiveAction.AUTONOMOUS_REPLAN_REQUIRED) { + // A replan can use a Todo for settlement while delivery is prohibited. + // Its continuation authority is the original semantic obligation. + const obligationId = normalizeHeartbeatReplanObligationId(launch.receipt_bound_replan_obligation_id); + return obligationId !== null && decision.normal_delivery_allowed === false && + jsonObject(decision.replan_action_packet)?.obligation_id === obligationId && + jsonObject(decision.selected_todo)?.todo_id === todoId ? null : rejected; + } + + // A Todo-bound receipt requires the live preparation for that same Todo. + const preparation = jsonObject(decision.execution_preparation); + if (preparation?.todo_id !== todoId || + preparation.delivery_allowed !== true) return rejected; + const planned = jsonObject(launch.planned_execution_preparation); + if (planned?.handoff_mode !== "hard_lease" || planned.state !== "ready") return null; + const original = decodeTaskLeaseProof(planned.lease_proof); + const current = decodeTaskLeaseProof(preparation.lease_proof); + return preparation.handoff_mode === "hard_lease" && preparation.state === "ready" && + original && current && original.idempotency_key === current.idempotency_key && + current.expected_version >= original.expected_version ? null : rejected; } /** diff --git a/loopx/control_plane/projects/registry_codec.py b/loopx/control_plane/projects/registry_codec.py index f4a86be3af..0f36e03e83 100644 --- a/loopx/control_plane/projects/registry_codec.py +++ b/loopx/control_plane/projects/registry_codec.py @@ -491,12 +491,14 @@ def _registry_transaction( require_writer: Callable[[_ProjectRegistryDocument], None], agent_id: str | None = None, allow_retired_activation: bool = False, + timeout_seconds: float | None = None, ) -> Iterator[ProjectRegistryTransaction]: expanded = path.expanduser() with exclusive_cross_runtime_file_lock( expanded, agent_id=agent_id, operation=operation, + timeout_seconds=timeout_seconds, ): existed = expanded.exists() if existed: @@ -597,6 +599,7 @@ def project_registry_transaction( operation: str, create: Callable[[], dict[str, Any]] | None = None, agent_id: str | None = None, + timeout_seconds: float | None = None, ) -> Iterator[ProjectRegistryTransaction]: """Hold one legacy/v1 project-registry transaction.""" @@ -611,6 +614,7 @@ def project_registry_transaction( create_document=create_document, require_writer=_require_supported_writer, agent_id=agent_id, + timeout_seconds=timeout_seconds, ) as transaction: yield transaction diff --git a/loopx/control_plane/quota/execution_preparation.py b/loopx/control_plane/quota/execution_preparation.py index 9382711dc6..988e1a68b6 100644 --- a/loopx/control_plane/quota/execution_preparation.py +++ b/loopx/control_plane/quota/execution_preparation.py @@ -35,6 +35,7 @@ def project_selected_todo_execution_preparation( goal_id: str, selected_todo: Mapping[str, Any], agent_id: str | None, + include_launch_lease_deadline: bool = False, ) -> dict[str, Any] | None: """Read the selected Todo's canonical lease and project one preparation step. @@ -79,9 +80,11 @@ def project_selected_todo_execution_preparation( if not isinstance(lease, dict) or lease.get("todo_id") != todo_id: raise ValueError("execution preparation returned a lease for a different Todo") status = str(lease.get("status") or "unknown").strip().lower() + active = lease_is_active(lease) leases.append({ "status": status if status in {"active", "released"} else "unknown", - "active": lease_is_active(dict(lease)), + "active": active, + **({"expires_at": lease.get("expires_at")} if include_launch_lease_deadline and active else {}), **({"owner": lease["owner"]} if normalize_todo_claimed_by(lease.get("owner")) else {}), "lease_proof": {"idempotency_key": lease.get("idempotency_key"), "expected_version": lease.get("version")}, @@ -124,6 +127,7 @@ def apply_selected_todo_execution_preparation( obligation: dict[str, Any] | None = None, recommendation: dict[str, Any] | None = None, receipt_bound: bool = False, + include_launch_lease_deadline: bool = False, ) -> None: selected = payload.get("selected_todo") if ( @@ -146,6 +150,7 @@ def apply_selected_todo_execution_preparation( goal_id=goal_id, selected_todo=selected, agent_id=agent_id, + include_launch_lease_deadline=include_launch_lease_deadline, ) # Existing receipt-bound recovery predates this projection and may be # reconstructed without canonical lease authority in synthetic/legacy @@ -158,10 +163,16 @@ def apply_selected_todo_execution_preparation( or preparation.get("state") == "authority_unavailable" ): return - if preparation is None or preparation.get("state") == "not_required": + if preparation is None: return - payload["execution_preparation"] = preparation state = preparation.get("state") + if state == "not_required": + # The live Host needs the canonical soft-claim result to distinguish + # permission from missing authority. Ordinary quota omits this detail. + if include_launch_lease_deadline: + payload["execution_preparation"] = preparation + return + payload["execution_preparation"] = preparation if state == "ready": return payload["normal_delivery_allowed"] = False @@ -191,7 +202,7 @@ def attach_interaction_execution_preparation( contract: dict[str, Any], payload: dict[str, Any], ) -> None: preparation = payload.get("execution_preparation") - if not isinstance(preparation, dict): + if not isinstance(preparation, dict) or preparation.get("state") == "not_required": return contract["agent_channel"]["execution_preparation"] = preparation contract["cli_channel"]["execution_preparation"] = preparation diff --git a/loopx/control_plane/quota/heartbeat_receipt.py b/loopx/control_plane/quota/heartbeat_receipt.py index b228abd519..e0389ea962 100644 --- a/loopx/control_plane/quota/heartbeat_receipt.py +++ b/loopx/control_plane/quota/heartbeat_receipt.py @@ -11,6 +11,7 @@ load_rollout_events, rollout_event_log_path, ) +from ..content_digest import ENVELOPED_SHA256_PATTERN from ..todos.contract import normalize_todo_id, normalize_todo_replan_obligation_id from .effect_program import SettlementBindingKind, SettlementIdentity from .error_codes import HeartbeatReceiptIdentityConflictError @@ -208,6 +209,7 @@ def ensure_turn_heartbeat_settlement_receipt( quota_spend_source: str | None = None, goal_ref: Mapping[str, object] | None = None, before_bind: Callable[[], None] | None = None, + turn_key: str | None = None, ) -> dict[str, object]: """Idempotently bind a Turn-created quota guard to its settlement identity. @@ -215,6 +217,8 @@ def ensure_turn_heartbeat_settlement_receipt( obligation was selected. Persist that explicit empty selection so an obligation opened while the host is running cannot retroactively reject the admitted Turn. Old envelopes without the field stay legacy-unscoped. + A journal key links the already bound receipt to a durably recorded native + Host attempt. This revision preserves the original admission facts. """ normalized_semantic_replan_obligation_id = ( @@ -253,6 +257,31 @@ def ensure_turn_heartbeat_settlement_receipt( "Turn heartbeat receipt belongs to another settlement identity" ) previous_details = effective.get("details") if effective is not None else None + if turn_key is not None: + if (identity.binding_kind is not SettlementBindingKind.TODO + or ENVELOPED_SHA256_PATTERN.fullmatch(turn_key) is None + or effective is None or observed != expected or not isinstance(previous_details, Mapping)): + raise HeartbeatReceiptIdentityConflictError( + "Native Host attempt requires a valid journal key and the original bound receipt" + ) + previous_key = previous_details.get("turn_key") + if previous_key == turn_key: + return effective + if "turn_key" in previous_details: + raise HeartbeatReceiptIdentityConflictError( + "Turn heartbeat receipt already identifies another journal" + ) + source_event_id = str(effective["event_id"]) + linked = build_rollout_event( + goal_id=identity.goal_id, goal_ref=goal_ref, event_kind="quota_should_run", + agent_id=identity.agent_id, todo_id=identity.todo_id, run_id=identity.turn_instance_id, + status=str(effective["status"]), recorded_at=str(effective["recorded_at"]), + summary="Original Turn receipt linked to its durable Host attempt", + source_event_id=source_event_id, caused_by=source_event_id, + details={**previous_details, "turn_key": turn_key}, + ) + _append_rollout_event_line(log_path, linked) + return linked previous_source = ( previous_details.get("quota_spend_source") if observed is not None and isinstance(previous_details, Mapping) else None diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index fde3e9c491..8a9e528830 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -697,6 +697,7 @@ def build_live_quota_should_run_decision( workspace_path: Path | None = None, goal_ref: Mapping[str, object] | None = None, inline_goal_source: bool = False, + include_launch_lease_deadline: bool = False, ) -> dict[str, Any]: """Build one live CLI decision while keeping host observation injectable.""" resolved_context = resolve_scheduler_execution_context(scheduler_execution_context) @@ -881,6 +882,7 @@ def observe_app_scheduler_state(readback: Mapping[str, Any]) -> None: workspace_path=workspace_path, goal_ref=goal_ref, authoritative_fallback_todo_items=authoritative_fallback_todo_items, + include_launch_lease_deadline=include_launch_lease_deadline, ) _project_receipt_replan_settlement(payload, settlement_readback, available_capabilities=available_capabilities, scheduler_execution_context=resolved_context, diff --git a/loopx/control_plane/quota/prior_turn_closeout.ts b/loopx/control_plane/quota/prior_turn_closeout.ts index 9f07b8416e..328a1c43f4 100644 --- a/loopx/control_plane/quota/prior_turn_closeout.ts +++ b/loopx/control_plane/quota/prior_turn_closeout.ts @@ -13,7 +13,9 @@ import { jsonObject } from "../runtime_decode.ts"; import { admitsAutonomousReplan, heartbeatReceiptBinding, + heartbeatReceiptDetails, heartbeatReceiptFactFromEvent, + optionalHeartbeatString, selectEffectiveHeartbeatReceipt, type HeartbeatReceiptBinding, type HeartbeatReceiptFact, @@ -32,11 +34,13 @@ export interface PriorHostTurnCloseoutCandidate extends JsonObject { settlement_effect_id: string | null; /** The effective guard admitted a replan under this Todo's settlement identity. */ admitted_todo_replan?: true; + unconfirmed_host_turn_key?: string; } interface CloseoutCandidateCheck { candidate: PriorHostTurnCloseoutCandidate; requireWritebackForRecovery: boolean; + turnKey: string | null; } function candidate( @@ -62,6 +66,8 @@ function candidate( * admission requires a verified outstanding writeback for this binding. * Explicit pending-closeout discovery can also find admitted replans before * writeback, whether their settlement identity is a Todo or an obligation. + * Admission preflight can also inspect the exact journal named by a native + * Todo guard when the receipts do not confirm closeout. * * Every prior Turn is identity-validated, so a conflicting receipt fails the * read instead of being hidden behind a newer Turn. Results are newest first @@ -78,7 +84,7 @@ export function selectCloseoutCandidates( goalId: string, agentId: string, excludeTurnInstanceId: string | null, - { includeUnwrittenReplans } = { includeUnwrittenReplans: false }, + { includeUnwrittenReplans = false, readTurnJournals = false } = {}, ): { candidates: CloseoutCandidateCheck[]; turnsValidated: number } { const perTurn = new Map(); const firstRecorded = new Map(); @@ -120,10 +126,12 @@ export function selectCloseoutCandidates( if (binding === null) continue; const admittedReplan = admitsAutonomousReplan(effective, fact); const admittedTodoReplan = admittedReplan && binding.binding_kind === "todo"; + const turnKey = readTurnJournals && binding.binding_kind === "todo" + ? optionalHeartbeatString(heartbeatReceiptDetails(effective).turn_key) : null; if (!fact.closeout_required && !admittedTodoReplan && - !(includeUnwrittenReplans && admittedReplan)) continue; + !(includeUnwrittenReplans && admittedReplan) && turnKey === null) continue; candidates.push({candidate: candidate(binding, turnInstanceId, fact, admittedTodoReplan), - requireWritebackForRecovery: !fact.closeout_required && !includeUnwrittenReplans}); + requireWritebackForRecovery: !fact.closeout_required && !includeUnwrittenReplans, turnKey}); } return {candidates, turnsValidated: validated}; } diff --git a/loopx/control_plane/quota/should_run.py b/loopx/control_plane/quota/should_run.py index 962f4540e7..5a8d5aca9c 100644 --- a/loopx/control_plane/quota/should_run.py +++ b/loopx/control_plane/quota/should_run.py @@ -580,6 +580,7 @@ def build_quota_should_run( goal_ref: Mapping[str, object] | None = None, authoritative_fallback_todo_items: FallbackTodoSource = None, replan_explore_reader: Callable[[str, str | None], Mapping[str, Any] | None] | None = None, + include_launch_lease_deadline: bool = False, app_scheduler_state_observer: Callable[[Mapping[str, Any]], None] | None = None, ) -> dict[str, Any]: safe_goal_id = str(goal_id or "").strip() @@ -690,6 +691,7 @@ def build_quota_should_run( include_agent_todo_detail=include_agent_todo_detail, runtime_root=runtime_root, goal_ref=goal_ref, + include_launch_lease_deadline=include_launch_lease_deadline, scheduler_state_read_observer=_app_scheduler_state_read_observer( direct_sources, app_scheduler_state_observer, ), diff --git a/loopx/control_plane/quota/should_run_packet.py b/loopx/control_plane/quota/should_run_packet.py index 294d2fc2a4..35ff5e3819 100644 --- a/loopx/control_plane/quota/should_run_packet.py +++ b/loopx/control_plane/quota/should_run_packet.py @@ -1689,6 +1689,7 @@ def _build_quota_should_run_payload( include_agent_todo_detail: bool = False, runtime_root: str | Path | None = None, goal_ref: Mapping[str, object] | None = None, + include_launch_lease_deadline: bool = False, scheduler_state_read_observer: ( Callable[[Mapping[str, Any]], None] | None ) = None, @@ -1726,6 +1727,7 @@ def _build_quota_should_run_payload( else None ), receipt_bound=prepared.receipt_bound_todo_id is not None, + include_launch_lease_deadline=include_launch_lease_deadline, ) _attach_automation_cadence_readback( payload, diff --git a/loopx/control_plane/quota/turn_envelope.ts b/loopx/control_plane/quota/turn_envelope.ts index a3ffe17e47..d978ef0619 100644 --- a/loopx/control_plane/quota/turn_envelope.ts +++ b/loopx/control_plane/quota/turn_envelope.ts @@ -2031,17 +2031,18 @@ function completeTaskTextSha256(decision: JsonObject, todoId: unknown): string | return capturedTodoTextSha256(declared, context, decision.recommended_action, todoId); } -/** Launch rechecks the admitted task, not the provider revision or the bounded - * action signature. Progress, diagnostics and receipt changes are not edits to - * the work. Losing the verified body on a fresh read is not a match. */ +/** Recheck the admitted task and execution lease. Progress and receipts leave + * that basis unchanged. Renewals retain the execution key. */ function selectedWorkLaunchBasis(decision: JsonObject, verifiedTextSha256?: string | null): string | null { const todoId = selectedWorkTodoId(decision); const selected = object(decision.selected_todo); if (!todoId || todoId !== selected.todo_id) return null; const textSha256 = verifiedTextSha256 === undefined ? completeTaskTextSha256(decision, todoId) : verifiedTextSha256; + const leaseKey = object(object(decision.execution_preparation).lease_proof).idempotency_key; return textSha256 === null ? null : canonicalHash({todo_id: todoId, text_sha256: textSha256, required_write_scopes: selected.required_write_scopes ?? [], - required_capabilities: selected.required_capabilities ?? []}); + required_capabilities: selected.required_capabilities ?? [], + ...(typeof leaseKey === "string" ? {task_lease_idempotency_key: leaseKey} : {})}); } export function composeCompleteTurnDecision(value: unknown): JsonObject { diff --git a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts index 8bb66871c0..f392b9c74b 100644 --- a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts +++ b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts @@ -14,8 +14,9 @@ * whose settlement already validates is answered by the preflight, so the * caller reads no bound facts for it. */ -import type { JsonObject } from "../effect_program.ts"; +import { settlementIdentityPayload, type JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; +import { turnJournalHasUnconfirmedHost } from "../turn_driver/turn_journal_query.ts"; import { jsonObject, requireJsonObject, @@ -242,13 +243,15 @@ async function preflightPriorHostTurnCloseoutForOwner( request.goal_id, request.agent_id, request.exclude_turn_instance_id, + {readTurnJournals: true}, ); // Resolve one snapshot on the first candidate. The empty path needs only // the guard log, and unwritten replans may all drop out after validation. let settlementSnapshot: Awaited> | undefined; let newestSettledTurn: string | null = null; let newestAcceptedCloseout: AcceptedCloseout = "validated_writeback_and_quota_spend"; - for (const {candidate: selected, requireWritebackForRecovery} of candidates) { + for (const {candidate, requireWritebackForRecovery, turnKey} of candidates) { + let selected = candidate; settlementSnapshot ??= await readQuotaSettlementSnapshot( request.runtime_root, request.goal_id, rolloutSnapshot, ); @@ -256,8 +259,7 @@ async function preflightPriorHostTurnCloseoutForOwner( settlementReadbackRequest(request, selected), settlementSnapshot, ); - const missingReceipts = missingCloseoutReceipts(readback, requireWritebackForRecovery); - if (missingReceipts === undefined) continue; + let missingReceipts = missingCloseoutReceipts(readback, requireWritebackForRecovery); if (missingReceipts === null) { // A newer settled Turn cannot hide an older missing closeout. Keep // scanning in persisted newest-first order until the first unsettled @@ -271,6 +273,19 @@ async function preflightPriorHostTurnCloseoutForOwner( } continue; } + if (turnKey !== null && await turnJournalHasUnconfirmedHost( + request.runtime_root, + turnKey, + settlementIdentityPayload({ + goal_id: request.goal_id, agent_id: request.agent_id, + ...candidateReadbackBinding(selected), + }), + request.owner_projection.goal_ref, + )) { + selected = {...selected, unconfirmed_host_turn_key: turnKey}; + missingReceipts = missingCloseoutReceipts(readback, false); + } + if (missingReceipts == null) continue; const monitorPoll = selected.binding_kind === "todo" ? committedMonitorPollFromSnapshot(settlementSnapshot, { goal_id: request.goal_id, agent_id: request.agent_id, @@ -335,6 +350,14 @@ function decodeCandidate(value: unknown): PriorHostTurnCloseoutCandidate { ); } const settlementEffectId = optionalHeartbeatString(raw.settlement_effect_id); + const unconfirmedHostTurnKey = optionalHeartbeatString(raw.unconfirmed_host_turn_key); + if (raw.unconfirmed_host_turn_key !== undefined && + (bindingKind !== "todo" || settlementEffectId === null || unconfirmedHostTurnKey === null || + !/^sha256:[a-f0-9]{64}$/u.test(unconfirmedHostTurnKey))) { + throw new EffectRuntimeRequestError( + "prior host Turn recovery candidate has an invalid unconfirmed dispatch locator", + ); + } if (raw.admitted_todo_replan !== undefined && (raw.admitted_todo_replan !== true || bindingKind !== "todo" || settlementEffectId === null)) { throw new EffectRuntimeRequestError( @@ -350,6 +373,7 @@ function decodeCandidate(value: unknown): PriorHostTurnCloseoutCandidate { binding_kind: bindingKind, binding_id: bindingId, settlement_effect_id: settlementEffectId, + ...(unconfirmedHostTurnKey === null ? {} : {unconfirmed_host_turn_key: unconfirmedHostTurnKey}), ...(raw.admitted_todo_replan === true ? {admitted_todo_replan: true as const} : {}), }; } @@ -554,7 +578,9 @@ export function reduceUnsettledHostTurnRecovery(value: unknown): JsonObject { } const replanSpendOwed = selected.admitted_todo_replan === true && awaitingQuotaSpend(missingReceipts); - const lifecycleCloseout = replanSpendOwed ? null : acceptedLifecycleCloseout(todo); + const unconfirmedHostTurnKey = selected.unconfirmed_host_turn_key; + const lifecycleCloseout = replanSpendOwed || unconfirmedHostTurnKey + ? null : acceptedLifecycleCloseout(todo); if (lifecycleCloseout !== null) { return { schema_version: UNSETTLED_HOST_TURN_RECOVERY_RESULT_SCHEMA, @@ -597,9 +623,22 @@ export function reduceUnsettledHostTurnRecovery(value: unknown): JsonObject { recovery.admission_checkpoint = checkpoint; recovery.admissions_without_progress = count; recovery.admission_limit = RECOVERY_ADMISSION_LIMIT; - const attention = count >= RECOVERY_ADMISSION_LIMIT; + const attention = unconfirmedHostTurnKey !== undefined || count >= RECOVERY_ADMISSION_LIMIT; const obligation = recoveryObligation(repair, selected.binding_id); - if (attention) { + if (unconfirmedHostTurnKey) { + recovery.state = "attention_required"; + recovery.reason_code = "host_outcome_unknown"; + recovery.turn_key = unconfirmedHostTurnKey; + Object.assign(obligation, { + must_attempt_work: false, + notify: "NOTIFY", + reason_code: "host_outcome_unknown", + reason: "an earlier Host dispatch has no confirmed outcome", + recommendation_reason: "the original Host dispatch needs reconciliation", + unsettled_reason: "an earlier Host dispatch remains unconfirmed", + recommended_action: "Inspect the original Turn journal. Restore it if unavailable, then reconcile the Host attempt before dispatching a fresh Turn.", + }); + } else if (attention) { recovery.state = "attention_required"; recovery.reason_code = "unsettled_recovery_admission_limit"; Object.assign(obligation, { diff --git a/loopx/control_plane/turn_driver/codex_cli.py b/loopx/control_plane/turn_driver/codex_cli.py index e2433070b0..11b7dcf337 100644 --- a/loopx/control_plane/turn_driver/codex_cli.py +++ b/loopx/control_plane/turn_driver/codex_cli.py @@ -824,7 +824,7 @@ def _run_codex_exec( timeout_seconds: float | None, on_stdout: Callable[[str], None], on_stderr: Callable[[str], None], - launch_guard: Callable[[], AbstractContextManager[None]], + launch_guard: Callable[[], AbstractContextManager[int | None]], check_running: Callable[[], None] | None, ) -> dict[str, Any]: try: diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 73675259e1..d0302346fd 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -21,6 +21,7 @@ ) from ..goals.first_party_host_admission import ( FirstPartyHostGoalAdmission, + FirstPartyHostLaunchRejected, FirstPartyHostRuntimeRejected, ) from ..goals.goal_vision import normalize_goal_vision_packet @@ -877,7 +878,7 @@ def _run_host( argv: Sequence[str], project: Path, timeout_seconds: float | None, - launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> dict[str, Any]: # Dispatch has checked the source and its internal effect adapter before # journal/cadence admission. External stdin gets the same single complete @@ -954,6 +955,8 @@ def _run_host_runner( *, runner: HostRunner, ) -> dict[str, Any]: + from ...extensions.process_runtime import ProcessStdinDispatchUnconfirmed + try: value = runner(request) except BuiltInHostError as exc: @@ -968,7 +971,7 @@ def _run_host_runner( else {} ), } - except FirstPartyHostRuntimeRejected: + except (FirstPartyHostRuntimeRejected, ProcessStdinDispatchUnconfirmed): raise except Exception as exc: # noqa: BLE001 - host adapters fail closed at boundary return {"ok": False, "reason": type(exc).__name__, "returncode": None} @@ -1013,9 +1016,10 @@ def _host_result_stage( persist_journal: JournalPersist, effects: dict[str, bool], confirm_start: Callable[[], None] | None = None, + record_host_attempt: Callable[[str], None] | None = None, usage_runtime_root: Path | None = None, usage_goal_id: str = "", - host_launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + host_launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> tuple[dict[str, Any] | None, list[str], dict[str, Any] | None]: completed_phases = list(journal.get("completed_phases") or []) from_journal = "typed_result" in completed_phases @@ -1036,6 +1040,9 @@ def _host_result_stage( # reservation. Confirmation failure stops before the host starts. if confirm_start is not None: confirm_start() + if record_host_attempt is not None: + record_host_attempt(str(journal["turn_key"])) + from ...extensions.process_runtime import ProcessStdinDispatchUnconfirmed from ...usage_goal import observe_goal_execution rejected_launch = None @@ -1056,10 +1063,20 @@ def _host_result_stage( launch_guard=host_launch_guard, ) ) + except ProcessStdinDispatchUnconfirmed as exc: + # The durable in-progress attempt already blocks blind replay. + # No Host result may enter validation or settlement without proof. + journal["reason"] = str(exc) + persist_journal(journal) + effects["host_invoked"] = True + unresolved = execution_payload(plan, journal, execute=True, replayed=False, effects=effects) + unresolved["error_code"] = "host_outcome_unknown" + return None, [], unresolved except FirstPartyHostRuntimeRejected as exc: # Stdio checks the Goal only before spawn. A native Host can reject # after dispatch too, so retain reconciliation for other failures. - if host_runner is not None and exc.code != "current_turn_work_changed": + if (host_runner is not None and not isinstance(exc, FirstPartyHostLaunchRejected) + and exc.code != "current_turn_work_changed"): raise rejected_launch = exc.code host_observation = {"ok": False, "reason": str(exc), "failure_kind": "contract_rejected"} @@ -1086,7 +1103,7 @@ def _host_result_stage( recovery_kind = host_observation.get("recovery_kind") if recovery_kind is not None: journal["host_recovery"] = build_host_recovery_record(recovery_kind) - else: + elif rejected_launch is None: journal.pop("host_recovery", None) persist_journal(journal) terminal = execution_payload(plan, journal, execute=True, replayed=False, effects=effects) @@ -1095,6 +1112,7 @@ def _host_result_stage( return None, [], terminal result = dict(host_observation["value"]) completed_phases = list(TRANSACTION_PHASES[:2]) + journal.pop("host_recovery", None) recorded_plan = journal.get("plan") validation = validate_loopx_turn_host_result( @@ -1582,8 +1600,9 @@ def run_loopx_turn_once( settlement_admission: Callable[[], AbstractContextManager] | None = None, admit_start: Callable[[Mapping[str, Any]], dict[str, Any]] | None = None, confirm_start: Callable[[], None] | None = None, + record_host_attempt: Callable[[str], None] | None = None, goal_admission: FirstPartyHostGoalAdmission | None = None, - host_launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + host_launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> dict[str, Any]: argv, host_projection = _prepare_host_invocation(plan, host_argv=host_argv, host_runner=host_runner) transaction_plan = ( @@ -1797,7 +1816,8 @@ def persist_source_journal(source_admission: Mapping[str, Any]) -> None: journal.pop("validation_stage", None) journal.pop("reason", None) journal.pop("receipt", None) - journal.pop("host_recovery", None) + # A launch refusal can leave the existing Host session available. + # Keep its recovery requirement until the next Host observation. journal.pop("host_failure", None) journal["status"] = "in_progress" persist_journal(journal) @@ -1855,6 +1875,7 @@ def finish_recovery(payload: dict[str, Any]) -> dict[str, Any]: persist_journal=persist_journal, effects=effects, confirm_start=confirm_start if reserved_start else None, + record_host_attempt=record_host_attempt, host_launch_guard=host_launch_guard, ) if terminal is not None: diff --git a/loopx/control_plane/turn_driver/host_process.ts b/loopx/control_plane/turn_driver/host_process.ts index b1dc031cbc..6f2c510956 100644 --- a/loopx/control_plane/turn_driver/host_process.ts +++ b/loopx/control_plane/turn_driver/host_process.ts @@ -55,8 +55,9 @@ function signalGroup(child: ChildProcessWithoutNullStreams, signal: NodeJS.Signa catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; throw error; } } -/** Optional caller hooks: record the spawned group, or keep stdin open for framed input. */ +/** Caller hooks for admission, spawn observation and framed input. */ export type HostProcessHooks = { + beforeSpawn?: () => void; spawned?: (item: HostProcessSpawned) => Promise; openInput?: (write: (text: string) => Promise) => void; }; @@ -64,10 +65,11 @@ export type HostProcessHooks = { export async function runHostProcess(request: HostProcessRequest, output: (item: HostProcessOutput) => Promise, signal?: AbortSignal, terminationGraceMs = HOST_PROCESS_TERMINATE_GRACE_MS, - {spawned, openInput}: HostProcessHooks = {}): Promise { + {beforeSpawn, spawned, openInput}: HostProcessHooks = {}): Promise { const base: HostProcessResult = {kind: "result", outcome: "spawn_failed", returncode: null, signal: null, output_complete: true, cleanup_scope: process.platform === "win32" ? "process_tree_best_effort" : "process_group", group_signal_sent: false}; + beforeSpawn?.(); if (signal?.aborted) return {...base, outcome: "cancelled"}; const child = spawn(request.argv[0], request.argv.slice(1), {cwd: request.cwd, stdio: ["pipe", "pipe", "pipe"], detached: process.platform !== "win32", windowsHide: true}); diff --git a/loopx/control_plane/turn_driver/host_process_bridge.ts b/loopx/control_plane/turn_driver/host_process_bridge.ts index ef1f97b78e..1bf6e65d36 100644 --- a/loopx/control_plane/turn_driver/host_process_bridge.ts +++ b/loopx/control_plane/turn_driver/host_process_bridge.ts @@ -3,7 +3,8 @@ import {once} from "node:events"; import {decodeHostProcessRequest, runHostProcess} from "./host_process.ts"; const owner = new AbortController(); // Private transport observations; none grants Goal or execution authority. -type LaunchObservation = {kind: "ready"} | {kind: "not_launched"}; +type LaunchObservation = {kind: "ready"} | {kind: "not_launched"; reason?: "goal_launch_not_allowed"}; +class HostLaunchLeaseExpired extends Error {} const launchDeadline = process.argv[2] === "--guarded-launch" ? Number(process.argv[3]) : null; if (launchDeadline !== null && (!Number.isFinite(launchDeadline) || launchDeadline <= 0)) throw new TypeError("invalid guarded Host bridge deadline"); @@ -41,18 +42,31 @@ process.stdin.on("data", (chunk: Buffer) => { } if (raw.timeout_ms !== null) raw.timeout_ms = Math.min(raw.timeout_ms, remaining); } + const expiresAt = raw.launch_lease_expires_at_ms; + delete raw.launch_lease_expires_at_ms; + if (expiresAt !== undefined && (launchDeadline === null || !Number.isSafeInteger(expiresAt))) + throw new TypeError("invalid guarded Host lease deadline"); + const beforeSpawn = expiresAt === undefined ? undefined : () => { + if (Date.now() >= expiresAt) throw new HostLaunchLeaseExpired(); + }; const lease = raw.delegated_lease; delete raw.delegated_lease; const request = decodeHostProcessRequest(raw); // The owned process group is reported before either path can run unaccounted. - if (lease === undefined) await emit(await runHostProcess(request, emit, owner.signal, undefined, {spawned: emit})); + if (lease === undefined) { + await emit(await runHostProcess(request, emit, owner.signal, undefined, {beforeSpawn, spawned: emit})); + } else { // Ordinary Hosts do not load canonical lease/provider modules. const {decodeDelegatedHostLease, runLeasedHostProcess} = await import("./leased_host_process.ts"); - await emit(await runLeasedHostProcess(request, decodeDelegatedHostLease(lease), emit, owner.signal, emit)); + await emit(await runLeasedHostProcess(request, decodeDelegatedHostLease(lease), emit, owner.signal, emit, beforeSpawn)); } } - catch { process.exitCode = 1; } + catch (error) { + if (error instanceof HostLaunchLeaseExpired) { + await emit({kind: "not_launched", reason: "goal_launch_not_allowed"} satisfies LaunchObservation); + } else process.exitCode = 1; + } finally { process.stdin.destroy(); } })(); }); diff --git a/loopx/control_plane/turn_driver/host_process_transport.py b/loopx/control_plane/turn_driver/host_process_transport.py index 07a1da28e0..330c0f6bcb 100644 --- a/loopx/control_plane/turn_driver/host_process_transport.py +++ b/loopx/control_plane/turn_driver/host_process_transport.py @@ -306,7 +306,7 @@ def run_host_process( on_stderr: Callable[[str], None] | None = None, delegated_lease: dict[str, Any] | None = None, environment: dict[str, str] | None = None, - launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, check_running: Callable[[], None] | None = None, interrupt_before_terminate: bool = False, ) -> dict[str, Any]: @@ -370,7 +370,9 @@ def run_host_process( if windows_job is not None: windows_job.assign(proc) if launch_guard is not None: - launch_scope.enter_context(launch_guard()) + expires_at_ms = launch_scope.enter_context(launch_guard()) + if expires_at_ms is not None: + request["launch_lease_expires_at_ms"] = expires_at_ms record = _begin_host_process_record(record_path, record_value=record_value, supervises=supervises, supervision=supervision, bridge_pid=proc.pid) if record is not None and windows_job is not None: @@ -417,6 +419,10 @@ def run_host_process( launch_scope.close() elif kind == "not_launched" and guarded is not None: guarded.not_launched = True + if event.get("reason") == "goal_launch_not_allowed": + from ..goals.first_party_host_admission import FirstPartyHostLaunchRejected + + raise FirstPartyHostLaunchRejected("goal_launch_not_allowed") raise TimeoutError("managed Host bridge launch deadline expired") else: raise RuntimeError("Invalid managed Host process observation") diff --git a/loopx/control_plane/turn_driver/leased_host_process.ts b/loopx/control_plane/turn_driver/leased_host_process.ts index e0371a9c0b..c24b877397 100644 --- a/loopx/control_plane/turn_driver/leased_host_process.ts +++ b/loopx/control_plane/turn_driver/leased_host_process.ts @@ -49,7 +49,8 @@ export function decodeDelegatedHostLease(raw: unknown): DelegatedHostLease { export async function runLeasedHostProcess(request: HostProcessRequest, context: DelegatedHostLease, output: (item: HostProcessOutput) => Promise, owner: AbortSignal, - spawned?: (item: HostProcessSpawned) => Promise): Promise { + spawned?: (item: HostProcessSpawned) => Promise, + beforeSpawn?: () => void): Promise { const controller = new AbortController(); const abort = () => controller.abort(); owner.addEventListener("abort", abort, {once: true}); @@ -142,7 +143,7 @@ export async function runLeasedHostProcess(request: HostProcessRequest, context: // The CLI's TERM adapter unwinds the nested Host transport (bounded at // five seconds). Allow that acknowledgement before a forced group kill. // This is private supervisor behavior, not a model-controlled timeout. - const result = await runHostProcess(request, output, controller.signal, 6000, {spawned}); + const result = await runHostProcess(request, output, controller.signal, 6000, {beforeSpawn, spawned}); // Keep the proved expiry armed while a renewal reply is in flight. A // returned model result cannot make a hung lease transport authoritative. finished = true; diff --git a/loopx/control_plane/turn_driver/turn_journal_query.ts b/loopx/control_plane/turn_driver/turn_journal_query.ts index 6ddf11cdfd..8f18277d85 100644 --- a/loopx/control_plane/turn_driver/turn_journal_query.ts +++ b/loopx/control_plane/turn_driver/turn_journal_query.ts @@ -27,6 +27,7 @@ import { interface JournalEvidence { turn_key: string; + recovery_reason: string; observed_capabilities: string[] | null; } @@ -136,6 +137,7 @@ function evidenceForJournal( } return { turn_key: turnKey, + recovery_reason: inspection.recovery_decision.reason, observed_capabilities: inspection.replay_legal ? observedCapabilities(turnPlanWorkSource(plan ?? {}, sourceNumberTokens(journal, parsedSourceNumberTokens(journal, numbers)).filter(({path}) => path[0] === "plan" && path[1] === "work_contract") @@ -143,19 +145,65 @@ function evidenceForJournal( }; } -async function queryTurnJournal( - runtimeRootValue: unknown, - identity: JournalAddress | BoundSettlementIdentity, - goalRef: ExactGoalRef | null, -): Promise { +async function readJournalSnapshot(path: string): Promise> { + // Journals are atomically replaced. Read a complete snapshot without a + // reader lock or following a link to a different authority. + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + if (!(await handle.stat()).isFile()) journalConflict("Turn journal snapshot is not a regular file"); + try { + return parseJsonSource(new TextDecoder("utf-8", { fatal: true }).decode(await handle.readFile())); + } catch (error) { + if (error instanceof SyntaxError || error instanceof TypeError) { + journalConflict("Turn journal history is unreadable; reconcile it before retrying"); + } + throw error; + } + } finally { + await handle.close(); + } +} + +function turnJournalDirectory(runtimeRootValue: unknown, goalId: string): string { const runtimeRoot = requireNonEmptyString(runtimeRootValue, "runtime_root"); if (!isAbsolute(runtimeRoot) || runtimeRoot.includes("\0")) { throw new EffectRuntimeRequestError("Turn journal runtime_root must be an absolute path"); } - if ([".", ".."].includes(identity.goal_id) || /[/\\\0]/u.test(identity.goal_id)) { + if ([".", ".."].includes(goalId) || /[/\\\0]/u.test(goalId)) { throw new EffectRuntimeRequestError("Turn journal goal_id must be a single path segment"); } - const directory = join(runtimeRoot, "goals", identity.goal_id, "turns"); + return join(runtimeRoot, "goals", goalId, "turns"); +} + +/** An attempt linked by the bound receipt cannot be excluded by a missing journal. */ +export async function turnJournalHasUnconfirmedHost( + runtimeRoot: string, turnKey: string, settlementIdentity: JsonObject, goalRef: unknown, +): Promise { + if (!/^sha256:[a-f0-9]{64}$/u.test(turnKey)) { + journalConflict("The admitted Turn journal locator is invalid"); + } + const identity = decodeIdentity(settlementIdentity); + const directory = turnJournalDirectory(runtimeRoot, identity.goal_id); + let parsed; + try { + parsed = await readJournalSnapshot(join(directory, turnKey.slice("sha256:".length) + ".json")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; + throw error; + } + const evidence = evidenceForJournal( + parsed.value, turnKey, identity, goalRef === undefined ? null : decodeGoalRef(goalRef), parsed.numbers, + ); + if (!evidence) journalConflict("The admitted Turn journal belongs to another settlement identity"); + return evidence.recovery_reason === "host_outcome_unknown"; +} + +async function queryTurnJournal( + runtimeRootValue: unknown, + identity: JournalAddress | BoundSettlementIdentity, + goalRef: ExactGoalRef | null, +): Promise { + const directory = turnJournalDirectory(runtimeRootValue, identity.goal_id); let entries; try { entries = await readdir(directory, { withFileTypes: true }); @@ -167,22 +215,7 @@ async function queryTurnJournal( for (const entry of entries.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) { if (!/^[a-f0-9]{64}\.json$/u.test(entry.name)) continue; if (!entry.isFile()) journalConflict("Turn journal history contains a non-regular snapshot; reconcile it before retrying"); - let parsed: ReturnType; - // Journals are atomic-replace files. Read one complete version without - // creating reader lock files; do not follow a replaced symlink. This is - // not a snapshot of the directory or proof of provider-side absence. - const handle = await open(join(directory, entry.name), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); - try { - if (!(await handle.stat()).isFile()) journalConflict("Turn journal snapshot is not a regular file"); - try { - parsed = parseJsonSource(new TextDecoder("utf-8", { fatal: true }).decode(await handle.readFile())); - } catch (error) { - if (error instanceof SyntaxError || error instanceof TypeError) journalConflict("Turn journal history is unreadable; reconcile it before retrying"); - throw error; - } - } finally { - await handle.close(); - } + const parsed = await readJournalSnapshot(join(directory, entry.name)); const evidence = evidenceForJournal( parsed.value, `sha256:${entry.name.slice(0, -5)}`, diff --git a/loopx/control_plane/work_items/execution_preparation.ts b/loopx/control_plane/work_items/execution_preparation.ts index 30c1f4ce17..882f0a7bc3 100644 --- a/loopx/control_plane/work_items/execution_preparation.ts +++ b/loopx/control_plane/work_items/execution_preparation.ts @@ -1,6 +1,7 @@ import {HANDOFF_MODES} from "../coordination/handoff_mode_vocabulary.ts"; import {HANDOFF_MODE_RETIRED_REASON, retiredHandoffModeRejection} from "../coordination/handoff_mode_policy.ts"; import type {JsonObject} from "../effect_program.ts"; +import {parseIsoTimestamp} from "../runtime_timestamp.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import { optionalNonEmptyString, @@ -36,8 +37,14 @@ function leaseFact(value: unknown, label: string): JsonObject { } proof = {idempotency_key: key, expected_version: version}; } + const expiresAt = optionalNonEmptyString(lease.expires_at, `${label}.expires_at`); + const deadline = expiresAt === null ? null : parseIsoTimestamp(expiresAt); + if (expiresAt !== null && deadline === null) { + throw new EffectRuntimeRequestError(`${label}.expires_at must be a valid ISO timestamp`); + } return {status, active, ...(owner === null ? {} : {owner}), - ...(proof === undefined ? {} : {lease_proof: proof})}; + ...(proof === undefined ? {} : {lease_proof: proof}), + ...(deadline === null ? {} : {lease_expires_at_ms: deadline.valueOf()})}; } function claimAction(goalId: string, todoId: string, agentId: string): JsonObject { @@ -128,7 +135,9 @@ export function projectExecutionPreparation(value: unknown): JsonObject { reason_code: "matching_active_task_lease", // Capture the admitted generation once. Recovery must not borrow a // replacement lease after the original Host result was produced. - ...(active[0].lease_proof === undefined ? {} : {lease_proof: active[0].lease_proof})}; + ...(active[0].lease_proof === undefined ? {} : {lease_proof: active[0].lease_proof}), + // Only the live Host launch read requests the same row's deadline. + ...(active[0].lease_expires_at_ms === undefined ? {} : {lease_expires_at_ms: active[0].lease_expires_at_ms})}; } const prior = leases[0]; diff --git a/loopx/extensions/process_runtime.py b/loopx/extensions/process_runtime.py index bb7875e279..a9d545a2d8 100644 --- a/loopx/extensions/process_runtime.py +++ b/loopx/extensions/process_runtime.py @@ -9,7 +9,8 @@ import sys import threading import time -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any, BinaryIO, IO, cast @@ -19,23 +20,118 @@ _PROCESS_GROUP_STOP_TIMEOUT_SECONDS = 1.0 +class ProcessStdinLeaseExpired(TimeoutError): + """The lease expired before a complete frame could be published.""" + + +class ProcessStdinDispatchUnconfirmed(Exception): + """The original pipe receiver may have received the frame without timely proof.""" + + +@contextmanager +def _nonblocking_pipe_writer(fd: int) -> Iterator[Callable[[memoryview], int]]: + if os.name == "nt": + # os.set_blocking only supports Windows pipes from Python 3.12. + import _winapi + import msvcrt + + handle = msvcrt.get_osfhandle(fd) + _winapi.SetNamedPipeHandleState(handle, 1, None, None) # PIPE_NOWAIT + try: + yield lambda data: _winapi.WriteFile(handle, data)[0] + finally: + _winapi.SetNamedPipeHandleState(handle, 0, None, None) # PIPE_WAIT + else: + was_blocking = os.get_blocking(fd) + os.set_blocking(fd, False) + try: + yield lambda data: os.write(fd, data) + finally: + os.set_blocking(fd, was_blocking) + + +def _retire_native_input_receiver( + process: subprocess.Popen[str], cancel_and_reap: Callable[[], None], +) -> None: + try: + cancel_and_reap() + except BaseException as exc: + raise RuntimeError("native input receiver cleanup unconfirmed") from exc + if process.poll() is None: + raise RuntimeError("native input receiver was not reaped") + + +def _write_lease_bound_frame( + process: subprocess.Popen[str], stream: IO[str], text: str, *, + deadline: float, expires_at_ms: int, cancel_and_reap: Callable[[], None], +) -> None: + if time.time() * 1000 >= expires_at_ms: + raise ProcessStdinLeaseExpired("native input lease expired before dispatch") + frame = memoryview(text.replace("\n", os.linesep).encode(stream.encoding, errors=stream.errors)) + try: + # A blocking write can publish late through backpressure. The final + # nonblocking write still needs a completion check because the sender + # can be preempted between checking time and entering the syscall. + with _nonblocking_pipe_writer(stream.fileno()) as write: + while frame: + chunk = frame[:_PROCESS_IO_CHUNK_BYTES] + remaining = (expires_at_ms - time.time() * 1000) / 1000 + if remaining <= 0: + raise ProcessStdinLeaseExpired("native input lease expired before dispatch") + budget = deadline - time.monotonic() + if budget <= 0: + raise TimeoutError("native input dispatch deadline expired") + try: + written = write(chunk) + except BlockingIOError: + written = 0 + if written: + frame = frame[written:] + if not frame and time.time() * 1000 >= expires_at_ms: + raise ProcessStdinDispatchUnconfirmed( + "The native frame may have reached the Host after lease expiry. " + "Reconcile the original Host attempt before retrying." + ) + else: + time.sleep(min(0.005, remaining, budget)) + except BaseException: + # Reap before closing stdin so an incomplete line cannot become an + # EOF-terminated request. Unconfirmed cleanup is still an unknown effect. + try: + _retire_native_input_receiver(process, cancel_and_reap) + except RuntimeError as cleanup_error: + raise ProcessStdinDispatchUnconfirmed( + "The original native Host could not be confirmed stopped. " + "Reconcile the original Host attempt before retrying." + ) from cleanup_error + raise + + def write_process_stdin_before_deadline( process: subprocess.Popen[str], text: str, *, deadline: float, - cancel_and_reap: Callable[[], None], + cancel_and_reap: Callable[[], None], lease_expires_at_ms: int | None = None, + retire_on_expired_deadline: bool = False, ) -> None: """Finish one native frame, or cancel its peer before returning failure. The caller owns the serialization and launch fence. A bounded wait keeps a full pipe from stranding that fence; the same process is reaped and its - writer joined before the caller releases it. Use the existing text stream - so successful large/Unicode frames retain their original wire semantics on - every supported Python platform, including Windows 3.11. + writer joined before the caller releases it. A lease-bound send checks + the deadline around nonblocking publication of the complete frame. + Both paths preserve the text stream's wire encoding. """ stream = process.stdin if stream is None: raise BrokenPipeError("native input stream is closed") if time.monotonic() >= deadline: + # Native admission can consume the RPC budget before a writer starts. + if retire_on_expired_deadline: + _retire_native_input_receiver(process, cancel_and_reap) raise TimeoutError("native input dispatch deadline expired") + if lease_expires_at_ms is not None: + _write_lease_bound_frame(process, stream, text, deadline=deadline, + expires_at_ms=lease_expires_at_ms, cancel_and_reap=cancel_and_reap) + return finished = threading.Event() failures: list[BaseException] = [] diff --git a/loopx/quota.py b/loopx/quota.py index 4d774c4980..ecc72331dc 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1088,6 +1088,7 @@ def build_quota_should_run( workspace_path: Path | None = None, goal_ref: Mapping[str, object] | None = None, authoritative_fallback_todo_items: FallbackTodoSource = None, + include_launch_lease_deadline: bool = False, app_scheduler_state_observer: Callable[[Mapping[str, Any]], None] | None = None, ) -> dict[str, Any]: from .control_plane.quota.should_run import ( @@ -1135,6 +1136,7 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None) -> dict[s goal_ref=goal_ref, authoritative_fallback_todo_items=authoritative_fallback_todo_items, replan_explore_reader=read_explore_context, + include_launch_lease_deadline=include_launch_lease_deadline, app_scheduler_state_observer=app_scheduler_state_observer, ) diff --git a/tests/capabilities/test_provider_gateway_observation.py b/tests/capabilities/test_provider_gateway_observation.py index f2e21156b0..1df1a6be52 100644 --- a/tests/capabilities/test_provider_gateway_observation.py +++ b/tests/capabilities/test_provider_gateway_observation.py @@ -17,13 +17,14 @@ class _Admission: """Expose the lock ordering without introducing a second authority policy.""" - def __init__(self): + def __init__(self, *, expires_at_ms=None): self.lock = threading.Lock() self.local = threading.local() self.entries = 0 self.pause_second = False self.paused = threading.Event() self.resume = threading.Event() + self.expires_at_ms = expires_at_ms def with_launch_decision(self, _check): return self @@ -37,7 +38,7 @@ def launch_lifetime(self, **_kwargs): if self.pause_second and self.entries == 2: self.paused.set() assert self.resume.wait(timeout=10) - yield + yield self.expires_at_ms finally: self.local.held = False @@ -76,7 +77,7 @@ def do_POST(self): thread.join(timeout=5) -def _guard(tmp_path, upstream, admission): +def _guard(tmp_path, upstream, admission, *, process=None): receipt = {} session = SimpleNamespace(admission=admission, goal_ref={}, session_id="synthetic-session") guard = ProviderRequestAdmission( @@ -84,7 +85,7 @@ def _guard(tmp_path, upstream, admission): "LOOPX_TURN": "synthetic-turn"}, session, receipt, tmp_path, ) - guard.started(SimpleNamespace(poll=lambda: None)) + guard.started(process if process is not None else SimpleNamespace(poll=lambda: None)) return guard, receipt @@ -111,6 +112,23 @@ def run(): return thread, result +def test_expired_host_authority_refuses_provider_post(tmp_path, monkeypatch): + signals = [] + monkeypatch.setattr("benchmark.runtime.provider_request_admission.os.killpg", + lambda pid, signal: signals.append((pid, signal))) + process = SimpleNamespace(pid=1, poll=lambda: None) + with _waiting_upstream() as (upstream, received, _, _): + guard, receipt = _guard(tmp_path, upstream, _Admission(expires_at_ms=1), process=process) + with guard.serve(["codex", "exec", "-"]): + client, result = _request(guard.base_url) + client.join(timeout=5) + assert not client.is_alive() and result.get("status") == 409, result + assert not received.is_set() + summary, = receipt["provider_request_invocations"] + assert summary["refusal_code"] == "goal_launch_not_allowed" and summary["sent"] == 0 + assert len(signals) == 1 and signals[0][0] == process.pid + + def test_provider_observes_dispatch_while_upstream_response_is_pending(tmp_path, monkeypatch): with _waiting_upstream() as (upstream, received, release, observed): admission = _Admission() diff --git a/tests/control_plane/host_process_fixture.py b/tests/control_plane/host_process_fixture.py index e55e6dd868..d0e31a6442 100644 --- a/tests/control_plane/host_process_fixture.py +++ b/tests/control_plane/host_process_fixture.py @@ -2,8 +2,124 @@ import os import json +import subprocess import sys +import time +from contextlib import contextmanager +from pathlib import Path + +WAIT_FOR_FIXTURE_LEASE_RELEASE = """\ +import os +import time + +release_path = os.environ.get("LOOPX_TEST_LEASE_RELEASED") +if release_path: + deadline = time.monotonic() + 10 + while not os.path.isfile(release_path): + if time.monotonic() >= deadline: + raise RuntimeError("lease writer did not finish while Host work was running") + time.sleep(0.025) +""" + + +@contextmanager +def expect_lease_release_after_dispatch(monkeypatch, *, operation, registry, runtime, goal_id, + todo_id, lease, dispatch_marker): + """Contend with the canonical writer from a separate process at dispatch.""" + import pytest + + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + + original = FirstPartyHostGoalAdmission.launch_lifetime + initial = dispatch_marker.read_bytes() if dispatch_marker.exists() else None + ready = registry.parent / "lease-release-ready" + released = registry.parent / "lease-released" + scope = dict(registry=str(registry), runtime=str(runtime), goal=goal_id, todo=todo_id, + owner=lease["owner"], key=lease["idempotency_key"], version=lease["version"], + ready=str(ready), released=str(released)) + release_client = """ +import json, sys from pathlib import Path +from loopx.control_plane.work_items.task_lease import inspect_task_lease, release_task_lease + +data = json.loads(sys.argv[1]) +scope = dict(registry_path=Path(data['registry']), runtime_root=Path(data['runtime']), + goal_id=data['goal'], todo_id=data['todo']) +assert inspect_task_lease(**scope)['lease']['version'] == data['version'] +Path(data['ready']).touch() +result = release_task_lease(**scope, owner=data['owner'], idempotency_key=data['key'], + expected_version=data['version']) +assert result['ok'] and result['released'], result +Path(data['released']).touch() +print(json.dumps(result)) +""" + child = None + + @contextmanager + def contend(admission, *, operation: str): + nonlocal child + with original(admission, operation=operation) as expires_at_ms: + if operation == dispatch_operation: + assert child is None + assert (dispatch_marker.read_bytes() if dispatch_marker.exists() else None) == initial + child = subprocess.Popen([sys.executable, "-c", release_client, json.dumps(scope)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + deadline = time.monotonic() + 8 + while not ready.exists() and child.poll() is None and time.monotonic() < deadline: + time.sleep(0.025) + assert ready.exists(), "separate lease writer failed to observe the original lease" + try: + output = child.communicate(timeout=1.5) + except subprocess.TimeoutExpired: + pass + else: + pytest.fail(f"lease release crossed the dispatch guard: {output}") + yield expires_at_ms + + dispatch_operation = operation + try: + with monkeypatch.context() as patch: + patch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", contend) + patch.setenv("LOOPX_TEST_LEASE_RELEASED", str(released)) + yield + assert child is not None, "expected Host dispatch guard was not entered" + stdout, stderr = child.communicate(timeout=15) + assert child.returncode == 0, (stdout, stderr) + result = json.loads(stdout) + assert result["ok"] and result["released"], result + finally: + if child is not None and child.poll() is None: + child.kill() + child.communicate(timeout=5) + + +@contextmanager +def expect_lease_expiry_during_dispatch(monkeypatch, *, operation, observed_lease): + """Let an unchanged canonical lease expire after the actual launch read.""" + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + from loopx.control_plane.work_items.task_lease import lease_expires_at, lease_is_active + + original = FirstPartyHostGoalAdmission.launch_lifetime + crossed = [] + + @contextmanager + def pause(admission, *, operation): + with original(admission, operation=operation) as expires_at_ms: + if operation == dispatch_operation: + lease = observed_lease() + assert lease_is_active(lease), lease + expires_at = lease_expires_at(lease) + assert expires_at is not None + while lease_is_active(lease): + time.sleep(min(0.02, max(0, expires_at.timestamp() - time.time()))) + crossed.append(lease["idempotency_key"]) + yield expires_at_ms + + dispatch_operation = operation + with monkeypatch.context() as patch: + patch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", pause) + yield + assert len(crossed) == 1, "expected one admitted dispatch to cross the lease deadline" def assert_host_task_context(model_input, source): diff --git a/tests/control_plane/test_benchmark_checkpoint_admission.py b/tests/control_plane/test_benchmark_checkpoint_admission.py index a19613ab93..aeb32e0ceb 100644 --- a/tests/control_plane/test_benchmark_checkpoint_admission.py +++ b/tests/control_plane/test_benchmark_checkpoint_admission.py @@ -167,8 +167,11 @@ def test_checkpoint_resume_rechecks_current_source_with_original_guard(tmp_path, @pytest.mark.parametrize("provider", ["file", "sqlite"]) -def test_checkpoint_fresh_todo_binding_keeps_original_turn(tmp_path, monkeypatch, provider): - _run_checkpoint(tmp_path, monkeypatch, provider, refusal=None, todo_bound=True) +@pytest.mark.parametrize(("handoff_mode", "refusal"), [ + ("soft_claim", None), ("hard_lease", None), ("soft_claim", "original_replan_changed"), +]) +def test_checkpoint_fresh_todo_binding_keeps_original_turn(tmp_path, monkeypatch, provider, handoff_mode, refusal): + _run_checkpoint(tmp_path, monkeypatch, provider, refusal=refusal, todo_bound=True, handoff_mode=handoff_mode) @pytest.mark.parametrize("provider", ["file", "sqlite"]) @@ -206,10 +209,12 @@ def test_checkpoint_request_admission_creates_a_new_invocation_for_original_turn assert all(endpoint.split("=", 1)[1] not in home.read_text() for endpoint in endpoints) -def _run_checkpoint(tmp_path, monkeypatch, provider, *, refusal, todo_bound, unbound=False, workspace_change=False, work_bound=False, provider_url=None): +def _run_checkpoint(tmp_path, monkeypatch, provider, *, refusal, todo_bound, unbound=False, workspace_change=False, + work_bound=False, provider_url=None, handoff_mode="soft_claim"): unbound = unbound or refusal == "original_event_changed" env, project, runtime, source, state = _environment( - tmp_path, monkeypatch, provider, replan="todo" if todo_bound or unbound else not work_bound) + tmp_path, monkeypatch, provider, replan="todo" if todo_bound or unbound else not work_bound, + handoff_mode=handoff_mode) env["CODEX_BIN"] = str(write_python_host_fixture(project / "checkpoint-host", f"#!{sys.executable}\n" + _HOST)) env = process_environment(Path(env["LOOPX_CODEX_HOME"]), base=env) env.update(LOOPX_TURN=OUTER_TURN, FIXTURE_OUTER_TURN=OUTER_TURN, FIXTURE_HELD_TURN=HELD_TURN) @@ -257,6 +262,8 @@ def captured_then_changed(held, deadline): assert held["quota"]["heartbeat_receipt"]["settlement_binding_owed"] is True if todo_bound: assert held["quota"]["heartbeat_receipt"]["settlement_identity"]["todo_id"] == fixture.SELECTED_REPLAN_TODO_ID + assert held["quota"]["normal_delivery_allowed"] is False + assert "execution_preparation" not in held["quota"] env["FIXTURE_HELD_QUOTA"] = str(held["path"]) if workspace_change: _private_workspace(project) @@ -292,6 +299,9 @@ def captured_then_changed(held, deadline): **original_receipt["settlement_identity"], "effect_id": "different-original-effect"}}} elif refusal == "original_action_changed": held["quota"] = {**held["quota"], "effective_action": "skip"} + elif refusal == "original_replan_changed": + held["quota"] = {**held["quota"], "heartbeat_receipt": { + **held["quota"]["heartbeat_receipt"], "semantic_replan_obligation_id": "replan-1000000000000001"}} return captured monkeypatch.setattr(hook, "_capture", captured_then_changed) @@ -330,6 +340,7 @@ def captured_then_changed(held, deadline): "stop":"goal_stopped", "original_event_changed":"heartbeat_launch_original_binding_changed", "original_settlement_changed":"heartbeat_launch_original_binding_changed", "original_action_changed":"goal_launch_not_allowed", + "original_replan_changed":"heartbeat_launch_original_binding_changed", }[refusal] in str(error) assert not (project / "checkpoint-business.json").exists() assert fixture._heartbeat_receipt_events(runtime, HELD_TURN) == observations["events"] diff --git a/tests/control_plane/test_benchmark_current_admission.py b/tests/control_plane/test_benchmark_current_admission.py index 62740ffc74..8728fe2bb8 100644 --- a/tests/control_plane/test_benchmark_current_admission.py +++ b/tests/control_plane/test_benchmark_current_admission.py @@ -1,8 +1,11 @@ """Real heartbeat Hosts consume current source eligibility before native launch.""" from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager import json from pathlib import Path +import subprocess import sys import time @@ -12,10 +15,12 @@ from benchmark.tests.test_shared_codex_runtime import worker_env from loopx.configure_goal import configure_goal from loopx.cli_commands import turn_decision +from loopx.control_plane.coordination.shadow_management import shadow_maintenance_lock_target from loopx.control_plane.goals import activation_service -from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected +from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission, FirstPartyHostRuntimeRejected from loopx import contract, status as status_owner from loopx.cli import main +from loopx.file_lock import LockAcquireTimeoutError, exclusive_mutation_file_lock from loopx.global_registry import sync_project_registry_to_global from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime from tests.control_plane.host_process_fixture import write_python_host_fixture @@ -37,7 +42,7 @@ def _private_workspace(project): return path -def _environment(tmp_path, monkeypatch, provider, *, replan=False): +def _environment(tmp_path, monkeypatch, provider, *, replan=False, handoff_mode="soft_claim"): isolate_sqlite_runtime(tmp_path, monkeypatch) monkeypatch.setenv("LOOPX_USAGE_PING", "0") monkeypatch.setenv("PYTHONPATH", str(ROOT)) @@ -55,7 +60,7 @@ def _environment(tmp_path, monkeypatch, provider, *, replan=False): fixture._configure_selected_todo_replan_fixture(project, source) elif replan: fixture._configure_autonomous_replan_fixture(project, runtime, source) - fixture._initialize_fixture_authority(project, runtime, provider=provider) + fixture._initialize_fixture_authority(project, runtime, provider=provider, handoff_mode=handoff_mode) synced = sync_project_registry_to_global( registry_path=source, runtime_root_override=str(runtime), goal_id=fixture.GOAL_ID, ) @@ -154,6 +159,88 @@ def after_session_selection(*arguments): assert not index.exists() or "quota_slot_spent" not in index.read_text() +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("expire_after_check", [False, True]) +def test_heartbeat_worker_consumes_hard_lease_deadline_at_spawn( + tmp_path, monkeypatch, provider, expire_after_check, +): + env, project, runtime, source, _ = _environment(tmp_path, monkeypatch, provider, handoff_mode="hard_lease") + lease_command = ["--goal-id", fixture.GOAL_ID, "--todo-id", fixture.TODO_ID, + "--owner", fixture.AGENT_ID, "--idempotency-key", "benchmark-original-lease"] + code, acquired = fixture._run_cli(source, runtime, "task-lease", "acquire", + *lease_command, "--expected-version", "0", "--ttl-seconds", "300") + assert code == 0 and acquired["ok"], acquired + if expire_after_check: + native_command = worker.native_command + launch_lifetime = FirstPartyHostGoalAdmission.launch_lifetime + + def shorten_lease(*args, **kwargs): + command = native_command(*args, **kwargs) + code, renewed = fixture._run_cli(source, runtime, "task-lease", "renew", + *lease_command, "--expected-version", str(acquired["lease"]["version"]), "--ttl-seconds", "6") + assert code == 0 and renewed["lease"]["lease_epoch"] == acquired["lease"]["lease_epoch"], renewed + return command + + @contextmanager + def wait_after_check(admission, *, operation): + with launch_lifetime(admission, operation=operation) as deadline: + if operation == "codex_agent_host_launch": + assert deadline is not None and deadline > time.time() * 1000 + time.sleep(max(0, deadline / 1000 - time.time()) + 0.01) + yield deadline + + monkeypatch.setattr(worker, "native_command", shorten_lease) + monkeypatch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", wait_after_check) + with pytest.raises(FirstPartyHostRuntimeRejected, match="goal_launch_not_allowed"): + worker.run_once(env) + assert not (project / "native-starts").exists() + assert not (project / "model-starts").exists() + else: + assert worker.run_once(env)["ok"] + assert (project / "native-starts").read_text().splitlines() == ["native"] + assert (project / "model-starts").read_text().splitlines() == ["model"] + assert fixture._spend_run_count(runtime) == 0 + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_heartbeat_worker_holds_the_lease_writer_until_native_spawn(tmp_path, monkeypatch, provider): + env, project, runtime, source, _ = _environment(tmp_path, monkeypatch, provider, handoff_mode="hard_lease") + lease_args = ["--goal-id", fixture.GOAL_ID, "--todo-id", fixture.TODO_ID, + "--owner", fixture.AGENT_ID, "--idempotency-key", "benchmark-worker-spawn-lease"] + code, acquired = fixture._run_cli(source, runtime, "task-lease", "acquire", + *lease_args, "--expected-version", "0", "--ttl-seconds", "60", cwd=project) + assert code == 0 and acquired["ok"], acquired + real_popen = subprocess.Popen + mutations = [] + + def release(): + code, result = fixture._run_cli(source, runtime, "task-lease", "release", + *lease_args, "--expected-version", str(acquired["lease"]["version"]), cwd=project) + assert code == 0 and result["ok"], result + return result["lease"] + + with ThreadPoolExecutor(max_workers=1) as writer: + def start_with_release(args, *extra, **kwargs): + if isinstance(args, list) and args and args[0] == env["CODEX_BIN"]: + target = shadow_maintenance_lock_target(runtime, fixture.GOAL_ID) + with pytest.raises(LockAcquireTimeoutError): + with exclusive_mutation_file_lock(target, timeout_seconds=0): + pytest.fail("canonical lease writer could enter before Popen") + mutations.append(writer.submit(release)) + return real_popen(args, *extra, **kwargs) + + monkeypatch.setattr(worker.subprocess, "Popen", start_with_release) + result = worker.run_once(env) + assert result["ok"], result + released, = mutations + assert released.result(timeout=10)["status"] == "released" + + assert (project / "native-starts").read_text().splitlines() == ["native"] + assert (project / "model-starts").read_text().splitlines() == ["model"] + assert fixture._heartbeat_receipt_count(runtime, result["turn_id"]) == 0 + assert fixture._spend_run_count(runtime) == 0 + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize("mode", ["heartbeat", "turn"]) def test_initial_benchmark_planner_can_launch_without_a_prior_heartbeat_receipt(tmp_path, monkeypatch, provider, mode): diff --git a/tests/control_plane/test_codex_operation_launch.py b/tests/control_plane/test_codex_operation_launch.py index cf853b7360..cf34261f70 100644 --- a/tests/control_plane/test_codex_operation_launch.py +++ b/tests/control_plane/test_codex_operation_launch.py @@ -19,6 +19,7 @@ from loopx.control_plane.goals import activation_service, first_party_host_admission from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission from loopx.control_plane.projects.registry_codec import project_registry_transaction +from loopx.control_plane.work_items.task_lease import lease_expires_at from loopx.control_plane.turn_driver import codex_operation_host from loopx.global_registry import sync_project_registry_to_global from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime @@ -27,9 +28,9 @@ from tests.test_loopx_turn_driver import _write_live_fixture -def _fixture(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str): +def _fixture(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, *, handoff_mode="soft_claim"): isolate_sqlite_runtime(tmp_path, monkeypatch) - project, runtime, registry = _write_live_fixture(tmp_path, provider=provider) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider, handoff_mode=handoff_mode) native, turns = tmp_path / "native-starts", tmp_path / "native-turns" executable = tmp_path / "fake-codex-operation" executable.write_text(FAKE_SERVER.replace( @@ -559,8 +560,185 @@ def lifetime(self, *, operation): assert not index.exists() or "quota_slot_spent" not in index.read_text() -def test_guarded_native_dispatch_preserves_large_unicode_input(tmp_path, monkeypatch): - _, _, _, turns, arguments = _fixture(tmp_path, monkeypatch, "sqlite") +def _lease(arguments, action, *, version, ttl): + code, payload = _cli([ + *arguments[:6], "task-lease", action, "--goal-id", "loopx-turn-fixture", + "--todo-id", "todo_fixture0001", "--owner", "codex-fixture", + "--idempotency-key", "native-input-lease", "--expected-version", str(version), + "--ttl-seconds", str(ttl), + ]) + assert code == 0, payload + return payload["lease"] + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_lease_expiry_during_final_pipe_write_keeps_original_attempt_unconfirmed(tmp_path, monkeypatch, provider): + from loopx.control_plane.turn_driver.journal_store import turn_journal_path + from loopx.extensions import process_runtime + + runtime, _, native, turns, arguments = _fixture(tmp_path, monkeypatch, provider, handoff_mode="hard_lease") + original = _lease(arguments, "acquire", version=0, ttl=300) + prepare_input = codex_operation_host.prepare_codex_cli_input + pipe_writer = process_runtime._nonblocking_pipe_writer + expires_at = [] + sent_at = [] + + def renew_before_dispatch(request): + prepared = prepare_input(request) + renewed = _lease(arguments, "renew", version=original["version"], ttl=10) + assert renewed["lease_epoch"] == original["lease_epoch"] + expires_at.append(lease_expires_at(renewed).timestamp()) + return prepared + + @contextlib.contextmanager + def preempt_final_write(fd): + with pipe_writer(fd) as write: + def delayed(chunk): + if chunk[-1] != ord("\n"): + return write(chunk) + if len(chunk) > 1: + # Withhold exactly the terminator so the fixture cannot + # interpret the real pipe's partial frame as a request. + return write(chunk[:-1]) + assert len(expires_at) == 1 and time.time() < expires_at[0] + time.sleep(expires_at[0] - time.time() + .03) + count = write(chunk) + assert count == 1 + sent_at.append(time.time()) + limit = time.monotonic() + 2 + while not (turns.exists() and turns.stat().st_size) and time.monotonic() < limit: + time.sleep(.005) + return count + + yield delayed + + monkeypatch.setattr(codex_operation_host, "prepare_codex_cli_input", renew_before_dispatch) + monkeypatch.setattr(process_runtime, "_nonblocking_pipe_writer", preempt_final_write) + code, result = _cli([*arguments, "--execute"]) + assert len(sent_at) == 1 and sent_at[0] >= expires_at[0] + assert native.read_text().splitlines() == ["native"] and len(turns.read_text().splitlines()) == 1 + assert code == 1 and result["status"] == "in_progress", result + assert result["error_code"] == "host_outcome_unknown", result + assert result["effects"] == { + "host_invoked": True, "state_written": False, "quota_spent": False, "scheduler_acknowledged": False, + } + assert _bindings(runtime) + key = result["resume_turn_key"] + path = turn_journal_path(runtime, goal_id="loopx-turn-fixture", turn_key=key) + saved = path.read_bytes() + journal = json.loads(saved) + assert journal["turn_key"] == key and journal["host_attempt_count"] == 1 + assert journal["status"] == "in_progress" and not journal["completed_phases"] + capture = path.parent / "decisions" / (key.removeprefix("sha256:") + ".json") + original_capture = capture.read_bytes() + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + resume = list(arguments) + index_of_turn = resume.index("--turn-instance-id") + resume[index_of_turn:index_of_turn + 2] = ["--resume-turn-key", key] + code, replay = _cli([*resume, "--retry-failed-turn", "--execute"]) + assert code == 1 and replay["recovery_decision"]["reason"] == "host_outcome_unknown", replay + assert replay["recovery_decision"]["reinvoke_host"] is False + assert native.read_text().splitlines() == ["native"] and len(turns.read_text().splitlines()) == 1 + assert path.read_bytes() == saved and capture.read_bytes() == original_capture + guard_arguments = [ + *arguments[:6], "quota", "should-run", "--goal-id", "loopx-turn-fixture", + "--agent-id", "codex-fixture", + "--context-delivery", "off", "--delta-delivery", "off", + ] + code, guard = _cli([*guard_arguments, "--turn-instance-id", "next-after-unconfirmed-dispatch"]) + assert code == 0, guard + assert guard["should_run"] is False and guard["reason_code"] == "host_outcome_unknown", guard + assert guard["unsettled_host_turn_recovery"]["turn_key"] == key + moved = path.with_suffix(".relocated") + path.rename(moved) + try: + code, guard = _cli([*guard_arguments, "--turn-instance-id", "next-after-original-journal-lost"]) + assert code == 0, guard + assert guard["should_run"] is False and guard["reason_code"] == "host_outcome_unknown", guard + assert guard["unsettled_host_turn_recovery"]["turn_key"] == key + assert not path.exists() + finally: + moved.rename(path) + assert native.read_text().splitlines() == ["native"] and len(turns.read_text().splitlines()) == 1 + assert not index.exists() or "quota_slot_spent" not in index.read_text() + assert path.read_bytes() == saved and capture.read_bytes() == original_capture + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_expired_lease_cannot_publish_backpressured_native_frame(tmp_path, monkeypatch, provider): + from loopx.extensions import process_runtime + + runtime, _, native, turns, arguments = _fixture(tmp_path, monkeypatch, provider, handoff_mode="hard_lease") + original = _lease(arguments, "acquire", version=0, ttl=300) + pending = tmp_path / "partial-native-frame" + release = tmp_path / "release-native-reader" + resumed = tmp_path / "native-reader-resumed" + executable = Path(arguments[arguments.index("--codex-bin") + 1]) + ready = ' emit({"id": row["id"], "result": {"thread": {"id": thread}, "model": "test-model", "reasoningEffort": "xhigh"}})' + source = executable.read_text() + assert ready in source + executable.write_text(source.replace( + "for line in sys.stdin:\n row = json.loads(line)", + "prefix = ''\nwhile line := sys.stdin.readline():\n" + " row = json.loads(prefix + line)\n prefix = ''", 1, + ).replace(ready, ready + "\n" + " import time\n" + " prefix = sys.stdin.read(2048)\n" + f" pathlib.Path({str(pending)!r}).touch()\n" + f" while not pathlib.Path({str(release)!r}).exists(): time.sleep(.005)\n" + f" pathlib.Path({str(resumed)!r}).touch()", 1)) + original_input = codex_operation_host.prepare_codex_cli_input + original_cancel = process_runtime.force_terminate_process + deadline = [] + + def oversized_input(request): + text, schema = original_input(request) + renewed = _lease(arguments, "renew", version=original["version"], ttl=8) + assert renewed["lease_epoch"] == original["lease_epoch"] + deadline.append(lease_expires_at(renewed).timestamp()) + return text + "\n" + "界" * (1024 * 1024), schema + + def delay_retirement(process, **kwargs): + # Let the receiver start draining after expiry but before termination. + # An outstanding blocking write could still publish the complete line. + limit = time.monotonic() + 2 + while not resumed.exists() and time.monotonic() < limit: + time.sleep(.005) + limit = time.monotonic() + .25 + while not turns.exists() and time.monotonic() < limit: + time.sleep(.005) + original_cancel(process, **kwargs) + + monkeypatch.setattr(codex_operation_host, "prepare_codex_cli_input", oversized_input) + monkeypatch.setattr(process_runtime, "force_terminate_process", delay_retirement) + with ThreadPoolExecutor(max_workers=1) as executor: + host = executor.submit(_cli, [*arguments, "--execute"]) + try: + limit = time.monotonic() + 20 + while not pending.exists() and not host.done() and time.monotonic() < limit: + time.sleep(.01) + assert pending.exists() and time.time() < deadline[0], "the active lease must reach real pipe backpressure" + time.sleep(max(0, deadline[0] - time.time()) + .025) + release.touch() + code, payload = host.result(timeout=12) + finally: + release.touch() + + assert native.exists() and resumed.exists() and not turns.exists() + assert code == 1 and payload.get("error_code") == "goal_launch_not_allowed", payload + assert payload["host_failure"]["kind"] == "contract_rejected" + assert not any(payload["effects"].values()), payload + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("handoff_mode", ["soft_claim", "hard_lease"]) +def test_guarded_native_dispatch_preserves_large_unicode_input(tmp_path, monkeypatch, handoff_mode): + _, _, _, turns, arguments = _fixture(tmp_path, monkeypatch, "sqlite", handoff_mode=handoff_mode) + if handoff_mode == "hard_lease": + _lease(arguments, "acquire", version=0, ttl=300) observed = tmp_path / "accepted-native-input" monkeypatch.setenv("FAKE_OPERATION_PROMPT_FILE", str(observed)) original_input = codex_operation_host.prepare_codex_cli_input diff --git a/tests/control_plane/test_host_current_admission.py b/tests/control_plane/test_host_current_admission.py index f3819ec0e2..5a0d4101c4 100644 --- a/tests/control_plane/test_host_current_admission.py +++ b/tests/control_plane/test_host_current_admission.py @@ -2,6 +2,8 @@ from __future__ import annotations import json +import textwrap +from contextlib import nullcontext from pathlib import Path import sys @@ -11,6 +13,11 @@ from loopx.control_plane.turn_driver import codex_cli, codex_operation_host from loopx.global_registry import sync_project_registry_to_global from tests.control_plane.test_codex_operation_launch import _bindings, _cli, _fixture +from tests.control_plane.host_process_fixture import ( + WAIT_FOR_FIXTURE_LEASE_RELEASE, + expect_lease_expiry_during_dispatch, + expect_lease_release_after_dispatch, +) from tests.test_loopx_turn_codex_cli import _fake_codex @@ -154,8 +161,9 @@ def edit_after_planning(request): @pytest.mark.parametrize("provider", ["file", "sqlite"]) -@pytest.mark.parametrize("edit_task", [False, True]) -def test_stdio_retry_checks_original_task_before_launch(tmp_path, monkeypatch, provider, edit_task): +@pytest.mark.parametrize("change", ["unchanged", "text", "renew_lease", "replace_lease", + "release_during_dispatch", "expire_during_dispatch"]) +def test_stdio_retry_checks_original_task_before_launch(tmp_path, monkeypatch, provider, change): from loopx.control_plane.coordination.runtime_shadow import build_todo_runtime_shadow_projection from loopx.control_plane.todos.active_state_todo_parser import parse_active_state_todos from tests.control_plane.canonical_authority_fixture import initialize_canonical_authority, isolate_sqlite_runtime @@ -166,8 +174,10 @@ def test_stdio_retry_checks_original_task_before_launch(tmp_path, monkeypatch, p project, runtime, workspace, registry = fixture._write_fixture(tmp_path, turn_count=2) state = project / ".codex/goals" / fixture.GOAL_ID / "ACTIVE_GOAL_STATE.md" todos = parse_active_state_todos(state.read_text(), state_path=state, item_limit=None) + uses_lease = change in {"renew_lease", "replace_lease", "release_during_dispatch", "expire_during_dispatch"} initialize_canonical_authority(runtime, fixture.GOAL_ID, - build_todo_runtime_shadow_projection(goal_id=fixture.GOAL_ID, handoff_mode="soft_claim", + build_todo_runtime_shadow_projection(goal_id=fixture.GOAL_ID, + handoff_mode="hard_lease" if uses_lease else "soft_claim", todos=todos["agent_todos"]["items"]), state_path=state, provider=provider) marker = workspace / "process-count" runner = tmp_path / "runner.py" @@ -181,6 +191,7 @@ def run_dsh_turn(**options): marker.write_text("1" if first else "2") if first: raise TimeoutError("Fixture failed before producing a result") +{textwrap.indent(WAIT_FOR_FIXTURE_LEASE_RELEASE, " ")} (workspace / {fixture.MARKER_NAME!r}).write_text({fixture._marker_value(1)!r}) return json.dumps({{"result_kind": "validated_progress", "classification": "checked_work", "summary": "The original work passed independent validation.", "next_action": "Continue.", @@ -188,6 +199,17 @@ def run_dsh_turn(**options): ''', encoding="utf-8") prefix = ["--registry", str(registry), "--runtime-root", str(runtime), "--format", "json"] subject = ["--goal-id", fixture.GOAL_ID, "--agent-id", fixture.AGENT_ID] + + def lease(action, key, version, *options): + code, payload = fixture._run_cli([ + *prefix, "task-lease", action, "--goal-id", fixture.GOAL_ID, + "--todo-id", fixture.TODO_ID, "--owner", fixture.AGENT_ID, + "--idempotency-key", key, "--expected-version", str(version), *options, + ]) + assert code == 0, payload + return payload["lease"] + + original = lease("acquire", "original-attempt", 0, "--ttl-seconds", "300") if uses_lease else None adapter = [sys.executable, "-m", "loopx.dsh_goal_mode", "--workspace", str(workspace), "--dsh-runner", str(runner), "--dsh-home", str(tmp_path / "dsh-home")] validator = [sys.executable, "-c", "from pathlib import Path; import json,sys; json.load(sys.stdin); " @@ -200,7 +222,7 @@ def run_dsh_turn(**options): code, first = fixture._run_cli([*argv, "--turn-instance-id", "bound-stdio-retry"]) assert code == 1 and first["status"] == "failed", first assert marker.read_text() == "1" and fixture._quota_spend_count(runtime) == 0 - if edit_task: + if change == "text": binding = [*subject, "--todo-id", fixture.TODO_ID] code, before = fixture._run_cli([*prefix, "todo", "list", *binding]) assert code == 0 @@ -209,14 +231,39 @@ def run_dsh_turn(**options): "--update-operation-id", "change-captured-work", "--update-expected-provider-revision", before["authority_read"]["provider_revision"]]) assert code == 0 and changed["changed"], changed + if change == "renew_lease": + assert original is not None + renewed = lease("renew", original["idempotency_key"], original["version"], "--ttl-seconds", "600") + assert renewed["lease_epoch"] == original["lease_epoch"] + if change == "replace_lease": + assert original is not None + released = lease("release", original["idempotency_key"], original["version"]) + replacement = lease("acquire", "replacement-attempt", released["version"], "--ttl-seconds", "300") + assert replacement["lease_epoch"] > original["lease_epoch"] recovery = [*argv, "--resume-turn-key", first["resume_turn_key"]] - code, result = fixture._run_cli([*recovery, "--retry-failed-turn"]) - if edit_task: - assert code == 1 and result["error_code"] == "current_turn_work_changed", result + dispatch = nullcontext() + if change == "release_during_dispatch": + dispatch = expect_lease_release_after_dispatch(monkeypatch, operation="turn_external_host_launch", + registry=registry, runtime=runtime, goal_id=fixture.GOAL_ID, todo_id=fixture.TODO_ID, + lease=original, dispatch_marker=marker) + elif change == "expire_during_dispatch": + renewed = lease("renew", original["idempotency_key"], original["version"], "--ttl-seconds", "8") + assert renewed["lease_epoch"] == original["lease_epoch"] + dispatch = expect_lease_expiry_during_dispatch(monkeypatch, operation="turn_external_host_launch", + observed_lease=lambda: renewed) + with dispatch: + code, result = fixture._run_cli([*recovery, "--retry-failed-turn"]) + refused = change in {"text", "replace_lease", "expire_during_dispatch"} + if refused: + expected = "goal_launch_not_allowed" if change == "expire_during_dispatch" else "current_turn_work_changed" + assert code == 1 and result.get("error_code") == expected, ( + code, result.get("status"), result.get("error_code"), result.get("effects"), + ) + assert result["host_failure"]["kind"] == "contract_rejected", result assert not any(result["effects"].values()) else: assert code == 0 and result["status"] == "committed", result _, replay = fixture._run_cli(recovery) assert replay["replayed"] and not any(replay["effects"].values()) - assert marker.read_text() == ("1" if edit_task else "2") - assert fixture._quota_spend_count(runtime) == (0 if edit_task else 1) + assert marker.read_text() == ("1" if refused else "2") + assert fixture._quota_spend_count(runtime) == (0 if refused else 1) diff --git a/tests/control_plane/test_turn_host_decision_capture.py b/tests/control_plane/test_turn_host_decision_capture.py index 2cacde11cd..87c9b0f4d4 100644 --- a/tests/control_plane/test_turn_host_decision_capture.py +++ b/tests/control_plane/test_turn_host_decision_capture.py @@ -594,14 +594,12 @@ def no_new_host(_request): @pytest.mark.parametrize("provider", ["file", "sqlite"]) -@pytest.mark.parametrize(("first_attempt_fails", "restore_live_work"), [ - pytest.param(False, False, id="committed"), - pytest.param(True, True, id="restore-live-work"), - pytest.param(True, False, id="changed-live-work"), +@pytest.mark.parametrize("attempt", [ + "committed", "restored_before_retry", "changed_before_retry", "missing_capability_before_retry", ]) @pytest.mark.parametrize("legacy_pretty", [False, True]) def test_real_host_reads_original_capture_after_source_change( - tmp_path, monkeypatch, provider, first_attempt_fails, restore_live_work, legacy_pretty, + tmp_path, monkeypatch, provider, attempt, legacy_pretty, ): from loopx.cli_commands import turn_decision @@ -647,9 +645,29 @@ def legacy_launch_selection(*_args, **_kwargs): build_todo_runtime_shadow_projection(goal_id=f.GOAL_ID, handoff_mode="soft_claim", todos=summary["agent_todos"]["items"]), state_path=state, provider=provider) + prefix = ["--registry", str(registry), "--runtime-root", str(runtime), "--format", "json"] + + def current_todo(): + code, current = f._run_cli([*prefix, "todo", "list", "--goal-id", f.GOAL_ID, "--todo-id", f.TODO_ID]) + assert code == 0, current + return current + + def update_current_todo(operation, *changes): + # Change the real provider through its native CAS CLI. + current = current_todo() + code, update = f._run_cli([*prefix, "todo", "update", "--goal-id", f.GOAL_ID, + "--todo-id", f.TODO_ID, "--agent-id", f.AGENT_ID, *changes, + "--update-operation-id", operation, "--update-expected-provider-revision", + current["authority_read"]["provider_revision"], "--execute"]) + assert code == 0, update + + capability_case = attempt == "missing_capability_before_retry" + if capability_case: + update_current_todo("require-shell", "--required-capability", "shell") executable = f._write_fake_codex(tmp_path) source = executable.read_text().replace("#!/usr/bin/env python3", "#!" + sys.executable) source = source.replace("prompt = sys.stdin.read()", 'prompt = sys.stdin.read()\npathlib.Path("received.txt").write_text(prompt)') + first_attempt_fails = attempt != "committed" if first_attempt_fails: source = source.replace("marker = pathlib.Path(", 'failed = pathlib.Path("failed-once")\n' @@ -659,10 +677,16 @@ def legacy_launch_selection(*_args, **_kwargs): ' print("Selected model is at capacity. Please try a different model.", file=sys.stderr)\n' ' raise SystemExit(9)\n' 'marker = pathlib.Path(') + if capability_case: + source = source.replace("marker = pathlib.Path(", + 'assert "resume" in args and "session-fixture-0001" in args\n' + 'marker = pathlib.Path(') executable.write_text(source) base = f._base_argv(registry=registry, runtime=runtime, workspace=workspace, codex_bin=executable, model=None, timeout_seconds=20, expected_marker=f._marker_value(1), turn_instance_id="original-observation") + if capability_case: + base.extend(["--available-capability", "shell"]) # Preview must not allocate a capture or invoke the provider. code, preview = f._run_cli(base) assert code == 0, preview @@ -701,22 +725,20 @@ def legacy_launch_selection(*_args, **_kwargs): for field in ("work_lane_contract", "execution_obligation"): assert source_a[field] assert request["task_context"][field] == source_a[field] - # Change the real provider through its native CAS CLI, not the projection. - prefix = ["--registry", str(registry), "--runtime-root", str(runtime), "--format", "json"] - code, current = f._run_cli([*prefix, "todo", "list", "--goal-id", f.GOAL_ID, "--todo-id", f.TODO_ID]) - assert code == 0 - original_text = current["todo"]["text"] - code, update = f._run_cli([*prefix, "todo", "update", "--goal-id", f.GOAL_ID, - "--todo-id", f.TODO_ID, "--agent-id", f.AGENT_ID, "--text", original_text + " SOURCE_B", - "--update-operation-id", "source-b", "--update-expected-provider-revision", - current["authority_read"]["provider_revision"], "--execute"]) - assert code == 0, update + + original_text = current_todo()["todo"]["text"] + + def change_current_task(operation): + updated_text = original_text + " SOURCE_B" + update_current_todo(operation, "--text", updated_text) + assert current_todo()["todo"]["text"] == updated_text + + if attempt in {"committed", "restored_before_retry", "changed_before_retry"}: + change_current_task("source-b") # Execute precisely what the actual subprocess received. detail_bytes = subprocess.run(shlex.split(detail["full_decision"]), check=True, capture_output=True).stdout assert detail_bytes == original and b"SOURCE_B" not in detail_bytes - code, current = f._run_cli([*prefix, "todo", "list", "--goal-id", f.GOAL_ID, "--todo-id", f.TODO_ID]) - assert code == 0 and "SOURCE_B" in current["todo"]["text"] plan = load_loopx_turn_plan_from_journal(runtime, goal_id=f.GOAL_ID, turn_key=request["turn_key"]) journal_path = turn_journal_path(runtime, goal_id=f.GOAL_ID, turn_key=request["turn_key"]) # An existing receipt must not let a malformed complete request bypass the @@ -778,30 +800,52 @@ def unexpected_effect(*_args, **_kwargs): assert (workspace / "received.txt").read_bytes() == received_before assert f._quota_spend_count(runtime) == 0 path.write_bytes(original) - # The snapshot remains readable when live work changes. Launching it - # again also requires the current Todo to match the original work. - if restore_live_work: - code, restored = f._run_cli([*prefix, "todo", "update", "--goal-id", f.GOAL_ID, - "--todo-id", f.TODO_ID, "--agent-id", f.AGENT_ID, "--text", original_text, - "--update-operation-id", "restore-source-a", "--update-expected-provider-revision", - current["authority_read"]["provider_revision"], "--execute"]) - assert code == 0 and restored["changed"], restored + if capability_case: + update_current_todo("require-network", "--required-capability", "shell", + "--required-capability", "network") + for extra, error in ( + ([], "goal_launch_not_allowed"), + (["--available-capability", "network"], "current_turn_work_changed"), + ): + code, refused = f._run_cli([*retry, *extra]) + assert code == 1 and refused.get("error_code") == error, ( + error, {key: refused.get(key) for key in + ("status", "reason", "error", "error_code", "host_failure", "recovery")}, + ) + assert refused["status"] == "failed" and not any(refused["effects"].values()) + assert refused["host_failure"]["kind"] == "contract_rejected" + assert json.loads(journal_path.read_bytes())["host_recovery"]["kind"] == "resume_session" + assert refused["recovery"]["actual"]["host_invoked"] is False + assert (workspace / "received.txt").read_bytes() == received_before + assert not (workspace / f.MARKER_NAME).exists() + assert f._quota_spend_count(runtime) == 0 + update_current_todo("restore-shell", "--required-capability", "shell") + if attempt == "restored_before_retry": + update_current_todo("restore-source-a", "--text", original_text) + assert current_todo()["todo"]["text"] == original_text code, recovered = f._run_cli(retry) - if restore_live_work: + if attempt == "changed_before_retry": + assert code == 1 and recovered["error_code"] == "current_turn_work_changed", recovered + assert recovered["status"] == "failed" and not any(recovered["effects"].values()) + assert recovered["host_failure"]["kind"] == "contract_rejected" + assert recovered["recovery"]["actual"]["host_invoked"] is False + assert (workspace / "received.txt").read_bytes() == received_before + else: assert code == 0 and recovered["status"] == "committed", json.dumps(recovered) retried = json.loads((workspace / "received.txt").read_text().split("Turn request:\n", 1)[1]) assert retried["task_context"] == request["task_context"] assert retried["detail_ref"] == detail - else: - assert code == 1 and recovered["error_code"] == "current_turn_work_changed", recovered - assert not any(recovered["effects"].values()) - assert (workspace / "received.txt").read_bytes() == received_before + change_current_task("source-b-after-retry") assert path.read_bytes() == original code, repeated = f._run_cli([*replay, "--resume-turn-key", request["turn_key"], "--execute"]) - blocked_retry = first_attempt_fails and not restore_live_work - assert code == (1 if blocked_retry else 0), repeated + if attempt == "changed_before_retry": + assert code == 1 and repeated["status"] == "failed", repeated + assert repeated["recovery"]["planned"]["reason"] == "failed_retry_not_requested" + assert repeated["host_failure"]["kind"] == "contract_rejected" + else: + assert code == 0, repeated assert repeated["replayed"] and not any(repeated["effects"].values()) - assert f._quota_spend_count(runtime) == (0 if blocked_retry else 1) + assert f._quota_spend_count(runtime) == (0 if attempt == "changed_before_retry" else 1) assert path.read_bytes() == original # Admission can run between source validation and publication. It cannot # replace or silently recreate an original capture changed in that window. diff --git a/tests/control_plane_ts/first_party_host_runtime.test.ts b/tests/control_plane_ts/first_party_host_runtime.test.ts index fd87ebe30e..fb84a2b1d3 100644 --- a/tests/control_plane_ts/first_party_host_runtime.test.ts +++ b/tests/control_plane_ts/first_party_host_runtime.test.ts @@ -103,11 +103,56 @@ test("legacy native launch consumes the current quota owner without weakening so planned_effective_action: "control_plane_health_repair", current: {should_run: true, status_health_ok: false, effective_action: "control_plane_health_repair"}, }}).kind, "reject"); + + const preparation = {todo_id: "todo-original", handoff_mode: "hard_lease", state: "ready", + delivery_allowed: true, lease_proof: {idempotency_key: "original-lease", expected_version: 2}}; + const boundLaunch = {...admitted, receipt_bound_todo_id: preparation.todo_id, + planned_execution_preparation: preparation}; + const withPreparation = (execution_preparation: unknown) => decideFirstPartyHostRuntime({ + ...legacy, launch_decision: {...boundLaunch, current: {...admitted.current, execution_preparation}}, + }); + assert.notEqual(withPreparation(preparation).kind, "reject"); + assert.notEqual(withPreparation({...preparation, + lease_proof: {...preparation.lease_proof, expected_version: 3}}).kind, "reject"); + for (const current of [null, {...preparation, state: "claim_required", delivery_allowed: false}, + {...preparation, todo_id: "todo-other"}, + {...preparation, handoff_mode: "soft_claim", state: "not_required"}, + {...preparation, lease_proof: {idempotency_key: "replacement-lease", expected_version: 3}}, + {...preparation, lease_proof: {...preparation.lease_proof, expected_version: 1}}]) { + assert.deepEqual(withPreparation(current), {kind: "reject", code: "goal_launch_not_allowed"}); + } assert.deepEqual(decideFirstPartyHostRuntime({...exact, planned_goal_ref: {goal_id: "release"}, launch_decision: admitted}), {kind: "reject", code: "goal_instance_id_missing"}); }); +test("a Todo-bound replan launch requires the same non-delivery obligation", () => { + const witness = {path: "/fixture/registry.json", sha256: "a".repeat(64)}; + const legacy = {profile_id: null, operation: "launch", goal_id: "release", + registry_path: witness.path, planned_registry_witness: witness, registry_witness: witness, + authority: {kind: "present", goal_id: "release", registration_status: "active", activation_state: "active"}}; + const todo = "todo_chain_000000000000"; + const obligation = "replan-1000000000000001"; + const current = { + effective_action: "autonomous_replan_required", should_run: true, + status_health_ok: true, normal_delivery_allowed: false, + selected_todo: {todo_id: todo}, replan_action_packet: {obligation_id: obligation}, + }; + const launch = {receipt_bound_todo_id: todo, + receipt_bound_replan_obligation_id: obligation, + planned_effective_action: current.effective_action, current}; + const decide = (launch_decision: unknown) => decideFirstPartyHostRuntime({...legacy, launch_decision}); + assert.deepEqual(decide(launch), {kind: "legacy"}); + for (const facts of [ + {...launch, receipt_bound_replan_obligation_id: undefined}, + {...launch, current: {...current, replan_action_packet: {obligation_id: "replan-1000000000000002"}}}, + {...launch, current: {...current, selected_todo: {todo_id: "todo_chain_000000000001"}}}, + {...launch, current: {...current, normal_delivery_allowed: true}}, + ]) { + assert.deepEqual(decide(facts), {kind: "reject", code: "goal_launch_not_allowed"}); + } +}); + test("a running legacy Turn requires its original authority and still sees Stop", () => { const entry = "a".repeat(64); const authority = {kind: "present", goal_id: "release", registration_status: "active", diff --git a/tests/test_host_launch_authority.py b/tests/test_host_launch_authority.py index 3ba70ad1f8..15d7cdbc92 100644 --- a/tests/test_host_launch_authority.py +++ b/tests/test_host_launch_authority.py @@ -9,6 +9,7 @@ import os import signal import sys +import textwrap import threading import time from pathlib import Path @@ -30,6 +31,11 @@ from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission from loopx.control_plane.turn_driver import host_process_transport from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from tests.control_plane.host_process_fixture import ( + WAIT_FOR_FIXTURE_LEASE_RELEASE, + expect_lease_expiry_during_dispatch, + expect_lease_release_after_dispatch, +) from tests.control_plane.test_effect_turn_live_quota_decision import ( _material_review_urgency, _turn_start_dispatch, @@ -194,7 +200,8 @@ def _host(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, operation_tools: bool f"pathlib.Path({str(native_start)!r}).write_text('started')", 1, ).replace( 'elif method == "turn/start":', 'elif method == "turn/start":\n' - f" pathlib.Path({str(turn_start)!r}).write_text('started')", 1, + f" pathlib.Path({str(turn_start)!r}).write_text('started')\n" + + textwrap.indent(WAIT_FOR_FIXTURE_LEASE_RELEASE.rstrip(), " "), 1, )) executable.chmod(0o700) else: @@ -202,6 +209,8 @@ def _host(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, operation_tools: bool executable.write_text(executable.read_text().replace( "args = sys.argv[1:]", f"pathlib.Path({str(native_start)!r}).write_text('started')\nargs = sys.argv[1:]", 1, + ).replace( + "turn_key = re.search", WAIT_FOR_FIXTURE_LEASE_RELEASE + "\nturn_key = re.search", 1, )) monkeypatch.setenv("FAKE_CODEX_LOG", str(turn_start)) arguments = ["--codex-bin", str(executable)] @@ -211,13 +220,14 @@ def _host(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, operation_tools: bool return arguments, native_start, turn_start -def _run(registry: Path, runtime: Path, project: Path, host_arguments: list[str]): +def _run(registry: Path, runtime: Path, project: Path, host_arguments: list[str], *, resume_key: str | None = None): + turn = (["--turn-instance-id", "launch-authority-fixture"] if resume_key is None + else ["--resume-turn-key", resume_key, "--retry-failed-turn"]) return _cli(registry, runtime, "turn", "run-once", "--goal-id", GOAL, "--agent-id", AGENT, "--host", "codex-cli", "--project", str(project), "--codex-sandbox", "read-only", "--no-global-sync", "--iteration-context", "fresh", - "--turn-instance-id", "launch-authority-fixture", - "--validation-command-json", json.dumps([sys.executable, "-c", "pass"]), + *turn, "--validation-command-json", json.dumps([sys.executable, "-c", "pass"]), *host_arguments, "--execute", ) @@ -349,6 +359,110 @@ def test_first_normal_turn_with_declared_state_needs_no_global_registry( assert not (runtime / "registry.global.json").exists() +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("operation_tools", [False, True]) +@pytest.mark.parametrize("change", ["renew", "replace", "release_during_dispatch", "expire_during_dispatch"]) +def test_actual_launch_preserves_original_task_lease_generation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation_tools: bool, change: str, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider, handoff_mode="hard_lease") + host_arguments, native_start, turn_start = _host(tmp_path, monkeypatch, operation_tools) + lease_scope = ["--goal-id", GOAL, "--todo-id", "todo_fixture0001", "--owner", AGENT] + + def lease(action, key, version, *options): + code, payload = _cli(registry, runtime, "task-lease", action, *lease_scope, + "--idempotency-key", key, "--expected-version", str(version), *options) + assert code == 0, payload + return payload["lease"] + + original = lease("acquire", "first-generation", 0, "--ttl-seconds", "300") + if change == "release_during_dispatch": + operation = "codex_operation_turn_start" if operation_tools else "codex_cli_host_launch" + with expect_lease_release_after_dispatch(monkeypatch, operation=operation, + registry=registry, runtime=runtime, goal_id=GOAL, todo_id="todo_fixture0001", + lease=original, dispatch_marker=turn_start, + ): + code, result = _run(registry, runtime, project, host_arguments) + assert native_start.exists() and turn_start.exists(), result + assert code == 0 and result["status"] == ("stopped" if operation_tools else "committed"), result + return + owner = codex_operation_host if operation_tools else codex_cli + prepare_input = owner.prepare_codex_cli_input + changes = [] + + def change_lease_before_dispatch(request): + if change in {"renew", "expire_during_dispatch"}: + ttl = "8" if change == "expire_during_dispatch" else "600" + current = lease("renew", original["idempotency_key"], original["version"], + "--ttl-seconds", ttl) + assert current["lease_epoch"] == original["lease_epoch"] + else: + released = lease("release", original["idempotency_key"], original["version"]) + current = lease("acquire", "next-generation", released["version"], "--ttl-seconds", "300") + assert current["lease_epoch"] > original["lease_epoch"] + assert current["version"] > original["version"] + changes.append(current) + return prepare_input(request) + + monkeypatch.setattr(owner, "prepare_codex_cli_input", change_lease_before_dispatch) + if change == "expire_during_dispatch": + operation = "codex_operation_turn_start" if operation_tools else "codex_cli_host_launch" + with expect_lease_expiry_during_dispatch(monkeypatch, operation=operation, observed_lease=lambda: changes[0]): + code, result = _run(registry, runtime, project, host_arguments) + else: + code, result = _run(registry, runtime, project, host_arguments) + assert len(changes) == 1, result + if change in {"replace", "expire_during_dispatch"}: + expected = "current_turn_work_changed" if change == "replace" else "goal_launch_not_allowed" + assert code == 1 and result.get("error_code") == expected, ( + code, result.get("status"), result.get("error_code"), result.get("effects"), + ) + assert result["host_failure"]["kind"] == "contract_rejected" + assert result["effects"]["host_invoked"] is False and not turn_start.exists() + assert native_start.exists() is operation_tools + _assert_no_settlement(runtime, result) + monkeypatch.setattr(owner, "prepare_codex_cli_input", prepare_input) + code, retry = _run(registry, runtime, project, host_arguments, resume_key=result["resume_turn_key"]) + assert code == 1 and retry.get("error_code") == expected, retry + assert retry["resume_turn_key"] == result["resume_turn_key"] + assert retry["effects"]["host_invoked"] is False and not turn_start.exists() + _assert_no_settlement(runtime, retry) + else: + assert code == 0 and result["status"] == ("stopped" if operation_tools else "committed"), result + assert native_start.exists() and turn_start.exists() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_busy_registry_refuses_bound_dispatch_and_retains_original_retry( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider) + arguments, native_start, turn_start = _host(tmp_path, monkeypatch, False) + original = FirstPartyHostGoalAdmission.launch_lifetime + + @contextlib.contextmanager + def hold_registry(admission, *, operation): + registry_writer = (project_registry_transaction(registry, operation="synthetic_registry_writer") + if operation == "codex_cli_host_launch" else contextlib.nullcontext()) + with registry_writer, original(admission, operation=operation): + yield + + with monkeypatch.context() as patch: + patch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", hold_registry) + code, result = _run(registry, runtime, project, arguments) + assert code == 1 and result["error_code"] == "goal_authority_unavailable", result + assert result["host_failure"]["kind"] == "contract_rejected", result + assert not result["effects"]["host_invoked"] and not native_start.exists() and not turn_start.exists() + _assert_no_settlement(runtime, result) + + code, retry = _run(registry, runtime, project, arguments, resume_key=result["resume_turn_key"]) + assert code == 0 and retry["status"] == "committed", retry + assert retry["resume_turn_key"] == result["resume_turn_key"] and turn_start.exists() + assert retry["effects"]["quota_spent"] is True, retry + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize("operation_tools", [False, True]) @pytest.mark.parametrize("late_material", [False, True]) diff --git a/tests/test_native_steward_turn_host.py b/tests/test_native_steward_turn_host.py index 3c11caab85..38da8bb668 100644 --- a/tests/test_native_steward_turn_host.py +++ b/tests/test_native_steward_turn_host.py @@ -542,16 +542,16 @@ def test_managed_work_never_commits_without_current_independent_postcondition(st observation.write_text("not ready") if fault == "stopped_after_model": from loopx.capabilities.native_chat.codex_adapter import CodexAppServerAdapter - original_start = CodexAppServerAdapter.start_turn + original_start = CodexAppServerAdapter.start_native_task_turn - def stopped(adapter, message, sink): - response = original_start(adapter, message, sink) + def stopped(adapter, message, sink, policy): + response = original_start(adapter, message, sink, policy) registry = json.loads(runtime.registry_path.read_text()) registry["goals"][0]["activation"] = dict(STOPPED_ACTIVATION) runtime.registry_path.write_text(json.dumps(registry)) return response - monkeypatch.setattr(CodexAppServerAdapter, "start_turn", stopped) + monkeypatch.setattr(CodexAppServerAdapter, "start_native_task_turn", stopped) calls = capture.read_bytes() accepted, _ = service.submit(session["session_id"], agent_id=actor["agent_id"], operation_id="unqualified", todo_id=todo_id) completed = finish(runtime, accepted) @@ -561,6 +561,10 @@ def stopped(adapter, message, sink): if fault == "missing_validator": assert observation.read_text() == "ready" assert not [json.loads(row) for row in capture.read_bytes()[len(calls):].splitlines() if json.loads(row)["method"] == "turn/start"] + if fault == "stopped_after_model": + dispatched = [json.loads(row) for row in capture.read_bytes()[len(calls):].splitlines()] + assert sum(row["method"] == "turn/start" for row in dispatched) == 1 + assert json.loads(runtime.registry_path.read_text())["goals"][0]["activation"]["state"] == "stopped" after = capture.read_bytes() assert not service.submit(session["session_id"], agent_id=actor["agent_id"], operation_id="unqualified", todo_id=todo_id)[1] assert capture.read_bytes() == after @@ -1104,6 +1108,197 @@ def changed(*args, **kwargs): assert store.load_session(sid)["upstream_thread_id"] == thread +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_managed_host_keeps_canonical_deadline_through_app_server_send(steward, tmp_path, monkeypatch, provider): + import time + + from loopx.capabilities.native_chat import steward_commands + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + from loopx.extensions import process_runtime + + service, session, actor, todo_id, _, capture = managed_fixture(steward, tmp_path, monkeypatch, provider) + store, runtime, *_ = steward + run_command = steward_commands.native_command + launch_lifetime = FirstPartyHostGoalAdmission.launch_lifetime + send_frame = process_runtime.write_process_stdin_before_deadline + admitted_deadlines = [] + crossed = [] + + def acquire_short_lease(controller, argv): + if argv[:2] == ["task-lease", "acquire"] and todo_id in argv: + argv = list(argv) + argv[argv.index("--ttl-seconds") + 1] = "12" + return run_command(controller, argv) + + @contextlib.contextmanager + def observe_admission(admission, *, operation): + with launch_lifetime(admission, operation=operation) as expires_at_ms: + if operation == "native_conversation_turn_launch": + assert type(expires_at_ms) is int + admitted_deadlines.append(expires_at_ms) + yield expires_at_ms + + def send_after_expiry(process, frame, **kwargs): + if json.loads(frame).get("method") == "turn/start": + assert len(admitted_deadlines) == 1 + remaining = admitted_deadlines[0] / 1000 - time.time() + assert remaining > 0, "the active lease must reach the App Server transport" + time.sleep(remaining + .03) + crossed.append(admitted_deadlines[0]) + return send_frame(process, frame, **kwargs) + + monkeypatch.setattr(steward_commands, "native_command", acquire_short_lease) + monkeypatch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", observe_admission) + monkeypatch.setattr(process_runtime, "write_process_stdin_before_deadline", send_after_expiry) + before = capture.read_bytes() + accepted, created = service.submit(session["session_id"], agent_id=actor["agent_id"], + operation_id="expired-at-app-server-transport", todo_id=todo_id) + assert created + completed = runtime.wait_for_turn(session_id=accepted["session_id"], turn_id=accepted["turn_id"], timeout_sec=90) + assert len(crossed) == 1 + calls = [json.loads(line) for line in capture.read_bytes()[len(before):].splitlines()] + assert all(row["method"] != "turn/start" for row in calls), [row["method"] for row in calls] + result = completed["response"]["native_loopx"] + assert result["error_code"] == "goal_launch_not_allowed", result + assert result["host_failure"]["kind"] == "contract_rejected" + assert not any(result["effects"].values()) + current = cli(runtime, "todo", "list", "--goal-id", actor["goal_id"], "--todo-id", todo_id)["todo"] + assert not current["done"] + assert store.load_session(session["session_id"])["upstream_thread_id"] == session["upstream_thread_id"] + index = runtime.coordination_runtime_root / "goals" / actor["goal_id"] / "runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("journal_available", [True, False], ids=["journal-present", "journal-missing"]) +def test_managed_host_reopen_with_new_request_reconciles_unconfirmed_send( + steward, tmp_path, monkeypatch, provider, journal_available, +): + import time + from types import SimpleNamespace + + from loopx.control_plane.turn_driver.journal_store import turn_journal_path + from loopx.extensions import process_runtime + from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + service, session, actor, todo_id, _, wire = managed_fixture(steward, tmp_path, monkeypatch, provider) + _, runtime, *_ = steward + sid = session["session_id"] + before = wire.stat().st_size + send_frame = process_runtime.write_process_stdin_before_deadline + pipe_writer = process_runtime._nonblocking_pipe_writer + + def expire_during_send(process, frame, **kwargs): + if json.loads(frame).get("method") != "turn/start": + return send_frame(process, frame, **kwargs) + expires = kwargs["lease_expires_at_ms"] + late = False + assert expires > time.time() * 1000 + + def wall_time(): + return (expires + 1) / 1000 if late else time.time() + + @contextlib.contextmanager + def send_last_byte_after_expiry(fd): + with pipe_writer(fd) as write: + def send(chunk): + nonlocal late + if chunk[-1] != ord("\n"): + return write(chunk) + if len(chunk) > 1: + return write(chunk[:-1]) + assert write(chunk) == 1 + late = True + deadline = time.monotonic() + 3 + while time.monotonic() < deadline and b'"method": "turn/start"' not in wire.read_bytes()[before:]: + time.sleep(.005) + return 1 + yield send + + with monkeypatch.context() as patch: + patch.setattr(process_runtime, "time", SimpleNamespace( + time=wall_time, monotonic=time.monotonic, sleep=time.sleep)) + patch.setattr(process_runtime, "_nonblocking_pipe_writer", send_last_byte_after_expiry) + return send_frame(process, frame, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(process_runtime, "write_process_stdin_before_deadline", expire_during_send) + first, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="first-late-frame", todo_id=todo_id) + assert created + original = finish(runtime, first)["response"]["native_loopx"] + + assert original["status"] == "in_progress" and original["error_code"] == "host_outcome_unknown" + key = original["resume_turn_key"] + journal = turn_journal_path(runtime.coordination_runtime_root, goal_id=actor["goal_id"], turn_key=key) + retained_journal = journal.read_bytes() + capture = journal.parent / "decisions" / (key.removeprefix("sha256:") + ".json") + retained_capture = capture.read_bytes() + after_first = wire.read_bytes() + assert [json.loads(line)["method"] for line in after_first[before:].splitlines()].count("turn/start") == 1 + events = load_rollout_events(rollout_event_log_path(runtime.coordination_runtime_root, actor["goal_id"])) + linked, = [event for event in events if event.get("event_kind") == "quota_should_run" + and event.get("details", {}).get("turn_key") == key] + source, = [event for event in events if event["event_id"] == linked["causality"]["source_event_id"]] + assert linked["recorded_at"] == source["recorded_at"] and linked["status"] == source["status"] + assert linked["details"] == {**source["details"], "turn_key": key} + + adapter = runtime.adapters.pop(sid, None) + if adapter is not None: + adapter.close_session() + moved = journal.with_suffix(".relocated") + if not journal_available: + journal.rename(moved) + try: + second, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="new-id-after-reopen", todo_id=todo_id) + assert created + effects = finish(runtime, second)["response"]["native_loopx"]["effects"] + replayed, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="first-late-frame", todo_id=todo_id) + assert not created and replayed["turn_id"] == first["turn_id"] + finally: + if not journal_available: + assert not journal.exists() + moved.rename(journal) + assert not any(effects.values()) + assert all(json.loads(line)["method"] != "turn/start" for line in wire.read_bytes()[len(after_first):].splitlines()) + assert journal.read_bytes() == retained_journal and capture.read_bytes() == retained_capture + assert not cli(runtime, "todo", "list", "--goal-id", actor["goal_id"], "--todo-id", todo_id)["todo"]["done"] + index = runtime.coordination_runtime_root / "goals" / actor["goal_id"] / "runs/index.jsonl" + assert not index.is_file() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_native_receipt_without_durable_attempt_allows_fresh_request(steward, tmp_path, monkeypatch, provider): + from loopx.control_plane.turn_driver import executor + from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + service, session, actor, todo_id, _, wire = managed_fixture(steward, tmp_path, monkeypatch, provider) + _, runtime, *_ = steward + sid = session["session_id"] + before = wire.stat().st_size + + def fail_before_first_checkpoint(path, snapshot, **kwargs): + assert snapshot["host_attempt_count"] == 0 + raise OSError("fixture could not persist the first journal checkpoint") + + with monkeypatch.context() as patch: + patch.setattr(executor, "_write_journal", fail_before_first_checkpoint) + first, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="unrecorded-first", todo_id=todo_id) + assert created + finish(runtime, first) + after_first = wire.read_bytes() + assert all(json.loads(line)["method"] != "turn/start" for line in after_first[before:].splitlines()) + events = load_rollout_events(rollout_event_log_path(runtime.coordination_runtime_root, actor["goal_id"])) + admitted = [event for event in events if event.get("event_kind") == "quota_should_run" + and event.get("todo_id") == todo_id] + assert admitted and all("turn_key" not in event["details"] for event in admitted) + + second, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="new-id-after-no-attempt", todo_id=todo_id) + assert created + effects = finish(runtime, second)["response"]["native_loopx"]["effects"] + assert effects["host_invoked"] is True + assert [json.loads(line)["method"] for line in wire.read_bytes()[len(after_first):].splitlines()].count("turn/start") == 1 + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_managed_host_does_not_borrow_another_execution_lease(steward, tmp_path, monkeypatch, provider): service, session, actor, todo_id, _, capture = managed_fixture(steward, tmp_path, monkeypatch, provider)