From aa16c477a3b0b6d3cfb14018b452310b60116fdb Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:35:48 +0800 Subject: [PATCH 01/11] fix(turn): reject changed task declarations before native launch Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/reference/protocols/turn-envelope-v0.md | 18 ++++-- loopx/cli_commands/turn_decision.py | 5 +- loopx/cli_commands/turn_run_once.py | 40 +++++-------- loopx/control_plane/quota/turn_envelope.ts | 19 ++++++- loopx/control_plane/turn_driver/driver.py | 4 ++ .../test_host_current_admission.py | 56 +++++++++++++++++++ tests/test_native_steward_turn_host.py | 35 ++++++++++++ 7 files changed, 145 insertions(+), 32 deletions(-) diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index a325346986..0e5763e959 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -482,10 +482,15 @@ and are published atomically with owner-only file permissions without replacing an existing observation. They can contain private Goal context and must not be included in public logs or artifacts. -Host retry validates and reuses the original journal's capture even if current -source text changes. Reading detail does not invoke quota or spend a slot. -Current source admission, cancellation, workspace/lease and mutation checks -retain their existing owners; the observation grants no fresh authority. +Host retry validates and reuses the original journal's capture. Before a +native Codex launch with a verified selected Todo, the current complete +contract must still match the original task body, required write scopes and +required capabilities. Progress notes do not invalidate that work. A mismatch +returns `current_turn_work_changed` before dispatching the model. Use a fresh +eligible Turn for the changed task; keep original captures and receipts for +readback. Reading detail does not invoke quota or spend a slot. Current source +admission, cancellation, workspace/lease and mutation checks retain their +existing owners; the observation grants no fresh authority. Missing, corrupt, wrong-identity or source-hash-mismatched captures refuse Host reinvocation. Restore the original evidence rather than reconstructing it from a new guard. A capture published before a journal write failure is retained; @@ -507,6 +512,11 @@ blocks resume. Journal inspection can report the invalid work contract without performing recovery or a debit. Retain the historical reader until the last supported journal and its recovery window are reconciled. +原生 Codex 启动前会用当前完整契约复核所选 Todo 的已验证正文、所需写入范围和 +能力。只改进度备注可以继续;改变工作声明会在模型调用前返回 +`current_turn_work_changed`。变更后的工作应通过新的合格 Turn 启动,原捕获与 +回执继续用于原轮回读。 + 恢复预览和 journal 解释共用同一个 typed plan reader。受支持的历史记录保留原版本和 字节。显式声明不支持的 plan、envelope 或动作签名版本及覆盖范围时,恢复会停止。 检查入口仍可报告契约无效,不执行恢复或扣额。保留历史读取器,直到最后一份受支持的 diff --git a/loopx/cli_commands/turn_decision.py b/loopx/cli_commands/turn_decision.py index 1c36db7f4f..edc067f02b 100644 --- a/loopx/cli_commands/turn_decision.py +++ b/loopx/cli_commands/turn_decision.py @@ -358,6 +358,7 @@ def current_turn_host_launch_decision( args: argparse.Namespace, *, registry_path: Path, runtime_root: Path, identity: SettlementIdentity, planned_effective_action: object, + planned_selected_work_basis: str | None, scheduler_execution_context: Mapping[str, Any], operator_inbox_urgency_projector: Callable[..., dict[str, Any]], turn_start_hook_dispatch: Mapping[str, Any] | None = None, @@ -416,7 +417,7 @@ def current_turn_host_launch_decision( plan = build_loopx_turn_plan( current_contract, host=args.host, execution_mode=args.execution_mode, scheduler_owner=args.scheduler_owner, turn_instance_id=identity.turn_instance_id, - goal_ref=goal_ref, + goal_ref=goal_ref, include_selected_work_basis=planned_selected_work_basis is not None, ) route = plan["route"] if route.get("would_invoke_host") is not True: @@ -425,6 +426,8 @@ def current_turn_host_launch_decision( raise FirstPartyHostRuntimeRejected("current_turn_selection_changed") if (identity.replan_obligation_id or identity.request_subject is not None) and plan["transaction"]["settlement_plan"]["identity"] != identity.as_dict(): raise FirstPartyHostRuntimeRejected("current_turn_selection_changed") + if planned_selected_work_basis is not None and plan.get("_selected_work_basis") != planned_selected_work_basis: + raise FirstPartyHostRuntimeRejected("current_turn_work_changed") return launch diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index b93a5df82f..706c77505a 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -767,18 +767,24 @@ def scheduler(_spend_payload: dict[str, object]) -> dict[str, object]: host_runner: Callable[[Mapping[str, Any]], dict[str, Any]] | None = None session_binding_resolver = None - if args.host == "codex-app-server": - def native_launch_decision(current_registry: Path) -> Mapping[str, Any]: - return current_turn_host_launch_decision(args, registry_path=current_registry, - runtime_root=runtime_root, identity=settlement_identity, - planned_effective_action=execution_context["effective_action"], - scheduler_execution_context=payload.get("scheduler_execution_context") or {}, - operator_inbox_urgency_projector=operator_inbox_urgency_projector, - turn_start_hook_dispatch=turn_start_hook_dispatch, goal_ref=goal_ref) + def current_launch_decision(current_registry: Path) -> Mapping[str, Any]: + # Both native transports recheck the original work at launch. + # A global route checks its source even if the mirror is unchanged. + return current_turn_host_launch_decision( + args, registry_path=current_registry, runtime_root=runtime_root, + identity=settlement_identity, + planned_effective_action=execution_context["effective_action"], + planned_selected_work_basis=execution_context.get("selected_work_basis"), + scheduler_execution_context=payload.get("scheduler_execution_context") or {}, + operator_inbox_urgency_projector=operator_inbox_urgency_projector, + turn_start_hook_dispatch=turn_start_hook_dispatch, goal_ref=goal_ref, + ) + + if args.host == "codex-app-server": native_admission = (goal_admission or FirstPartyHostGoalAdmission.for_plan( registry_path=registry_path, goal_id=args.goal_id, planned_goal_ref=goal_ref, - )).with_launch_decision(native_launch_decision) + )).with_launch_decision(current_launch_decision) def run_native_host(request: Mapping[str, Any]) -> dict[str, Any]: return native_host_runner(request, lambda: native_admission.launch_lifetime( @@ -787,22 +793,6 @@ def run_native_host(request: Mapping[str, Any]) -> dict[str, Any]: host_runner = run_native_host session_binding_resolver = native_session_binding_resolver elif args.host == "codex-cli": - - def current_launch_decision(current_registry: Path) -> Mapping[str, Any]: - # Revalidate the already admitted identity through the same - # owner. In a global route the source is checked first, even - # when its mirror still advertises old quota or health. - return current_turn_host_launch_decision( - args, registry_path=current_registry, runtime_root=runtime_root, - identity=settlement_identity, - planned_effective_action=execution_context["effective_action"], - scheduler_execution_context=payload.get("scheduler_execution_context") or {}, - operator_inbox_urgency_projector=operator_inbox_urgency_projector, - # Keep this Turn's already dispatched required reads at launch. - turn_start_hook_dispatch=turn_start_hook_dispatch, - goal_ref=goal_ref, - ) - host_runner = build_codex_cli_host_runner( args, runtime_root=runtime_root, project=project, registry_path=registry_path, goal_ref=goal_ref, diff --git a/loopx/control_plane/quota/turn_envelope.ts b/loopx/control_plane/quota/turn_envelope.ts index b4ed32dd2d..2de57ea07e 100644 --- a/loopx/control_plane/quota/turn_envelope.ts +++ b/loopx/control_plane/quota/turn_envelope.ts @@ -1957,6 +1957,7 @@ export function completePlanningFacts(request: JsonObject): JsonObject { return {...lineage, work, source_decision_hash: decisionSourceHash(decision), action_source_hash: canonicalHash(turnEnvelopeActionSignatureDocument(work)), scoped_settlement_identity: scopedIdentity, scoped_identity_error: identityError, + ...(request.include_selected_work_basis === true ? {selected_work_basis: selectedWorkLaunchBasis(decision)} : {}), ...(request.inspect_plan === true && (lineage.route === "ready_for_host" || lineage.route === "replan_required") ? {inspection_paths: turnPlanInspectionPaths(decision, lineage.route)} : {})}; @@ -2010,6 +2011,19 @@ function completeTaskTextSha256(decision: JsonObject, todoId: unknown): string | return capturedTodoTextSha256(declared, context, decision.recommended_action, todoId); } +/** Launch rechecks the admitted task, not the provider revision or the bounded + * action signature. Progress, diagnostics and receipt changes are not edits to + * the work. Losing the verified body on a fresh read is not a match. */ +function selectedWorkLaunchBasis(decision: JsonObject, verifiedTextSha256?: string | null): string | null { + const todoId = selectedWorkTodoId(decision); + const selected = object(decision.selected_todo); + if (!todoId || todoId !== selected.todo_id) return null; + const textSha256 = verifiedTextSha256 === undefined ? completeTaskTextSha256(decision, todoId) : verifiedTextSha256; + return textSha256 === null ? null : canonicalHash({todo_id: todoId, text_sha256: textSha256, + required_write_scopes: selected.required_write_scopes ?? [], + required_capabilities: selected.required_capabilities ?? []}); +} + export function composeCompleteTurnDecision(value: unknown): JsonObject { const request = requireJsonObject(value, "complete Turn decision request"); const payload = requireJsonObject(request.payload, "complete Turn decision source"); @@ -2655,8 +2669,9 @@ export function evaluateTurnEnvelope(value: unknown): JsonObject { const {decision, projection} = completeHostSource(request); const work = {selected_todo: object(projection.action).selected_todo, task_orchestration_contract: projection.task_orchestration_contract}; - return {selected_todo: selectedWorkTodo(work), - sha256: completeTaskTextSha256(decision, selectedWorkTodoId(work) ?? ""), + const sha256 = completeTaskTextSha256(decision, selectedWorkTodoId(work) ?? ""); + return {selected_todo: selectedWorkTodo(work), sha256, + selected_work_basis: selectedWorkLaunchBasis(decision, sha256), effective_action: decision.effective_action ?? null, writeback: object(projection.writeback), task_lease_proof: object(object(decision.execution_preparation).lease_proof), diff --git a/loopx/control_plane/turn_driver/driver.py b/loopx/control_plane/turn_driver/driver.py index 375a73ccff..105f20fb2d 100644 --- a/loopx/control_plane/turn_driver/driver.py +++ b/loopx/control_plane/turn_driver/driver.py @@ -431,6 +431,7 @@ def build_loopx_turn_plan( goal_ref: Mapping[str, object] | None = None, work_contract: object = _ABSENT_WORK_CONTRACT, inspect_plan: bool = False, + include_selected_work_basis: bool = False, ) -> dict[str, Any]: """Plan from one complete source; retain the recorded v0 input codec. @@ -483,6 +484,7 @@ def build_loopx_turn_plan( projection = _turn_envelope_result("planning_facts", payload=work_contract, turn_instance_id=turn_instance_id, **({"inspect_plan": True} if inspect_plan else {}), + **({"include_selected_work_basis": True} if include_selected_work_basis else {}), **({} if source_only_input else {"envelope": supplied})) envelope = _mapping(projection["work"]) else: @@ -611,6 +613,8 @@ def build_loopx_turn_plan( if current: payload["work_contract"] = work_contract payload["source_decision_hash"] = projection["source_decision_hash"] + if include_selected_work_basis: + payload["_selected_work_basis"] = projection["selected_work_basis"] if "inspection_paths" in projection: payload["_inspection_paths"] = projection["inspection_paths"] if not source_only_input: diff --git a/tests/control_plane/test_host_current_admission.py b/tests/control_plane/test_host_current_admission.py index 3fb995e6d1..b1492b8a1d 100644 --- a/tests/control_plane/test_host_current_admission.py +++ b/tests/control_plane/test_host_current_admission.py @@ -94,3 +94,59 @@ def lose_state_before_start(**kwargs): assert not state.exists() index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("change", ["task_text", "write_scope", "progress_note"]) +@pytest.mark.parametrize("host", ["exec", "operation"]) +def test_native_launch_rechecks_same_todo_after_planning( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, change: str, host: str, +) -> None: + runtime, registry, native, model, arguments = _fixture(tmp_path, monkeypatch, provider) + project = registry.parent.parent + if host == "exec": + executable, model = _fake_codex(tmp_path) + executable.write_text(executable.read_text().replace( + "args = sys.argv[1:]", f"pathlib.Path({str(native)!r}).write_text('native')\nargs = sys.argv[1:]", 1, + )) + monkeypatch.setenv("FAKE_CODEX_LOG", str(model)) + arguments.remove("--codex-operation-tools") + arguments[arguments.index("--codex-bin") + 1] = str(executable) + old = _quota(registry, runtime, project) + assert old["should_run"] is True, old + todo_id = old["selected_todo"]["todo_id"] + mutation = { + "task_text": ["--text", "[P0] Verify the replacement fixture before any work."], + "write_scope": ["--required-write-scope", "docs/evidence/**"], + "progress_note": ["--note", "Independent observation received."], + }[change] + host_module = codex_cli if host == "exec" else codex_operation_host + original_input = host_module.prepare_codex_cli_input + readbacks = [] + + def edit_after_planning(request): + code, result = _cli([ + "--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", + "todo", "update", "--goal-id", "loopx-turn-fixture", "--agent-id", "codex-fixture", + "--todo-id", todo_id, *mutation, + ]) + assert code == 0 and result["ok"], result + current = _quota(registry, runtime, project) + assert current["should_run"] is True and current["effective_action"] == old["effective_action"], current + assert current["selected_todo"]["todo_id"] == todo_id + readbacks.append((result, _bindings(runtime))) + return original_input(request) + + monkeypatch.setattr(host_module, "prepare_codex_cli_input", edit_after_planning) + code, payload = _cli([*arguments, "--execute"]) + assert len(readbacks) == 1 + if change == "progress_note": + assert native.exists() and model.exists(), payload + else: + # Operation mode already started app-server; it must not start a + # model Turn or publish its Session binding on the stale request. + assert native.exists() is (host == "operation") and not model.exists(), payload + assert code == 1 and "current_turn_work_changed" in json.dumps(payload), payload + assert _bindings(runtime) == readbacks[0][1] + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() diff --git a/tests/test_native_steward_turn_host.py b/tests/test_native_steward_turn_host.py index 4d0deb96d6..3c11caab85 100644 --- a/tests/test_native_steward_turn_host.py +++ b/tests/test_native_steward_turn_host.py @@ -940,6 +940,41 @@ def test_native_turn_uses_original_app_server_and_kernel_without_a_new_session(s assert capture.read_bytes() == calls +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_managed_task_edit_after_planning_does_not_start_native_turn(steward, tmp_path, monkeypatch, provider): + from loopx.cli_commands import turn_run_once + + service, session, actor, todo_id, _, capture = managed_fixture(steward, tmp_path, monkeypatch, provider) + store, runtime, *_ = steward + original_request = turn_run_once.prepare_turn_host_request + revised = "[P1] Check the corrected readiness condition before continuing." + edits = [] + + def edit_after_planning(*args, **kwargs): + prepared = original_request(*args, **kwargs) + assert args[0].host == "codex-app-server" + edits.append(cli(runtime, "todo", "update", "--goal-id", actor["goal_id"], + "--todo-id", todo_id, "--agent-id", actor["agent_id"], "--text", revised)) + return prepared + + monkeypatch.setattr(turn_run_once, "prepare_turn_host_request", edit_after_planning) + before = capture.read_bytes() + accepted, created = service.submit(session["session_id"], agent_id=actor["agent_id"], + operation_id="changed-task-before-native-launch", todo_id=todo_id) + assert created + completed = runtime.wait_for_turn(session_id=accepted["session_id"], turn_id=accepted["turn_id"], timeout_sec=90) + result = completed["response"]["native_loopx"] + assert len(edits) == 1 and edits[0]["ok"] + assert not result["ok"] and "current_turn_work_changed" in json.dumps(result), result + calls = [json.loads(line) for line in capture.read_bytes()[len(before):].splitlines()] + assert all(row["method"] != "turn/start" for row in calls) + assert store.load_session(session["session_id"])["upstream_thread_id"] == session["upstream_thread_id"] + current = cli(runtime, "todo", "list", "--goal-id", actor["goal_id"], "--todo-id", todo_id)["todo"] + assert current["text"] == revised and not current["done"] + index = Path(runtime.coordination_runtime_root) / "goals" / actor["goal_id"] / "runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + def test_unselected_or_unavailable_work_does_not_invoke_the_model(steward, tmp_path, monkeypatch): sid, actor, _ = prepare(steward, "sqlite", tmp_path, monkeypatch) store, runtime, _, _, binding, capture, _ = steward From 6028bb9de4d8cc0c1d8ad79544b504be1b52b679 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:44:57 +0800 Subject: [PATCH 02/11] Retain saved Host sessions across pre-dispatch refusals Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/reference/protocols/turn-envelope-v0.md | 5 +- loopx/control_plane/turn_driver/executor.py | 6 +- .../test_turn_host_decision_capture.py | 95 +++++++++++++++---- 3 files changed, 81 insertions(+), 25 deletions(-) diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index 71115e0e72..97d0f59995 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -494,7 +494,8 @@ from the live check through process spawn or the native `turn/start` send. A writer that commits first changes the inputs to the live check. A writer that arrives after the check can continue once dispatch completes, while the Host runs. Registry contention can refuse dispatch with `goal_authority_unavailable`. -The journal retains the original capture and identity for retry or readback. +The journal retains the original capture, identity and saved session recovery +for retry or readback. Use a fresh eligible Turn for changed work. Reading detail consumes no quota. Source admission, cancellation, workspace and mutation authority remain with their existing owners. @@ -526,7 +527,7 @@ supported journal and its recovery window are reconciled. 对本地权威工作,从实时检查到进程启动或发送原生 `turn/start`,派发与 Todo 和 租约写入共用同一锁。写入先提交时,实时检查读取变更后的事实。写入在检查后到达时, 可以在派发完成、Host 仍运行时继续。遇到 registry 写入竞争,派发可以返回 -`goal_authority_unavailable`。journal 保留原捕获与身份,用于重试或读回。 +`goal_authority_unavailable`。journal 保留原捕获、身份和已保存的会话恢复信息,用于重试或读回。 变更后的工作通过新的合格 Turn 执行。读取详情不扣额。来源准入、取消、工作区和 写入权限继续由原模块维护。 diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 8d4df2c666..6fc3406dd7 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -1088,7 +1088,7 @@ def _host_result_stage( recovery_kind = host_observation.get("recovery_kind") if recovery_kind is not None: journal["host_recovery"] = build_host_recovery_record(recovery_kind) - else: + elif rejected_launch is None: journal.pop("host_recovery", None) persist_journal(journal) terminal = execution_payload(plan, journal, execute=True, replayed=False, effects=effects) @@ -1097,6 +1097,7 @@ def _host_result_stage( return None, [], terminal result = dict(host_observation["value"]) completed_phases = list(TRANSACTION_PHASES[:2]) + journal.pop("host_recovery", None) recorded_plan = journal.get("plan") validation = validate_loopx_turn_host_result( @@ -1799,7 +1800,8 @@ def persist_source_journal(source_admission: Mapping[str, Any]) -> None: journal.pop("validation_stage", None) journal.pop("reason", None) journal.pop("receipt", None) - journal.pop("host_recovery", None) + # A launch refusal can leave the existing Host session available. + # Keep its recovery requirement until the next Host observation. journal.pop("host_failure", None) journal["status"] = "in_progress" persist_journal(journal) diff --git a/tests/control_plane/test_turn_host_decision_capture.py b/tests/control_plane/test_turn_host_decision_capture.py index 648f63cdcd..c8a4b74b88 100644 --- a/tests/control_plane/test_turn_host_decision_capture.py +++ b/tests/control_plane/test_turn_host_decision_capture.py @@ -594,10 +594,10 @@ def no_new_host(_request): @pytest.mark.parametrize("provider", ["file", "sqlite"]) -@pytest.mark.parametrize("first_attempt_fails", [False, True]) +@pytest.mark.parametrize("attempt", ["committed", "retry", "changed_before_retry", "missing_capability_before_retry"]) @pytest.mark.parametrize("legacy_pretty", [False, True]) def test_real_host_reads_original_capture_after_source_change( - tmp_path, monkeypatch, provider, first_attempt_fails, legacy_pretty, + tmp_path, monkeypatch, provider, attempt, legacy_pretty, ): from loopx.cli_commands import turn_decision @@ -643,9 +643,29 @@ def legacy_launch_selection(*_args, **_kwargs): build_todo_runtime_shadow_projection(goal_id=f.GOAL_ID, handoff_mode="soft_claim", todos=summary["agent_todos"]["items"]), state_path=state, provider=provider) + prefix = ["--registry", str(registry), "--runtime-root", str(runtime), "--format", "json"] + + def current_todo(): + code, current = f._run_cli([*prefix, "todo", "list", "--goal-id", f.GOAL_ID, "--todo-id", f.TODO_ID]) + assert code == 0, current + return current + + def update_current_todo(operation, *changes): + # Change the real provider through its native CAS CLI. + current = current_todo() + code, update = f._run_cli([*prefix, "todo", "update", "--goal-id", f.GOAL_ID, + "--todo-id", f.TODO_ID, "--agent-id", f.AGENT_ID, *changes, + "--update-operation-id", operation, "--update-expected-provider-revision", + current["authority_read"]["provider_revision"], "--execute"]) + assert code == 0, update + + capability_case = attempt == "missing_capability_before_retry" + if capability_case: + update_current_todo("require-shell", "--required-capability", "shell") executable = f._write_fake_codex(tmp_path) source = executable.read_text().replace("#!/usr/bin/env python3", "#!" + sys.executable) source = source.replace("prompt = sys.stdin.read()", 'prompt = sys.stdin.read()\npathlib.Path("received.txt").write_text(prompt)') + first_attempt_fails = attempt != "committed" if first_attempt_fails: source = source.replace("marker = pathlib.Path(", 'failed = pathlib.Path("failed-once")\n' @@ -655,10 +675,16 @@ def legacy_launch_selection(*_args, **_kwargs): ' print("Selected model is at capacity. Please try a different model.", file=sys.stderr)\n' ' raise SystemExit(9)\n' 'marker = pathlib.Path(') + if capability_case: + source = source.replace("marker = pathlib.Path(", + 'assert "resume" in args and "session-fixture-0001" in args\n' + 'marker = pathlib.Path(') executable.write_text(source) base = f._base_argv(registry=registry, runtime=runtime, workspace=workspace, codex_bin=executable, model=None, timeout_seconds=20, expected_marker=f._marker_value(1), turn_instance_id="original-observation") + if capability_case: + base.extend(["--available-capability", "shell"]) # Preview must not allocate a capture or invoke the provider. code, preview = f._run_cli(base) assert code == 0, preview @@ -697,21 +723,16 @@ def legacy_launch_selection(*_args, **_kwargs): for field in ("work_lane_contract", "execution_obligation"): assert source_a[field] assert request["task_context"][field] == source_a[field] - # Change the real provider through its native CAS CLI, not the projection. - prefix = ["--registry", str(registry), "--runtime-root", str(runtime), "--format", "json"] - code, current = f._run_cli([*prefix, "todo", "list", "--goal-id", f.GOAL_ID, "--todo-id", f.TODO_ID]) - assert code == 0 - code, update = f._run_cli([*prefix, "todo", "update", "--goal-id", f.GOAL_ID, - "--todo-id", f.TODO_ID, "--agent-id", f.AGENT_ID, "--text", current["todo"]["text"] + " SOURCE_B", - "--update-operation-id", "source-b", "--update-expected-provider-revision", - current["authority_read"]["provider_revision"], "--execute"]) - assert code == 0, update + def change_current_task(): + update_current_todo("source-b", "--text", current_todo()["todo"]["text"] + " SOURCE_B") + assert "SOURCE_B" in current_todo()["todo"]["text"] + + if attempt in {"committed", "changed_before_retry"}: + change_current_task() # Execute precisely what the actual subprocess received. detail_bytes = subprocess.run(shlex.split(detail["full_decision"]), check=True, capture_output=True).stdout assert detail_bytes == original and b"SOURCE_B" not in detail_bytes - code, current = f._run_cli([*prefix, "todo", "list", "--goal-id", f.GOAL_ID, "--todo-id", f.TODO_ID]) - assert code == 0 and "SOURCE_B" in current["todo"]["text"] plan = load_loopx_turn_plan_from_journal(runtime, goal_id=f.GOAL_ID, turn_key=request["turn_key"]) journal_path = turn_journal_path(runtime, goal_id=f.GOAL_ID, turn_key=request["turn_key"]) # An existing receipt must not let a malformed complete request bypass the @@ -773,17 +794,49 @@ def unexpected_effect(*_args, **_kwargs): assert (workspace / "received.txt").read_bytes() == received_before assert f._quota_spend_count(runtime) == 0 path.write_bytes(original) - code, recovered = f._run_cli([*replay, "--resume-turn-key", request["turn_key"], - "--retry-failed-turn", "--execute"]) - assert code == 0 and recovered["status"] == "committed", json.dumps(recovered) - retried = json.loads((workspace / "received.txt").read_text().split("Turn request:\n", 1)[1]) - assert retried["task_context"] == request["task_context"] - assert retried["detail_ref"] == detail + if capability_case: + update_current_todo("require-network", "--required-capability", "shell", + "--required-capability", "network") + for extra, error in ( + ([], "goal_launch_not_allowed"), + (["--available-capability", "network"], "current_turn_work_changed"), + ): + code, refused = f._run_cli([*retry, *extra]) + assert code == 1 and refused.get("error_code") == error, ( + error, {key: refused.get(key) for key in + ("status", "reason", "error", "error_code", "host_failure", "recovery")}, + ) + assert refused["status"] == "failed" and not any(refused["effects"].values()) + assert refused["host_failure"]["kind"] == "contract_rejected" + assert json.loads(journal_path.read_bytes())["host_recovery"]["kind"] == "resume_session" + assert refused["recovery"]["actual"]["host_invoked"] is False + assert (workspace / "received.txt").read_bytes() == received_before + assert not (workspace / f.MARKER_NAME).exists() + assert f._quota_spend_count(runtime) == 0 + update_current_todo("restore-shell", "--required-capability", "shell") + code, recovered = f._run_cli(retry) + if attempt == "changed_before_retry": + assert code == 1 and recovered["error_code"] == "current_turn_work_changed", recovered + assert recovered["status"] == "failed" and not any(recovered["effects"].values()) + assert recovered["host_failure"]["kind"] == "contract_rejected" + assert recovered["recovery"]["actual"]["host_invoked"] is False + assert (workspace / "received.txt").read_bytes() == received_before + else: + assert code == 0 and recovered["status"] == "committed", json.dumps(recovered) + retried = json.loads((workspace / "received.txt").read_text().split("Turn request:\n", 1)[1]) + assert retried["task_context"] == request["task_context"] + assert retried["detail_ref"] == detail + change_current_task() assert path.read_bytes() == original code, repeated = f._run_cli([*replay, "--resume-turn-key", request["turn_key"], "--execute"]) - assert code == 0, repeated + if attempt == "changed_before_retry": + assert code == 1 and repeated["status"] == "failed", repeated + assert repeated["recovery"]["planned"]["reason"] == "failed_retry_not_requested" + assert repeated["host_failure"]["kind"] == "contract_rejected" + else: + assert code == 0, repeated assert not any(repeated["effects"].values()) - assert f._quota_spend_count(runtime) == 1 + assert f._quota_spend_count(runtime) == (0 if attempt == "changed_before_retry" else 1) assert path.read_bytes() == original # Admission can run between source validation and publication. It cannot # replace or silently recreate an original capture changed in that window. From 497a97f7f97891f0192096fbd2c2d89f0c9be46b Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 03:18:50 +0800 Subject: [PATCH 03/11] Fence bound Host dispatch at the active lease deadline Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/reference/protocols/turn-envelope-v0.md | 12 ++- loopx/chat_agent.py | 40 ++++---- loopx/cli_commands/turn_decision.py | 1 + .../goals/first_party_host_admission.py | 40 ++++---- .../quota/execution_preparation.py | 7 +- loopx/control_plane/quota/live_decision.py | 2 + loopx/control_plane/quota/should_run.py | 2 + .../control_plane/quota/should_run_packet.py | 2 + loopx/control_plane/turn_driver/codex_cli.py | 2 +- loopx/control_plane/turn_driver/executor.py | 6 +- .../control_plane/turn_driver/host_process.ts | 6 +- .../turn_driver/host_process_bridge.ts | 22 ++++- .../turn_driver/host_process_transport.py | 10 +- .../turn_driver/leased_host_process.ts | 5 +- .../work_items/execution_preparation.ts | 13 ++- loopx/extensions/process_runtime.py | 91 ++++++++++++++++++- loopx/quota.py | 2 + tests/control_plane/host_process_fixture.py | 33 ++++++- .../test_codex_operation_launch.py | 91 ++++++++++++++++++- .../test_host_current_admission.py | 18 +++- tests/test_host_launch_authority.py | 22 +++-- 21 files changed, 352 insertions(+), 75 deletions(-) diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index 97d0f59995..0db045735a 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -493,9 +493,19 @@ For local canonical work, the Todo and lease writers share the dispatch lock from the live check through process spawn or the native `turn/start` send. A writer that commits first changes the inputs to the live check. A writer that arrives after the check can continue once dispatch completes, while the Host -runs. Registry contention can refuse dispatch with `goal_authority_unavailable`. +runs. The same live hard-lease read supplies the dispatch deadline. The process +supervisor checks it at spawn, and the native sender uses nonblocking writes so +a backed-up pipe cannot publish a complete `turn/start` after expiry. A proven +pre-dispatch expiry returns `goal_launch_not_allowed` with no Host invocation, +writeback or debit. An unconfirmed dispatch still follows the existing recovery +path. Registry contention can refuse dispatch with `goal_authority_unavailable`. The journal retains the original capture, identity and saved session recovery for retry or readback. + +硬租约的派发截止时间来自同一次权威读。进程启动前检查到期时间,原生 +`turn/start` 使用非阻塞写入,防止管道积压导致到期后仍发布完整帧。确认 +到期前尚未派发时,返回 `goal_launch_not_allowed`,不启动模型工作、不写回、 +不扣额。派发状态不明时沿用原恢复路径和原回执身份。 Use a fresh eligible Turn for changed work. Reading detail consumes no quota. Source admission, cancellation, workspace and mutation authority remain with their existing owners. diff --git a/loopx/chat_agent.py b/loopx/chat_agent.py index 033b57975a..70b100ccd4 100644 --- a/loopx/chat_agent.py +++ b/loopx/chat_agent.py @@ -717,7 +717,7 @@ class CodexChatAgentSession: read_only_review: bool = False answer_review: bool = False metadata_review: bool = False - _launch_guard: Callable[[], AbstractContextManager[None]] | None = field( + _launch_guard: Callable[[], AbstractContextManager[int | None]] | None = field( default=None, repr=False ) _native_task_policy_used: bool = field(default=False, repr=False) @@ -751,7 +751,7 @@ def start( reasoning_effort: str | None = None, dynamic_tools: list[dict[str, Any]] | None = None, host_config: dict[str, Any] | None = None, - launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, account_runtime_root: Path | None = None, account_consumer: str = "worker", _compatibility_catalog_path: Path | None = None, @@ -853,7 +853,11 @@ def start( try: # The admitting caller owns this short launch fence. Native startup # RPCs and model execution must not hold the Goal writer lock. - with launch_guard() if launch_guard is not None else nullcontext(): + with launch_guard() if launch_guard is not None else nullcontext() as expires_at_ms: + if expires_at_ms is not None and time.time() * 1000 >= expires_at_ms: + from .control_plane.goals.first_party_host_admission import FirstPartyHostLaunchRejected + + raise FirstPartyHostLaunchRejected("goal_launch_not_allowed") try: process = subprocess.Popen( command, @@ -1174,7 +1178,7 @@ def _runtime_error(self, summary: str) -> CodexChatAgentError: def _write( self, payload: dict[str, Any], *, - write_guard: Callable[[], AbstractContextManager[None]] | None = None, + write_guard: Callable[[], AbstractContextManager[int | None]] | None = None, deadline: float | None = None, ) -> None: if self.process.stdin is None: @@ -1193,10 +1197,14 @@ def _write( ) from exc def _guarded_write( - self, frame: str, write_guard: Callable[[], AbstractContextManager[None]], *, + self, frame: str, write_guard: Callable[[], AbstractContextManager[int | None]], *, deadline: float | None, ) -> None: - from .extensions.process_runtime import force_terminate_process, write_process_stdin_before_deadline + from .extensions.process_runtime import ( + ProcessStdinLeaseExpired, + force_terminate_process, + write_process_stdin_before_deadline, + ) # Serialization, delivery and response share the original RPC/Turn # deadline. No shorter send budget may reject an otherwise valid frame. @@ -1204,24 +1212,20 @@ def _guarded_write( if not self._write_lock.acquire(timeout=max(0, dispatch_deadline - time.monotonic())): raise self._timeout_error("response_timeout", "Codex input dispatch timed out.") try: - with write_guard(): + with write_guard() as expires_at_ms: process = self.process try: - if time.monotonic() >= dispatch_deadline: - # Canonical admission itself can consume the remaining - # budget. No writer started, but this Session still - # retires its peer before releasing the launch fence. - try: - force_terminate_process(process, process_tree_owned=self.process_tree_owned) - except Exception as error: - raise RuntimeError("native input receiver cleanup unconfirmed") from error - raise TimeoutError("native input dispatch deadline expired") write_process_stdin_before_deadline( process, frame, deadline=dispatch_deadline, cancel_and_reap=lambda: force_terminate_process( process, process_tree_owned=self.process_tree_owned, ), + lease_expires_at_ms=expires_at_ms, retire_on_expired_deadline=True, ) + except ProcessStdinLeaseExpired as exc: + from .control_plane.goals.first_party_host_admission import FirstPartyHostLaunchRejected + + raise FirstPartyHostLaunchRejected("goal_launch_not_allowed") from exc except TimeoutError as exc: raise self._timeout_error( "response_timeout", "Codex input dispatch timed out." @@ -1389,7 +1393,7 @@ def _request( *, request_id: int | None = None, deadline: float | None = None, - write_guard: Callable[[], AbstractContextManager[None]] | None = None, + write_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> dict[str, Any]: if request_id is None: with self._request_id_lock: @@ -1734,7 +1738,7 @@ def send( output_schema: dict[str, Any] | None = None, project_collaboration: bool = False, on_native_item: Callable[[dict[str, Any]], None] | None = None, - turn_start_guard: Callable[[], AbstractContextManager[None]] | None = None, + turn_start_guard: Callable[[], AbstractContextManager[int | None]] | None = None, check_running: Callable[[], None] | None = None, _native_task: bool = False, sandbox_policy: dict[str, object] | None = None, diff --git a/loopx/cli_commands/turn_decision.py b/loopx/cli_commands/turn_decision.py index edc067f02b..f150944295 100644 --- a/loopx/cli_commands/turn_decision.py +++ b/loopx/cli_commands/turn_decision.py @@ -390,6 +390,7 @@ def current_turn_host_launch_decision( turn_instance_id=identity.turn_instance_id, turn_start_hook_dispatch=turn_start_hook_dispatch, workspace_path=Path(args.project).expanduser().resolve(), goal_ref=goal_ref, + include_launch_lease_deadline=True, interaction_projection_hooks=(periodic_report_pending_intent_interaction_hook( registry_path=registry_path, runtime_root=runtime_root, goal_id=identity.goal_id, agent_id=identity.agent_id, diff --git a/loopx/control_plane/goals/first_party_host_admission.py b/loopx/control_plane/goals/first_party_host_admission.py index e9a37242a6..f15f9fe55f 100644 --- a/loopx/control_plane/goals/first_party_host_admission.py +++ b/loopx/control_plane/goals/first_party_host_admission.py @@ -421,7 +421,7 @@ def for_plan( def launch_scope_for_plan( cls, *, admission: FirstPartyHostGoalAdmission | None, registry_path: Path | None, goal_id: str, planned_goal_ref: object | None, - ) -> Callable[[str], AbstractContextManager[None]]: + ) -> Callable[[str], AbstractContextManager[int | None]]: """Capture once, then reuse the same admission at each native boundary. A provided Turn admission keeps its quota/health reader. Legacy callers @@ -432,7 +432,7 @@ def launch_scope_for_plan( admission = cls.for_plan(registry_path=registry_path, goal_id=goal_id, planned_goal_ref=planned_goal_ref) - def scope(operation: str) -> AbstractContextManager[None]: + def scope(operation: str) -> AbstractContextManager[int | None]: return admission.launch_lifetime(operation=operation) if admission is not None else nullcontext() return scope @@ -553,14 +553,15 @@ def current_lifetime(self, *, operation: str) -> Iterator[None]: yield @contextmanager - def launch_lifetime(self, *, operation: str) -> Iterator[None]: + def launch_lifetime(self, *, operation: str) -> Iterator[int | None]: """Recheck the original launch authority under its existing writer lock. Legacy registries have no exact Goal instance identity. Their original byte witness can reject stale admission without promoting an alias to one; a changed registry requires a fresh plan. A caller with the original Turn binding supplies its full current admission check. This lifecycle - witness alone does not certify quota/health. Hold only through dispatch. + witness alone does not certify quota/health. A bound hard lease also + supplies its epoch-ms expiry for the transport to check at dispatch. """ if self.source_profile: with self.current_lifetime(operation=operation): @@ -568,8 +569,8 @@ def launch_lifetime(self, *, operation: str) -> Iterator[None]: return work_root = self.launch_work_runtime_root if work_root is None: - with self._legacy_lifetime(operation=operation, launch=True): - yield + with self._legacy_lifetime(operation=operation, launch=True) as expires_at_ms: + yield expires_at_ms return with ExitStack() as scope: @@ -580,14 +581,14 @@ def launch_lifetime(self, *, operation: str) -> Iterator[None]: shadow_maintenance_lock_target(work_root, self.goal_id), operation=operation, )) - scope.enter_context(self._legacy_lifetime( + expires_at_ms = scope.enter_context(self._legacy_lifetime( operation=operation, launch=True, registry_lock_timeout=0.0, )) except FirstPartyHostRuntimeRejected as exc: raise FirstPartyHostLaunchRejected(exc.code) from exc except OSError as exc: raise FirstPartyHostLaunchRejected("goal_authority_unavailable") from exc - yield + yield expires_at_ms def require_running(self, *, operation: str) -> None: """Recheck the admitted Goal for an already-dispatched native Turn.""" @@ -604,12 +605,13 @@ def require_running(self, *, operation: str) -> None: @contextmanager def _legacy_lifetime( self, *, operation: str, launch: bool, registry_lock_timeout: float | None = None, - ) -> Iterator[None]: + ) -> Iterator[int | None]: with ExitStack() as scope: + expires_at_ms = None if self.legacy_source_admission is not None: # Activation writes source then global projection under these # same locks. A failed mirror sync must not hide a source stop. - scope.enter_context(self.legacy_source_admission._legacy_lifetime( + expires_at_ms = scope.enter_context(self.legacy_source_admission._legacy_lifetime( operation=operation, launch=launch, registry_lock_timeout=registry_lock_timeout, )) try: @@ -623,15 +625,19 @@ def _legacy_lifetime( raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") from exc authority = _legacy_goal_authority(transaction.payload_copy(), self.goal_id, self.agent_id) if launch: - self._decision( - "launch", legacy_authority=authority, - registry_witness=transaction.source_read_witness(), - launch_decision=(self.launch_decision_reader(self.registry_path) - if self.launch_decision_reader is not None else None), - ) + live = self.launch_decision_reader(self.registry_path) if self.launch_decision_reader is not None else None + self._decision("launch", legacy_authority=authority, + registry_witness=transaction.source_read_witness(), launch_decision=live) + if live is not None: + preparation = live["current"].get("execution_preparation") or {} + if preparation.get("state") == "ready" and preparation.get("handoff_mode") == "hard_lease": + deadline = preparation.get("lease_expires_at_ms") + if type(deadline) is not int: + raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") + expires_at_ms = deadline if expires_at_ms is None else min(expires_at_ms, deadline) else: self._decision("require_current", legacy_authority=authority) - yield + yield expires_at_ms @contextmanager def source_journal_admission( diff --git a/loopx/control_plane/quota/execution_preparation.py b/loopx/control_plane/quota/execution_preparation.py index 9382711dc6..8972e553e9 100644 --- a/loopx/control_plane/quota/execution_preparation.py +++ b/loopx/control_plane/quota/execution_preparation.py @@ -35,6 +35,7 @@ def project_selected_todo_execution_preparation( goal_id: str, selected_todo: Mapping[str, Any], agent_id: str | None, + include_launch_lease_deadline: bool = False, ) -> dict[str, Any] | None: """Read the selected Todo's canonical lease and project one preparation step. @@ -79,9 +80,11 @@ def project_selected_todo_execution_preparation( if not isinstance(lease, dict) or lease.get("todo_id") != todo_id: raise ValueError("execution preparation returned a lease for a different Todo") status = str(lease.get("status") or "unknown").strip().lower() + active = lease_is_active(lease) leases.append({ "status": status if status in {"active", "released"} else "unknown", - "active": lease_is_active(dict(lease)), + "active": active, + **({"expires_at": lease.get("expires_at")} if include_launch_lease_deadline and active else {}), **({"owner": lease["owner"]} if normalize_todo_claimed_by(lease.get("owner")) else {}), "lease_proof": {"idempotency_key": lease.get("idempotency_key"), "expected_version": lease.get("version")}, @@ -124,6 +127,7 @@ def apply_selected_todo_execution_preparation( obligation: dict[str, Any] | None = None, recommendation: dict[str, Any] | None = None, receipt_bound: bool = False, + include_launch_lease_deadline: bool = False, ) -> None: selected = payload.get("selected_todo") if ( @@ -146,6 +150,7 @@ def apply_selected_todo_execution_preparation( goal_id=goal_id, selected_todo=selected, agent_id=agent_id, + include_launch_lease_deadline=include_launch_lease_deadline, ) # Existing receipt-bound recovery predates this projection and may be # reconstructed without canonical lease authority in synthetic/legacy diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index cfe569dcfe..8a00b5f84f 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -685,6 +685,7 @@ def build_live_quota_should_run_decision( workspace_path: Path | None = None, goal_ref: Mapping[str, object] | None = None, inline_goal_source: bool = False, + include_launch_lease_deadline: bool = False, ) -> dict[str, Any]: """Build one live CLI decision while keeping host observation injectable.""" resolved_context = resolve_scheduler_execution_context(scheduler_execution_context) @@ -853,6 +854,7 @@ def build_live_quota_should_run_decision( workspace_path=workspace_path, goal_ref=goal_ref, authoritative_fallback_todo_items=authoritative_fallback_todo_items, + include_launch_lease_deadline=include_launch_lease_deadline, ) _project_receipt_replan_settlement(payload, settlement_readback, available_capabilities=available_capabilities, scheduler_execution_context=resolved_context, diff --git a/loopx/control_plane/quota/should_run.py b/loopx/control_plane/quota/should_run.py index 5635697ada..0eb65a1b86 100644 --- a/loopx/control_plane/quota/should_run.py +++ b/loopx/control_plane/quota/should_run.py @@ -576,6 +576,7 @@ def build_quota_should_run( goal_ref: Mapping[str, object] | None = None, authoritative_fallback_todo_items: FallbackTodoSource = None, replan_explore_reader: Callable[[str, str | None], Mapping[str, Any] | None] | None = None, + include_launch_lease_deadline: bool = False, ) -> dict[str, Any]: safe_goal_id = str(goal_id or "").strip() direct_sources: dict[str, dict[str, Any]] = {} @@ -684,6 +685,7 @@ def build_quota_should_run( include_agent_todo_detail=include_agent_todo_detail, runtime_root=runtime_root, goal_ref=goal_ref, + include_launch_lease_deadline=include_launch_lease_deadline, scheduler_state_read_observer=_app_scheduler_state_read_observer( direct_sources ), diff --git a/loopx/control_plane/quota/should_run_packet.py b/loopx/control_plane/quota/should_run_packet.py index 188a422a11..a4be0e0f3b 100644 --- a/loopx/control_plane/quota/should_run_packet.py +++ b/loopx/control_plane/quota/should_run_packet.py @@ -1688,6 +1688,7 @@ def _build_quota_should_run_payload( include_agent_todo_detail: bool = False, runtime_root: str | Path | None = None, goal_ref: Mapping[str, object] | None = None, + include_launch_lease_deadline: bool = False, scheduler_state_read_observer: ( Callable[[Mapping[str, Any]], None] | None ) = None, @@ -1725,6 +1726,7 @@ def _build_quota_should_run_payload( else None ), receipt_bound=prepared.receipt_bound_todo_id is not None, + include_launch_lease_deadline=include_launch_lease_deadline, ) _attach_automation_cadence_readback( payload, diff --git a/loopx/control_plane/turn_driver/codex_cli.py b/loopx/control_plane/turn_driver/codex_cli.py index dd4d453b5e..7cde8a3d4e 100644 --- a/loopx/control_plane/turn_driver/codex_cli.py +++ b/loopx/control_plane/turn_driver/codex_cli.py @@ -823,7 +823,7 @@ def _run_codex_exec( timeout_seconds: float | None, on_stdout: Callable[[str], None], on_stderr: Callable[[str], None], - launch_guard: Callable[[], AbstractContextManager[None]], + launch_guard: Callable[[], AbstractContextManager[int | None]], check_running: Callable[[], None] | None, ) -> dict[str, Any]: try: diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 6fc3406dd7..101af459c9 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -878,7 +878,7 @@ def _run_host( argv: Sequence[str], project: Path, timeout_seconds: float | None, - launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> dict[str, Any]: # Dispatch has checked the source and its internal effect adapter before # journal/cadence admission. External stdin gets the same single complete @@ -1016,7 +1016,7 @@ def _host_result_stage( confirm_start: Callable[[], None] | None = None, usage_runtime_root: Path | None = None, usage_goal_id: str = "", - host_launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + host_launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> tuple[dict[str, Any] | None, list[str], dict[str, Any] | None]: completed_phases = list(journal.get("completed_phases") or []) from_journal = "typed_result" in completed_phases @@ -1586,7 +1586,7 @@ def run_loopx_turn_once( admit_start: Callable[[Mapping[str, Any]], dict[str, Any]] | None = None, confirm_start: Callable[[], None] | None = None, goal_admission: FirstPartyHostGoalAdmission | None = None, - host_launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + host_launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> dict[str, Any]: argv, host_projection = _prepare_host_invocation(plan, host_argv=host_argv, host_runner=host_runner) transaction_plan = ( diff --git a/loopx/control_plane/turn_driver/host_process.ts b/loopx/control_plane/turn_driver/host_process.ts index b1dc031cbc..6f2c510956 100644 --- a/loopx/control_plane/turn_driver/host_process.ts +++ b/loopx/control_plane/turn_driver/host_process.ts @@ -55,8 +55,9 @@ function signalGroup(child: ChildProcessWithoutNullStreams, signal: NodeJS.Signa catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; throw error; } } -/** Optional caller hooks: record the spawned group, or keep stdin open for framed input. */ +/** Caller hooks for admission, spawn observation and framed input. */ export type HostProcessHooks = { + beforeSpawn?: () => void; spawned?: (item: HostProcessSpawned) => Promise; openInput?: (write: (text: string) => Promise) => void; }; @@ -64,10 +65,11 @@ export type HostProcessHooks = { export async function runHostProcess(request: HostProcessRequest, output: (item: HostProcessOutput) => Promise, signal?: AbortSignal, terminationGraceMs = HOST_PROCESS_TERMINATE_GRACE_MS, - {spawned, openInput}: HostProcessHooks = {}): Promise { + {beforeSpawn, spawned, openInput}: HostProcessHooks = {}): Promise { const base: HostProcessResult = {kind: "result", outcome: "spawn_failed", returncode: null, signal: null, output_complete: true, cleanup_scope: process.platform === "win32" ? "process_tree_best_effort" : "process_group", group_signal_sent: false}; + beforeSpawn?.(); if (signal?.aborted) return {...base, outcome: "cancelled"}; const child = spawn(request.argv[0], request.argv.slice(1), {cwd: request.cwd, stdio: ["pipe", "pipe", "pipe"], detached: process.platform !== "win32", windowsHide: true}); diff --git a/loopx/control_plane/turn_driver/host_process_bridge.ts b/loopx/control_plane/turn_driver/host_process_bridge.ts index ef1f97b78e..1bf6e65d36 100644 --- a/loopx/control_plane/turn_driver/host_process_bridge.ts +++ b/loopx/control_plane/turn_driver/host_process_bridge.ts @@ -3,7 +3,8 @@ import {once} from "node:events"; import {decodeHostProcessRequest, runHostProcess} from "./host_process.ts"; const owner = new AbortController(); // Private transport observations; none grants Goal or execution authority. -type LaunchObservation = {kind: "ready"} | {kind: "not_launched"}; +type LaunchObservation = {kind: "ready"} | {kind: "not_launched"; reason?: "goal_launch_not_allowed"}; +class HostLaunchLeaseExpired extends Error {} const launchDeadline = process.argv[2] === "--guarded-launch" ? Number(process.argv[3]) : null; if (launchDeadline !== null && (!Number.isFinite(launchDeadline) || launchDeadline <= 0)) throw new TypeError("invalid guarded Host bridge deadline"); @@ -41,18 +42,31 @@ process.stdin.on("data", (chunk: Buffer) => { } if (raw.timeout_ms !== null) raw.timeout_ms = Math.min(raw.timeout_ms, remaining); } + const expiresAt = raw.launch_lease_expires_at_ms; + delete raw.launch_lease_expires_at_ms; + if (expiresAt !== undefined && (launchDeadline === null || !Number.isSafeInteger(expiresAt))) + throw new TypeError("invalid guarded Host lease deadline"); + const beforeSpawn = expiresAt === undefined ? undefined : () => { + if (Date.now() >= expiresAt) throw new HostLaunchLeaseExpired(); + }; const lease = raw.delegated_lease; delete raw.delegated_lease; const request = decodeHostProcessRequest(raw); // The owned process group is reported before either path can run unaccounted. - if (lease === undefined) await emit(await runHostProcess(request, emit, owner.signal, undefined, {spawned: emit})); + if (lease === undefined) { + await emit(await runHostProcess(request, emit, owner.signal, undefined, {beforeSpawn, spawned: emit})); + } else { // Ordinary Hosts do not load canonical lease/provider modules. const {decodeDelegatedHostLease, runLeasedHostProcess} = await import("./leased_host_process.ts"); - await emit(await runLeasedHostProcess(request, decodeDelegatedHostLease(lease), emit, owner.signal, emit)); + await emit(await runLeasedHostProcess(request, decodeDelegatedHostLease(lease), emit, owner.signal, emit, beforeSpawn)); } } - catch { process.exitCode = 1; } + catch (error) { + if (error instanceof HostLaunchLeaseExpired) { + await emit({kind: "not_launched", reason: "goal_launch_not_allowed"} satisfies LaunchObservation); + } else process.exitCode = 1; + } finally { process.stdin.destroy(); } })(); }); diff --git a/loopx/control_plane/turn_driver/host_process_transport.py b/loopx/control_plane/turn_driver/host_process_transport.py index 07a1da28e0..330c0f6bcb 100644 --- a/loopx/control_plane/turn_driver/host_process_transport.py +++ b/loopx/control_plane/turn_driver/host_process_transport.py @@ -306,7 +306,7 @@ def run_host_process( on_stderr: Callable[[str], None] | None = None, delegated_lease: dict[str, Any] | None = None, environment: dict[str, str] | None = None, - launch_guard: Callable[[], AbstractContextManager[None]] | None = None, + launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, check_running: Callable[[], None] | None = None, interrupt_before_terminate: bool = False, ) -> dict[str, Any]: @@ -370,7 +370,9 @@ def run_host_process( if windows_job is not None: windows_job.assign(proc) if launch_guard is not None: - launch_scope.enter_context(launch_guard()) + expires_at_ms = launch_scope.enter_context(launch_guard()) + if expires_at_ms is not None: + request["launch_lease_expires_at_ms"] = expires_at_ms record = _begin_host_process_record(record_path, record_value=record_value, supervises=supervises, supervision=supervision, bridge_pid=proc.pid) if record is not None and windows_job is not None: @@ -417,6 +419,10 @@ def run_host_process( launch_scope.close() elif kind == "not_launched" and guarded is not None: guarded.not_launched = True + if event.get("reason") == "goal_launch_not_allowed": + from ..goals.first_party_host_admission import FirstPartyHostLaunchRejected + + raise FirstPartyHostLaunchRejected("goal_launch_not_allowed") raise TimeoutError("managed Host bridge launch deadline expired") else: raise RuntimeError("Invalid managed Host process observation") diff --git a/loopx/control_plane/turn_driver/leased_host_process.ts b/loopx/control_plane/turn_driver/leased_host_process.ts index e0371a9c0b..c24b877397 100644 --- a/loopx/control_plane/turn_driver/leased_host_process.ts +++ b/loopx/control_plane/turn_driver/leased_host_process.ts @@ -49,7 +49,8 @@ export function decodeDelegatedHostLease(raw: unknown): DelegatedHostLease { export async function runLeasedHostProcess(request: HostProcessRequest, context: DelegatedHostLease, output: (item: HostProcessOutput) => Promise, owner: AbortSignal, - spawned?: (item: HostProcessSpawned) => Promise): Promise { + spawned?: (item: HostProcessSpawned) => Promise, + beforeSpawn?: () => void): Promise { const controller = new AbortController(); const abort = () => controller.abort(); owner.addEventListener("abort", abort, {once: true}); @@ -142,7 +143,7 @@ export async function runLeasedHostProcess(request: HostProcessRequest, context: // The CLI's TERM adapter unwinds the nested Host transport (bounded at // five seconds). Allow that acknowledgement before a forced group kill. // This is private supervisor behavior, not a model-controlled timeout. - const result = await runHostProcess(request, output, controller.signal, 6000, {spawned}); + const result = await runHostProcess(request, output, controller.signal, 6000, {beforeSpawn, spawned}); // Keep the proved expiry armed while a renewal reply is in flight. A // returned model result cannot make a hung lease transport authoritative. finished = true; diff --git a/loopx/control_plane/work_items/execution_preparation.ts b/loopx/control_plane/work_items/execution_preparation.ts index 30c1f4ce17..882f0a7bc3 100644 --- a/loopx/control_plane/work_items/execution_preparation.ts +++ b/loopx/control_plane/work_items/execution_preparation.ts @@ -1,6 +1,7 @@ import {HANDOFF_MODES} from "../coordination/handoff_mode_vocabulary.ts"; import {HANDOFF_MODE_RETIRED_REASON, retiredHandoffModeRejection} from "../coordination/handoff_mode_policy.ts"; import type {JsonObject} from "../effect_program.ts"; +import {parseIsoTimestamp} from "../runtime_timestamp.ts"; import {EffectRuntimeRequestError} from "../effect_runtime_errors.ts"; import { optionalNonEmptyString, @@ -36,8 +37,14 @@ function leaseFact(value: unknown, label: string): JsonObject { } proof = {idempotency_key: key, expected_version: version}; } + const expiresAt = optionalNonEmptyString(lease.expires_at, `${label}.expires_at`); + const deadline = expiresAt === null ? null : parseIsoTimestamp(expiresAt); + if (expiresAt !== null && deadline === null) { + throw new EffectRuntimeRequestError(`${label}.expires_at must be a valid ISO timestamp`); + } return {status, active, ...(owner === null ? {} : {owner}), - ...(proof === undefined ? {} : {lease_proof: proof})}; + ...(proof === undefined ? {} : {lease_proof: proof}), + ...(deadline === null ? {} : {lease_expires_at_ms: deadline.valueOf()})}; } function claimAction(goalId: string, todoId: string, agentId: string): JsonObject { @@ -128,7 +135,9 @@ export function projectExecutionPreparation(value: unknown): JsonObject { reason_code: "matching_active_task_lease", // Capture the admitted generation once. Recovery must not borrow a // replacement lease after the original Host result was produced. - ...(active[0].lease_proof === undefined ? {} : {lease_proof: active[0].lease_proof})}; + ...(active[0].lease_proof === undefined ? {} : {lease_proof: active[0].lease_proof}), + // Only the live Host launch read requests the same row's deadline. + ...(active[0].lease_expires_at_ms === undefined ? {} : {lease_expires_at_ms: active[0].lease_expires_at_ms})}; } const prior = leases[0]; diff --git a/loopx/extensions/process_runtime.py b/loopx/extensions/process_runtime.py index bb7875e279..f35066d577 100644 --- a/loopx/extensions/process_runtime.py +++ b/loopx/extensions/process_runtime.py @@ -9,7 +9,8 @@ import sys import threading import time -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any, BinaryIO, IO, cast @@ -19,23 +20,103 @@ _PROCESS_GROUP_STOP_TIMEOUT_SECONDS = 1.0 +class ProcessStdinLeaseExpired(TimeoutError): + """The lease expired before a complete frame could be published.""" + + +@contextmanager +def _nonblocking_pipe_writer(fd: int) -> Iterator[Callable[[memoryview], int]]: + if os.name == "nt": + # os.set_blocking only supports Windows pipes from Python 3.12. + import _winapi + import msvcrt + + handle = msvcrt.get_osfhandle(fd) + _winapi.SetNamedPipeHandleState(handle, 1, None, None) # PIPE_NOWAIT + try: + yield lambda data: _winapi.WriteFile(handle, data)[0] + finally: + _winapi.SetNamedPipeHandleState(handle, 0, None, None) # PIPE_WAIT + else: + was_blocking = os.get_blocking(fd) + os.set_blocking(fd, False) + try: + yield lambda data: os.write(fd, data) + finally: + os.set_blocking(fd, was_blocking) + + +def _retire_native_input_receiver( + process: subprocess.Popen[str], cancel_and_reap: Callable[[], None], +) -> None: + try: + cancel_and_reap() + except BaseException as exc: + raise RuntimeError("native input receiver cleanup unconfirmed") from exc + if process.poll() is None: + raise RuntimeError("native input receiver was not reaped") + + +def _write_lease_bound_frame( + process: subprocess.Popen[str], stream: IO[str], text: str, *, + deadline: float, expires_at_ms: int, cancel_and_reap: Callable[[], None], +) -> None: + if time.time() * 1000 >= expires_at_ms: + raise ProcessStdinLeaseExpired("native input lease expired before dispatch") + frame = memoryview(text.replace("\n", os.linesep).encode(stream.encoding, errors=stream.errors)) + try: + # A blocking write can publish the final bytes after expiry despite a + # time-bounded wait in the parent. The original receiver gets only + # nonblocking writes, each authorized immediately before submission. + with _nonblocking_pipe_writer(stream.fileno()) as write: + while frame: + chunk = frame[:_PROCESS_IO_CHUNK_BYTES] + remaining = (expires_at_ms - time.time() * 1000) / 1000 + if remaining <= 0: + raise ProcessStdinLeaseExpired("native input lease expired before dispatch") + budget = deadline - time.monotonic() + if budget <= 0: + raise TimeoutError("native input dispatch deadline expired") + try: + written = write(chunk) + except BlockingIOError: + written = 0 + if written: + frame = frame[written:] + else: + time.sleep(min(0.005, remaining, budget)) + except BaseException: + # Reap before closing stdin so an incomplete line cannot become an + # EOF-terminated request. Unconfirmed cleanup is still an unknown effect. + _retire_native_input_receiver(process, cancel_and_reap) + raise + + def write_process_stdin_before_deadline( process: subprocess.Popen[str], text: str, *, deadline: float, - cancel_and_reap: Callable[[], None], + cancel_and_reap: Callable[[], None], lease_expires_at_ms: int | None = None, + retire_on_expired_deadline: bool = False, ) -> None: """Finish one native frame, or cancel its peer before returning failure. The caller owns the serialization and launch fence. A bounded wait keeps a full pipe from stranding that fence; the same process is reaped and its - writer joined before the caller releases it. Use the existing text stream - so successful large/Unicode frames retain their original wire semantics on - every supported Python platform, including Windows 3.11. + writer joined before the caller releases it. A lease-bound send uses + nonblocking writes so the receiver cannot get a complete frame after the + lease expires. Both paths preserve the text stream's wire encoding. """ stream = process.stdin if stream is None: raise BrokenPipeError("native input stream is closed") if time.monotonic() >= deadline: + # Native admission can consume the RPC budget before a writer starts. + if retire_on_expired_deadline: + _retire_native_input_receiver(process, cancel_and_reap) raise TimeoutError("native input dispatch deadline expired") + if lease_expires_at_ms is not None: + _write_lease_bound_frame(process, stream, text, deadline=deadline, + expires_at_ms=lease_expires_at_ms, cancel_and_reap=cancel_and_reap) + return finished = threading.Event() failures: list[BaseException] = [] diff --git a/loopx/quota.py b/loopx/quota.py index 4dac35fb65..ffdd711603 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1087,6 +1087,7 @@ def build_quota_should_run( workspace_path: Path | None = None, goal_ref: Mapping[str, object] | None = None, authoritative_fallback_todo_items: FallbackTodoSource = None, + include_launch_lease_deadline: bool = False, ) -> dict[str, Any]: from .control_plane.quota.should_run import ( build_quota_should_run as _build_quota_should_run, @@ -1132,6 +1133,7 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None) -> dict[s goal_ref=goal_ref, authoritative_fallback_todo_items=authoritative_fallback_todo_items, replan_explore_reader=read_explore_context, + include_launch_lease_deadline=include_launch_lease_deadline, ) diff --git a/tests/control_plane/host_process_fixture.py b/tests/control_plane/host_process_fixture.py index 6929f95d03..d0e31a6442 100644 --- a/tests/control_plane/host_process_fixture.py +++ b/tests/control_plane/host_process_fixture.py @@ -58,7 +58,7 @@ def expect_lease_release_after_dispatch(monkeypatch, *, operation, registry, run @contextmanager def contend(admission, *, operation: str): nonlocal child - with original(admission, operation=operation): + with original(admission, operation=operation) as expires_at_ms: if operation == dispatch_operation: assert child is None assert (dispatch_marker.read_bytes() if dispatch_marker.exists() else None) == initial @@ -74,7 +74,7 @@ def contend(admission, *, operation: str): pass else: pytest.fail(f"lease release crossed the dispatch guard: {output}") - yield + yield expires_at_ms dispatch_operation = operation try: @@ -93,6 +93,35 @@ def contend(admission, *, operation: str): child.communicate(timeout=5) +@contextmanager +def expect_lease_expiry_during_dispatch(monkeypatch, *, operation, observed_lease): + """Let an unchanged canonical lease expire after the actual launch read.""" + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + from loopx.control_plane.work_items.task_lease import lease_expires_at, lease_is_active + + original = FirstPartyHostGoalAdmission.launch_lifetime + crossed = [] + + @contextmanager + def pause(admission, *, operation): + with original(admission, operation=operation) as expires_at_ms: + if operation == dispatch_operation: + lease = observed_lease() + assert lease_is_active(lease), lease + expires_at = lease_expires_at(lease) + assert expires_at is not None + while lease_is_active(lease): + time.sleep(min(0.02, max(0, expires_at.timestamp() - time.time()))) + crossed.append(lease["idempotency_key"]) + yield expires_at_ms + + dispatch_operation = operation + with monkeypatch.context() as patch: + patch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", pause) + yield + assert len(crossed) == 1, "expected one admitted dispatch to cross the lease deadline" + + def assert_host_task_context(model_input, source): """Check source parity outside the executor-owned presentation boundary. diff --git a/tests/control_plane/test_codex_operation_launch.py b/tests/control_plane/test_codex_operation_launch.py index cf853b7360..9dad8a14f0 100644 --- a/tests/control_plane/test_codex_operation_launch.py +++ b/tests/control_plane/test_codex_operation_launch.py @@ -19,6 +19,7 @@ from loopx.control_plane.goals import activation_service, first_party_host_admission from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission from loopx.control_plane.projects.registry_codec import project_registry_transaction +from loopx.control_plane.work_items.task_lease import lease_expires_at from loopx.control_plane.turn_driver import codex_operation_host from loopx.global_registry import sync_project_registry_to_global from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime @@ -27,9 +28,9 @@ from tests.test_loopx_turn_driver import _write_live_fixture -def _fixture(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str): +def _fixture(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, *, handoff_mode="soft_claim"): isolate_sqlite_runtime(tmp_path, monkeypatch) - project, runtime, registry = _write_live_fixture(tmp_path, provider=provider) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider, handoff_mode=handoff_mode) native, turns = tmp_path / "native-starts", tmp_path / "native-turns" executable = tmp_path / "fake-codex-operation" executable.write_text(FAKE_SERVER.replace( @@ -559,8 +560,90 @@ def lifetime(self, *, operation): assert not index.exists() or "quota_slot_spent" not in index.read_text() -def test_guarded_native_dispatch_preserves_large_unicode_input(tmp_path, monkeypatch): - _, _, _, turns, arguments = _fixture(tmp_path, monkeypatch, "sqlite") +def _lease(arguments, action, *, version, ttl): + code, payload = _cli([ + *arguments[:6], "task-lease", action, "--goal-id", "loopx-turn-fixture", + "--todo-id", "todo_fixture0001", "--owner", "codex-fixture", + "--idempotency-key", "native-input-lease", "--expected-version", str(version), + "--ttl-seconds", str(ttl), + ]) + assert code == 0, payload + return payload["lease"] + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_expired_lease_cannot_publish_backpressured_native_frame(tmp_path, monkeypatch, provider): + from loopx.extensions import process_runtime + + runtime, _, native, turns, arguments = _fixture(tmp_path, monkeypatch, provider, handoff_mode="hard_lease") + original = _lease(arguments, "acquire", version=0, ttl=300) + pending = tmp_path / "partial-native-frame" + release = tmp_path / "release-native-reader" + resumed = tmp_path / "native-reader-resumed" + executable = Path(arguments[arguments.index("--codex-bin") + 1]) + ready = ' emit({"id": row["id"], "result": {"thread": {"id": thread}, "model": "test-model", "reasoningEffort": "xhigh"}})' + source = executable.read_text() + assert ready in source + executable.write_text(source.replace( + "for line in sys.stdin:\n row = json.loads(line)", + "prefix = ''\nwhile line := sys.stdin.readline():\n" + " row = json.loads(prefix + line)\n prefix = ''", 1, + ).replace(ready, ready + "\n" + " import time\n" + " prefix = sys.stdin.read(2048)\n" + f" pathlib.Path({str(pending)!r}).touch()\n" + f" while not pathlib.Path({str(release)!r}).exists(): time.sleep(.005)\n" + f" pathlib.Path({str(resumed)!r}).touch()", 1)) + original_input = codex_operation_host.prepare_codex_cli_input + original_cancel = process_runtime.force_terminate_process + deadline = [] + + def oversized_input(request): + text, schema = original_input(request) + renewed = _lease(arguments, "renew", version=original["version"], ttl=8) + assert renewed["lease_epoch"] == original["lease_epoch"] + deadline.append(lease_expires_at(renewed).timestamp()) + return text + "\n" + "界" * (1024 * 1024), schema + + def delay_retirement(process, **kwargs): + # Let the receiver start draining after expiry but before termination. + # An outstanding blocking write could still publish the complete line. + limit = time.monotonic() + 2 + while not resumed.exists() and time.monotonic() < limit: + time.sleep(.005) + limit = time.monotonic() + .25 + while not turns.exists() and time.monotonic() < limit: + time.sleep(.005) + original_cancel(process, **kwargs) + + monkeypatch.setattr(codex_operation_host, "prepare_codex_cli_input", oversized_input) + monkeypatch.setattr(process_runtime, "force_terminate_process", delay_retirement) + with ThreadPoolExecutor(max_workers=1) as executor: + host = executor.submit(_cli, [*arguments, "--execute"]) + try: + limit = time.monotonic() + 20 + while not pending.exists() and not host.done() and time.monotonic() < limit: + time.sleep(.01) + assert pending.exists() and time.time() < deadline[0], "the active lease must reach real pipe backpressure" + time.sleep(max(0, deadline[0] - time.time()) + .025) + release.touch() + code, payload = host.result(timeout=12) + finally: + release.touch() + + assert native.exists() and resumed.exists() and not turns.exists() + assert code == 1 and payload.get("error_code") == "goal_launch_not_allowed", payload + assert payload["host_failure"]["kind"] == "contract_rejected" + assert not any(payload["effects"].values()), payload + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("handoff_mode", ["soft_claim", "hard_lease"]) +def test_guarded_native_dispatch_preserves_large_unicode_input(tmp_path, monkeypatch, handoff_mode): + _, _, _, turns, arguments = _fixture(tmp_path, monkeypatch, "sqlite", handoff_mode=handoff_mode) + if handoff_mode == "hard_lease": + _lease(arguments, "acquire", version=0, ttl=300) observed = tmp_path / "accepted-native-input" monkeypatch.setenv("FAKE_OPERATION_PROMPT_FILE", str(observed)) original_input = codex_operation_host.prepare_codex_cli_input diff --git a/tests/control_plane/test_host_current_admission.py b/tests/control_plane/test_host_current_admission.py index f7c19c4505..5a0d4101c4 100644 --- a/tests/control_plane/test_host_current_admission.py +++ b/tests/control_plane/test_host_current_admission.py @@ -15,6 +15,7 @@ from tests.control_plane.test_codex_operation_launch import _bindings, _cli, _fixture from tests.control_plane.host_process_fixture import ( WAIT_FOR_FIXTURE_LEASE_RELEASE, + expect_lease_expiry_during_dispatch, expect_lease_release_after_dispatch, ) from tests.test_loopx_turn_codex_cli import _fake_codex @@ -160,7 +161,8 @@ def edit_after_planning(request): @pytest.mark.parametrize("provider", ["file", "sqlite"]) -@pytest.mark.parametrize("change", ["unchanged", "text", "renew_lease", "replace_lease", "release_during_dispatch"]) +@pytest.mark.parametrize("change", ["unchanged", "text", "renew_lease", "replace_lease", + "release_during_dispatch", "expire_during_dispatch"]) def test_stdio_retry_checks_original_task_before_launch(tmp_path, monkeypatch, provider, change): from loopx.control_plane.coordination.runtime_shadow import build_todo_runtime_shadow_projection from loopx.control_plane.todos.active_state_todo_parser import parse_active_state_todos @@ -172,7 +174,7 @@ def test_stdio_retry_checks_original_task_before_launch(tmp_path, monkeypatch, p project, runtime, workspace, registry = fixture._write_fixture(tmp_path, turn_count=2) state = project / ".codex/goals" / fixture.GOAL_ID / "ACTIVE_GOAL_STATE.md" todos = parse_active_state_todos(state.read_text(), state_path=state, item_limit=None) - uses_lease = change in {"renew_lease", "replace_lease", "release_during_dispatch"} + uses_lease = change in {"renew_lease", "replace_lease", "release_during_dispatch", "expire_during_dispatch"} initialize_canonical_authority(runtime, fixture.GOAL_ID, build_todo_runtime_shadow_projection(goal_id=fixture.GOAL_ID, handoff_mode="hard_lease" if uses_lease else "soft_claim", @@ -244,11 +246,19 @@ def lease(action, key, version, *options): dispatch = expect_lease_release_after_dispatch(monkeypatch, operation="turn_external_host_launch", registry=registry, runtime=runtime, goal_id=fixture.GOAL_ID, todo_id=fixture.TODO_ID, lease=original, dispatch_marker=marker) + elif change == "expire_during_dispatch": + renewed = lease("renew", original["idempotency_key"], original["version"], "--ttl-seconds", "8") + assert renewed["lease_epoch"] == original["lease_epoch"] + dispatch = expect_lease_expiry_during_dispatch(monkeypatch, operation="turn_external_host_launch", + observed_lease=lambda: renewed) with dispatch: code, result = fixture._run_cli([*recovery, "--retry-failed-turn"]) - refused = change in {"text", "replace_lease"} + refused = change in {"text", "replace_lease", "expire_during_dispatch"} if refused: - assert code == 1 and result["error_code"] == "current_turn_work_changed", result + expected = "goal_launch_not_allowed" if change == "expire_during_dispatch" else "current_turn_work_changed" + assert code == 1 and result.get("error_code") == expected, ( + code, result.get("status"), result.get("error_code"), result.get("effects"), + ) assert result["host_failure"]["kind"] == "contract_rejected", result assert not any(result["effects"].values()) else: diff --git a/tests/test_host_launch_authority.py b/tests/test_host_launch_authority.py index f0b0e9c177..15d7cdbc92 100644 --- a/tests/test_host_launch_authority.py +++ b/tests/test_host_launch_authority.py @@ -33,6 +33,7 @@ from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime from tests.control_plane.host_process_fixture import ( WAIT_FOR_FIXTURE_LEASE_RELEASE, + expect_lease_expiry_during_dispatch, expect_lease_release_after_dispatch, ) from tests.control_plane.test_effect_turn_live_quota_decision import ( @@ -360,7 +361,7 @@ def test_first_normal_turn_with_declared_state_needs_no_global_registry( @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize("operation_tools", [False, True]) -@pytest.mark.parametrize("change", ["renew", "replace", "release_during_dispatch"]) +@pytest.mark.parametrize("change", ["renew", "replace", "release_during_dispatch", "expire_during_dispatch"]) def test_actual_launch_preserves_original_task_lease_generation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation_tools: bool, change: str, ): @@ -391,9 +392,10 @@ def lease(action, key, version, *options): changes = [] def change_lease_before_dispatch(request): - if change == "renew": + if change in {"renew", "expire_during_dispatch"}: + ttl = "8" if change == "expire_during_dispatch" else "600" current = lease("renew", original["idempotency_key"], original["version"], - "--ttl-seconds", "600") + "--ttl-seconds", ttl) assert current["lease_epoch"] == original["lease_epoch"] else: released = lease("release", original["idempotency_key"], original["version"]) @@ -404,10 +406,16 @@ def change_lease_before_dispatch(request): return prepare_input(request) monkeypatch.setattr(owner, "prepare_codex_cli_input", change_lease_before_dispatch) - code, result = _run(registry, runtime, project, host_arguments) + if change == "expire_during_dispatch": + operation = "codex_operation_turn_start" if operation_tools else "codex_cli_host_launch" + with expect_lease_expiry_during_dispatch(monkeypatch, operation=operation, observed_lease=lambda: changes[0]): + code, result = _run(registry, runtime, project, host_arguments) + else: + code, result = _run(registry, runtime, project, host_arguments) assert len(changes) == 1, result - if change == "replace": - assert code == 1 and result.get("error_code") == "current_turn_work_changed", ( + if change in {"replace", "expire_during_dispatch"}: + expected = "current_turn_work_changed" if change == "replace" else "goal_launch_not_allowed" + assert code == 1 and result.get("error_code") == expected, ( code, result.get("status"), result.get("error_code"), result.get("effects"), ) assert result["host_failure"]["kind"] == "contract_rejected" @@ -416,7 +424,7 @@ def change_lease_before_dispatch(request): _assert_no_settlement(runtime, result) monkeypatch.setattr(owner, "prepare_codex_cli_input", prepare_input) code, retry = _run(registry, runtime, project, host_arguments, resume_key=result["resume_turn_key"]) - assert code == 1 and retry.get("error_code") == "current_turn_work_changed", retry + assert code == 1 and retry.get("error_code") == expected, retry assert retry["resume_turn_key"] == result["resume_turn_key"] assert retry["effects"]["host_invoked"] is False and not turn_start.exists() _assert_no_settlement(runtime, retry) From e65b0740fd9d705a7ccd6ac6d790d892308b6a29 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:20:42 +0800 Subject: [PATCH 04/11] Honor the live Host lease at Benchmark dispatch Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../runtime/provider_request_admission.py | 11 +++- benchmark/runtime/worker.py | 6 ++- loopx/cli_commands/turn_decision.py | 1 + .../test_provider_gateway_observation.py | 26 ++++++++-- .../test_benchmark_current_admission.py | 50 +++++++++++++++++-- 5 files changed, 84 insertions(+), 10 deletions(-) diff --git a/benchmark/runtime/provider_request_admission.py b/benchmark/runtime/provider_request_admission.py index de03bb9961..e67ce438d8 100644 --- a/benchmark/runtime/provider_request_admission.py +++ b/benchmark/runtime/provider_request_admission.py @@ -149,13 +149,20 @@ def lifetime(self): self._require_live() entered = False try: - with self._invocation.admission.launch_lifetime(operation="provider_request_dispatch"): + with self._invocation.admission.launch_lifetime(operation="provider_request_dispatch") as expires_at_ms: entered = True with self._lock: # Authority/registry contention can outlive the process or # phase. Revalidate after every wait and before any POST. self._require_live() - yield min(time.monotonic() + 2, self._invocation.deadline) + now = time.monotonic() + deadline = min(now + 2, self._invocation.deadline) + if expires_at_ms is not None: + remaining = expires_at_ms / 1000 - time.time() + if remaining <= 0: + self._reject("goal_launch_not_allowed") + deadline = min(deadline, now + remaining) + yield deadline except ProviderGatewayRequestRejected: raise except FirstPartyHostRuntimeRejected as error: diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index 98e33f6109..238eff35b7 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -149,9 +149,13 @@ def child_process(command: list[str], *, env: dict[str, str], stdout, stderr, # Session selection may precede rendering/capture. Serialize only actual # launch with the existing Goal lifetime owner, never the whole model run. with (goal_admission.launch_lifetime(operation="codex_agent_host_launch") - if goal_admission is not None else nullcontext()): + if goal_admission is not None else nullcontext()) as expires_at_ms: # The child can write a native event before Popen returns. launch_started_at_unix = time.time() + if expires_at_ms is not None and launch_started_at_unix * 1000 >= expires_at_ms: + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostLaunchRejected + + raise FirstPartyHostLaunchRejected("goal_launch_not_allowed") process = subprocess.Popen( command, cwd=env["LOOPX_PROJECT"], diff --git a/loopx/cli_commands/turn_decision.py b/loopx/cli_commands/turn_decision.py index f150944295..65a68c9db7 100644 --- a/loopx/cli_commands/turn_decision.py +++ b/loopx/cli_commands/turn_decision.py @@ -487,6 +487,7 @@ def current_heartbeat_host_launch_decision( receipt_bound_replan_obligation_id=selection.receipt_bound_replan_obligation_id, retained_action_selection_todo_id=selection.retained_todo_id_for_decision, turn_instance_id=original_turn, workspace_path=project, goal_ref=goal_ref, + include_launch_lease_deadline=True, interaction_projection_hooks=(periodic_report_pending_intent_interaction_hook( registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id, agent_id=agent_id, ),), diff --git a/tests/capabilities/test_provider_gateway_observation.py b/tests/capabilities/test_provider_gateway_observation.py index f2e21156b0..1df1a6be52 100644 --- a/tests/capabilities/test_provider_gateway_observation.py +++ b/tests/capabilities/test_provider_gateway_observation.py @@ -17,13 +17,14 @@ class _Admission: """Expose the lock ordering without introducing a second authority policy.""" - def __init__(self): + def __init__(self, *, expires_at_ms=None): self.lock = threading.Lock() self.local = threading.local() self.entries = 0 self.pause_second = False self.paused = threading.Event() self.resume = threading.Event() + self.expires_at_ms = expires_at_ms def with_launch_decision(self, _check): return self @@ -37,7 +38,7 @@ def launch_lifetime(self, **_kwargs): if self.pause_second and self.entries == 2: self.paused.set() assert self.resume.wait(timeout=10) - yield + yield self.expires_at_ms finally: self.local.held = False @@ -76,7 +77,7 @@ def do_POST(self): thread.join(timeout=5) -def _guard(tmp_path, upstream, admission): +def _guard(tmp_path, upstream, admission, *, process=None): receipt = {} session = SimpleNamespace(admission=admission, goal_ref={}, session_id="synthetic-session") guard = ProviderRequestAdmission( @@ -84,7 +85,7 @@ def _guard(tmp_path, upstream, admission): "LOOPX_TURN": "synthetic-turn"}, session, receipt, tmp_path, ) - guard.started(SimpleNamespace(poll=lambda: None)) + guard.started(process if process is not None else SimpleNamespace(poll=lambda: None)) return guard, receipt @@ -111,6 +112,23 @@ def run(): return thread, result +def test_expired_host_authority_refuses_provider_post(tmp_path, monkeypatch): + signals = [] + monkeypatch.setattr("benchmark.runtime.provider_request_admission.os.killpg", + lambda pid, signal: signals.append((pid, signal))) + process = SimpleNamespace(pid=1, poll=lambda: None) + with _waiting_upstream() as (upstream, received, _, _): + guard, receipt = _guard(tmp_path, upstream, _Admission(expires_at_ms=1), process=process) + with guard.serve(["codex", "exec", "-"]): + client, result = _request(guard.base_url) + client.join(timeout=5) + assert not client.is_alive() and result.get("status") == 409, result + assert not received.is_set() + summary, = receipt["provider_request_invocations"] + assert summary["refusal_code"] == "goal_launch_not_allowed" and summary["sent"] == 0 + assert len(signals) == 1 and signals[0][0] == process.pid + + def test_provider_observes_dispatch_while_upstream_response_is_pending(tmp_path, monkeypatch): with _waiting_upstream() as (upstream, received, release, observed): admission = _Admission() diff --git a/tests/control_plane/test_benchmark_current_admission.py b/tests/control_plane/test_benchmark_current_admission.py index 62740ffc74..c0fddb575a 100644 --- a/tests/control_plane/test_benchmark_current_admission.py +++ b/tests/control_plane/test_benchmark_current_admission.py @@ -5,6 +5,7 @@ from pathlib import Path import sys import time +from contextlib import contextmanager import pytest @@ -13,7 +14,7 @@ from loopx.configure_goal import configure_goal from loopx.cli_commands import turn_decision from loopx.control_plane.goals import activation_service -from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected +from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission, FirstPartyHostRuntimeRejected from loopx import contract, status as status_owner from loopx.cli import main from loopx.global_registry import sync_project_registry_to_global @@ -37,7 +38,7 @@ def _private_workspace(project): return path -def _environment(tmp_path, monkeypatch, provider, *, replan=False): +def _environment(tmp_path, monkeypatch, provider, *, replan=False, handoff_mode="soft_claim"): isolate_sqlite_runtime(tmp_path, monkeypatch) monkeypatch.setenv("LOOPX_USAGE_PING", "0") monkeypatch.setenv("PYTHONPATH", str(ROOT)) @@ -55,7 +56,7 @@ def _environment(tmp_path, monkeypatch, provider, *, replan=False): fixture._configure_selected_todo_replan_fixture(project, source) elif replan: fixture._configure_autonomous_replan_fixture(project, runtime, source) - fixture._initialize_fixture_authority(project, runtime, provider=provider) + fixture._initialize_fixture_authority(project, runtime, provider=provider, handoff_mode=handoff_mode) synced = sync_project_registry_to_global( registry_path=source, runtime_root_override=str(runtime), goal_id=fixture.GOAL_ID, ) @@ -154,6 +155,49 @@ def after_session_selection(*arguments): assert not index.exists() or "quota_slot_spent" not in index.read_text() +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("expire_after_check", [False, True]) +def test_heartbeat_worker_consumes_hard_lease_deadline_at_spawn( + tmp_path, monkeypatch, provider, expire_after_check, +): + env, project, runtime, source, _ = _environment(tmp_path, monkeypatch, provider, handoff_mode="hard_lease") + lease_command = ["--goal-id", fixture.GOAL_ID, "--todo-id", fixture.TODO_ID, + "--owner", fixture.AGENT_ID, "--idempotency-key", "benchmark-original-lease"] + code, acquired = fixture._run_cli(source, runtime, "task-lease", "acquire", + *lease_command, "--expected-version", "0", "--ttl-seconds", "300") + assert code == 0 and acquired["ok"], acquired + if expire_after_check: + native_command = worker.native_command + launch_lifetime = FirstPartyHostGoalAdmission.launch_lifetime + + def shorten_lease(*args, **kwargs): + command = native_command(*args, **kwargs) + code, renewed = fixture._run_cli(source, runtime, "task-lease", "renew", + *lease_command, "--expected-version", str(acquired["lease"]["version"]), "--ttl-seconds", "6") + assert code == 0 and renewed["lease"]["lease_epoch"] == acquired["lease"]["lease_epoch"], renewed + return command + + @contextmanager + def wait_after_check(admission, *, operation): + with launch_lifetime(admission, operation=operation) as deadline: + if operation == "codex_agent_host_launch": + assert deadline is not None and deadline > time.time() * 1000 + time.sleep(max(0, deadline / 1000 - time.time()) + 0.01) + yield deadline + + monkeypatch.setattr(worker, "native_command", shorten_lease) + monkeypatch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", wait_after_check) + with pytest.raises(FirstPartyHostRuntimeRejected, match="goal_launch_not_allowed"): + worker.run_once(env) + assert not (project / "native-starts").exists() + assert not (project / "model-starts").exists() + else: + assert worker.run_once(env)["ok"] + assert (project / "native-starts").read_text().splitlines() == ["native"] + assert (project / "model-starts").read_text().splitlines() == ["model"] + assert fixture._spend_run_count(runtime) == 0 + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize("mode", ["heartbeat", "turn"]) def test_initial_benchmark_planner_can_launch_without_a_prior_heartbeat_receipt(tmp_path, monkeypatch, provider, mode): From 79de0cd7d1c0abc85bbddef2405907017154d1d9 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:50:45 +0800 Subject: [PATCH 05/11] Carry the admitted task lease deadline through native Chat dispatch Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../capabilities/native_chat/codex_adapter.py | 4 +- .../capabilities/native_chat/steward_turn.py | 11 ++-- tests/test_native_steward_turn_host.py | 61 +++++++++++++++++++ 3 files changed, 70 insertions(+), 6 deletions(-) diff --git a/loopx/capabilities/native_chat/codex_adapter.py b/loopx/capabilities/native_chat/codex_adapter.py index 7fd5dd5d58..161adba9d7 100644 --- a/loopx/capabilities/native_chat/codex_adapter.py +++ b/loopx/capabilities/native_chat/codex_adapter.py @@ -1020,9 +1020,9 @@ def read_turn(): if previous_guard is None: yield else: - with previous_guard(): + with previous_guard() as expires_at_ms: check() - yield + yield expires_at_ms # Native sandboxPolicy is sticky across subsequent Turns. Reassert the # current managed Chat read boundary without changing Session defaults. return {"sandbox_policy": {"type": "readOnly", "networkAccess": False}, diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index 96083a9160..1378cb339b 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -801,9 +801,10 @@ def launch(): observed = self._observe(actor) with ExitStack() as stack: try: - if previous_guard is not None: - stack.enter_context(previous_guard()) - stack.enter_context(launch_scope()) + previous_expiry = ( + stack.enter_context(previous_guard()) if previous_guard is not None else None + ) + expires_at_ms = stack.enter_context(launch_scope()) stack.enter_context(nullcontext() if actor.get("schema_version") == "loopx_managed_session_actor_v0" else exclusive_file_lock(self.bindings.path, operation="native_steward_launch")) current(observed) @@ -815,7 +816,9 @@ def launch(): # failure; exceptions after yield remain unknown. from ...control_plane.turn_driver.host_failure import BuiltInHostError raise BuiltInHostError(str(exc), failure_kind="contract_rejected") from exc - yield + if previous_expiry is not None and (expires_at_ms is None or previous_expiry < expires_at_ms): + expires_at_ms = previous_expiry + yield expires_at_ms adapter.session._launch_guard = launch try: diff --git a/tests/test_native_steward_turn_host.py b/tests/test_native_steward_turn_host.py index 42f9d914ee..3c9b94bc8e 100644 --- a/tests/test_native_steward_turn_host.py +++ b/tests/test_native_steward_turn_host.py @@ -1108,6 +1108,67 @@ def changed(*args, **kwargs): assert store.load_session(sid)["upstream_thread_id"] == thread +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_managed_host_keeps_canonical_deadline_through_app_server_send(steward, tmp_path, monkeypatch, provider): + import time + + from loopx.capabilities.native_chat import steward_commands + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission + from loopx.extensions import process_runtime + + service, session, actor, todo_id, _, capture = managed_fixture(steward, tmp_path, monkeypatch, provider) + store, runtime, *_ = steward + run_command = steward_commands.native_command + launch_lifetime = FirstPartyHostGoalAdmission.launch_lifetime + send_frame = process_runtime.write_process_stdin_before_deadline + admitted_deadlines = [] + crossed = [] + + def acquire_short_lease(controller, argv): + if argv[:2] == ["task-lease", "acquire"] and todo_id in argv: + argv = list(argv) + argv[argv.index("--ttl-seconds") + 1] = "12" + return run_command(controller, argv) + + @contextlib.contextmanager + def observe_admission(admission, *, operation): + with launch_lifetime(admission, operation=operation) as expires_at_ms: + if operation == "native_conversation_turn_launch": + assert type(expires_at_ms) is int + admitted_deadlines.append(expires_at_ms) + yield expires_at_ms + + def send_after_expiry(process, frame, **kwargs): + if json.loads(frame).get("method") == "turn/start": + assert len(admitted_deadlines) == 1 + remaining = admitted_deadlines[0] / 1000 - time.time() + assert remaining > 0, "the active lease must reach the App Server transport" + time.sleep(remaining + .03) + crossed.append(admitted_deadlines[0]) + return send_frame(process, frame, **kwargs) + + monkeypatch.setattr(steward_commands, "native_command", acquire_short_lease) + monkeypatch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", observe_admission) + monkeypatch.setattr(process_runtime, "write_process_stdin_before_deadline", send_after_expiry) + before = capture.read_bytes() + accepted, created = service.submit(session["session_id"], agent_id=actor["agent_id"], + operation_id="expired-at-app-server-transport", todo_id=todo_id) + assert created + completed = runtime.wait_for_turn(session_id=accepted["session_id"], turn_id=accepted["turn_id"], timeout_sec=90) + assert len(crossed) == 1 + calls = [json.loads(line) for line in capture.read_bytes()[len(before):].splitlines()] + assert all(row["method"] != "turn/start" for row in calls), [row["method"] for row in calls] + result = completed["response"]["native_loopx"] + assert result["error_code"] == "goal_launch_not_allowed", result + assert result["host_failure"]["kind"] == "contract_rejected" + assert not any(result["effects"].values()) + current = cli(runtime, "todo", "list", "--goal-id", actor["goal_id"], "--todo-id", todo_id)["todo"] + assert not current["done"] + assert store.load_session(session["session_id"])["upstream_thread_id"] == session["upstream_thread_id"] + index = runtime.coordination_runtime_root / "goals" / actor["goal_id"] / "runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_managed_host_does_not_borrow_another_execution_lease(steward, tmp_path, monkeypatch, provider): service, session, actor, todo_id, _, capture = managed_fixture(steward, tmp_path, monkeypatch, provider) From 0243ed71e2703fa824514b0806d1f6fc851bd6ab Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:11:44 +0800 Subject: [PATCH 06/11] Hold native Turn recovery when lease-bound dispatch timing is unconfirmed Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/reference/protocols/turn-envelope-v0.md | 22 +++--- loopx/control_plane/turn_driver/executor.py | 14 +++- loopx/extensions/process_runtime.py | 29 ++++++-- .../test_codex_operation_launch.py | 73 +++++++++++++++++++ 4 files changed, 121 insertions(+), 17 deletions(-) diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index 0db045735a..87c9ebfd11 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -494,18 +494,22 @@ from the live check through process spawn or the native `turn/start` send. A writer that commits first changes the inputs to the live check. A writer that arrives after the check can continue once dispatch completes, while the Host runs. The same live hard-lease read supplies the dispatch deadline. The process -supervisor checks it at spawn, and the native sender uses nonblocking writes so -a backed-up pipe cannot publish a complete `turn/start` after expiry. A proven -pre-dispatch expiry returns `goal_launch_not_allowed` with no Host invocation, -writeback or debit. An unconfirmed dispatch still follows the existing recovery -path. Registry contention can refuse dispatch with `goal_authority_unavailable`. +supervisor checks it at spawn. The native sender checks the deadline before +each nonblocking write and after the complete `turn/start` frame is accepted +by the pipe. A proven pre-dispatch expiry returns `goal_launch_not_allowed` +with no Host invocation, writeback or debit. If a complete frame may have been +delivered after expiry, the sender retires the receiver and the original +attempt stays in progress with `host_outcome_unknown`. Recovery holds retry +pending reconciliation of that attempt. Registry contention can refuse dispatch +with `goal_authority_unavailable`. The journal retains the original capture, identity and saved session recovery for retry or readback. -硬租约的派发截止时间来自同一次权威读。进程启动前检查到期时间,原生 -`turn/start` 使用非阻塞写入,防止管道积压导致到期后仍发布完整帧。确认 -到期前尚未派发时,返回 `goal_launch_not_allowed`,不启动模型工作、不写回、 -不扣额。派发状态不明时沿用原恢复路径和原回执身份。 +硬租约的派发截止时间来自同一次权威读。进程启动前检查到期时间。原生 +`turn/start` 每次非阻塞写入前都检查时限,管道接受完整帧后再检查一次。 +确认未派发即到期时,返回 `goal_launch_not_allowed`,不启动模型工作、不写回、 +不扣额。完整帧可能在到期后送达时,发送方终止接收进程,原尝试保留在执行中并 +标为 `host_outcome_unknown`。恢复须先核对原尝试,才能再次派发。 Use a fresh eligible Turn for changed work. Reading detail consumes no quota. Source admission, cancellation, workspace and mutation authority remain with their existing owners. diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 101af459c9..446c181fa3 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -955,6 +955,8 @@ def _run_host_runner( *, runner: HostRunner, ) -> dict[str, Any]: + from ...extensions.process_runtime import ProcessStdinDispatchUnconfirmed + try: value = runner(request) except BuiltInHostError as exc: @@ -969,7 +971,7 @@ def _run_host_runner( else {} ), } - except FirstPartyHostRuntimeRejected: + except (FirstPartyHostRuntimeRejected, ProcessStdinDispatchUnconfirmed): raise except Exception as exc: # noqa: BLE001 - host adapters fail closed at boundary return {"ok": False, "reason": type(exc).__name__, "returncode": None} @@ -1037,6 +1039,7 @@ def _host_result_stage( # reservation. Confirmation failure stops before the host starts. if confirm_start is not None: confirm_start() + from ...extensions.process_runtime import ProcessStdinDispatchUnconfirmed from ...usage_goal import observe_goal_execution rejected_launch = None @@ -1057,6 +1060,15 @@ def _host_result_stage( launch_guard=host_launch_guard, ) ) + except ProcessStdinDispatchUnconfirmed as exc: + # The durable in-progress attempt already blocks blind replay. + # No Host result may enter validation or settlement without proof. + journal["reason"] = str(exc) + persist_journal(journal) + effects["host_invoked"] = True + unresolved = execution_payload(plan, journal, execute=True, replayed=False, effects=effects) + unresolved["error_code"] = "host_outcome_unknown" + return None, [], unresolved except FirstPartyHostRuntimeRejected as exc: # Stdio checks the Goal only before spawn. A native Host can reject # after dispatch too, so retain reconciliation for other failures. diff --git a/loopx/extensions/process_runtime.py b/loopx/extensions/process_runtime.py index f35066d577..a9d545a2d8 100644 --- a/loopx/extensions/process_runtime.py +++ b/loopx/extensions/process_runtime.py @@ -24,6 +24,10 @@ class ProcessStdinLeaseExpired(TimeoutError): """The lease expired before a complete frame could be published.""" +class ProcessStdinDispatchUnconfirmed(Exception): + """The original pipe receiver may have received the frame without timely proof.""" + + @contextmanager def _nonblocking_pipe_writer(fd: int) -> Iterator[Callable[[memoryview], int]]: if os.name == "nt": @@ -65,9 +69,9 @@ def _write_lease_bound_frame( raise ProcessStdinLeaseExpired("native input lease expired before dispatch") frame = memoryview(text.replace("\n", os.linesep).encode(stream.encoding, errors=stream.errors)) try: - # A blocking write can publish the final bytes after expiry despite a - # time-bounded wait in the parent. The original receiver gets only - # nonblocking writes, each authorized immediately before submission. + # A blocking write can publish late through backpressure. The final + # nonblocking write still needs a completion check because the sender + # can be preempted between checking time and entering the syscall. with _nonblocking_pipe_writer(stream.fileno()) as write: while frame: chunk = frame[:_PROCESS_IO_CHUNK_BYTES] @@ -83,12 +87,23 @@ def _write_lease_bound_frame( written = 0 if written: frame = frame[written:] + if not frame and time.time() * 1000 >= expires_at_ms: + raise ProcessStdinDispatchUnconfirmed( + "The native frame may have reached the Host after lease expiry. " + "Reconcile the original Host attempt before retrying." + ) else: time.sleep(min(0.005, remaining, budget)) except BaseException: # Reap before closing stdin so an incomplete line cannot become an # EOF-terminated request. Unconfirmed cleanup is still an unknown effect. - _retire_native_input_receiver(process, cancel_and_reap) + try: + _retire_native_input_receiver(process, cancel_and_reap) + except RuntimeError as cleanup_error: + raise ProcessStdinDispatchUnconfirmed( + "The original native Host could not be confirmed stopped. " + "Reconcile the original Host attempt before retrying." + ) from cleanup_error raise @@ -101,9 +116,9 @@ def write_process_stdin_before_deadline( The caller owns the serialization and launch fence. A bounded wait keeps a full pipe from stranding that fence; the same process is reaped and its - writer joined before the caller releases it. A lease-bound send uses - nonblocking writes so the receiver cannot get a complete frame after the - lease expires. Both paths preserve the text stream's wire encoding. + writer joined before the caller releases it. A lease-bound send checks + the deadline around nonblocking publication of the complete frame. + Both paths preserve the text stream's wire encoding. """ stream = process.stdin if stream is None: diff --git a/tests/control_plane/test_codex_operation_launch.py b/tests/control_plane/test_codex_operation_launch.py index 9dad8a14f0..93ea7959b2 100644 --- a/tests/control_plane/test_codex_operation_launch.py +++ b/tests/control_plane/test_codex_operation_launch.py @@ -571,6 +571,79 @@ def _lease(arguments, action, *, version, ttl): return payload["lease"] +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_lease_expiry_during_final_pipe_write_keeps_original_attempt_unconfirmed(tmp_path, monkeypatch, provider): + from loopx.control_plane.turn_driver.journal_store import turn_journal_path + from loopx.extensions import process_runtime + + runtime, _, native, turns, arguments = _fixture(tmp_path, monkeypatch, provider, handoff_mode="hard_lease") + original = _lease(arguments, "acquire", version=0, ttl=300) + prepare_input = codex_operation_host.prepare_codex_cli_input + pipe_writer = process_runtime._nonblocking_pipe_writer + expires_at = [] + sent_at = [] + + def renew_before_dispatch(request): + prepared = prepare_input(request) + renewed = _lease(arguments, "renew", version=original["version"], ttl=10) + assert renewed["lease_epoch"] == original["lease_epoch"] + expires_at.append(lease_expires_at(renewed).timestamp()) + return prepared + + @contextlib.contextmanager + def preempt_final_write(fd): + with pipe_writer(fd) as write: + def delayed(chunk): + if chunk[-1] != ord("\n"): + return write(chunk) + if len(chunk) > 1: + # Withhold exactly the terminator so the fixture cannot + # interpret the real pipe's partial frame as a request. + return write(chunk[:-1]) + assert len(expires_at) == 1 and time.time() < expires_at[0] + time.sleep(expires_at[0] - time.time() + .03) + count = write(chunk) + assert count == 1 + sent_at.append(time.time()) + limit = time.monotonic() + 2 + while not turns.exists() and time.monotonic() < limit: + time.sleep(.005) + return count + + yield delayed + + monkeypatch.setattr(codex_operation_host, "prepare_codex_cli_input", renew_before_dispatch) + monkeypatch.setattr(process_runtime, "_nonblocking_pipe_writer", preempt_final_write) + code, result = _cli([*arguments, "--execute"]) + assert len(sent_at) == 1 and sent_at[0] >= expires_at[0] + assert native.read_text().splitlines() == ["native"] and len(turns.read_text().splitlines()) == 1 + assert code == 1 and result["status"] == "in_progress", result + assert result["error_code"] == "host_outcome_unknown", result + assert result["effects"] == { + "host_invoked": True, "state_written": False, "quota_spent": False, "scheduler_acknowledged": False, + } + assert _bindings(runtime) + key = result["resume_turn_key"] + path = turn_journal_path(runtime, goal_id="loopx-turn-fixture", turn_key=key) + saved = path.read_bytes() + journal = json.loads(saved) + assert journal["turn_key"] == key and journal["host_attempt_count"] == 1 + assert journal["status"] == "in_progress" and not journal["completed_phases"] + capture = path.parent / "decisions" / (key.removeprefix("sha256:") + ".json") + original_capture = capture.read_bytes() + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + resume = list(arguments) + index_of_turn = resume.index("--turn-instance-id") + resume[index_of_turn:index_of_turn + 2] = ["--resume-turn-key", key] + code, replay = _cli([*resume, "--retry-failed-turn", "--execute"]) + assert code == 1 and replay["recovery_decision"]["reason"] == "host_outcome_unknown", replay + assert replay["recovery_decision"]["reinvoke_host"] is False + assert native.read_text().splitlines() == ["native"] and len(turns.read_text().splitlines()) == 1 + assert path.read_bytes() == saved and capture.read_bytes() == original_capture + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_expired_lease_cannot_publish_backpressured_native_frame(tmp_path, monkeypatch, provider): from loopx.extensions import process_runtime From 954ec7c9cd47e3bf9622bd78d2056963dc313e66 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:32:52 +0800 Subject: [PATCH 07/11] Keep reviewer protocol links readable by the docs publisher (cherry picked from commit 8cd1d8d8699b1d1007b02989d9c97e7a1c9e880a) Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/issue_fix/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/loopx/capabilities/issue_fix/README.md b/loopx/capabilities/issue_fix/README.md index 3338cccdbe..04f1ce376e 100644 --- a/loopx/capabilities/issue_fix/README.md +++ b/loopx/capabilities/issue_fix/README.md @@ -345,14 +345,13 @@ candidate so a maintainer can audit why that person was selected. `CODEOWNERS` remains the strongest repository-native signal. Commit volume is only evidence of familiarity; it is not proof of maintainership, availability, -or review authority. See the [reviewer recommendation -contract](docs/protocols/issue-fix-reviewer-recommendation-v0.md) for scoring, -identity, and future-signal details, and the [reviewer request -contract](docs/protocols/issue-fix-reviewer-request-v0.md) for the external-write, +or review authority. See the +[reviewer recommendation contract](docs/protocols/issue-fix-reviewer-recommendation-v0.md) for +scoring, identity, and future-signal details, and the +[reviewer request contract](docs/protocols/issue-fix-reviewer-request-v0.md) for the external-write, idempotency, and verification rules. Secondary delivery, dedicated-bot isolation, local-private identity mapping, and readback are defined by the -[reviewer notification sink -contract](docs/protocols/issue-fix-reviewer-notification-sinks-v0.md). +[reviewer notification sink contract](docs/protocols/issue-fix-reviewer-notification-sinks-v0.md). ### 6. PR publication and public-write boundary From b589d5d4841f0ef43ebb1c6ad60ff187dc69731f Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:22:07 +0800 Subject: [PATCH 08/11] Reconcile a durable native Host attempt before starting another Turn Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/reference/protocols/turn-envelope-v0.md | 12 +- loopx/cli_commands/turn_run_once.py | 8 +- .../control_plane/quota/heartbeat_receipt.py | 29 ++++ .../quota/prior_turn_closeout.ts | 14 +- .../quota/unsettled_host_turn_recovery.ts | 53 ++++++- loopx/control_plane/turn_driver/executor.py | 5 + .../turn_driver/turn_journal_query.ts | 79 +++++++---- .../test_codex_operation_launch.py | 24 +++- tests/test_native_steward_turn_host.py | 130 ++++++++++++++++++ 9 files changed, 315 insertions(+), 39 deletions(-) diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index 87c9ebfd11..fc7fdafe68 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -500,8 +500,12 @@ by the pipe. A proven pre-dispatch expiry returns `goal_launch_not_allowed` with no Host invocation, writeback or debit. If a complete frame may have been delivered after expiry, the sender retires the receiver and the original attempt stays in progress with `host_outcome_unknown`. Recovery holds retry -pending reconciliation of that attempt. Registry contention can refuse dispatch -with `goal_authority_unavailable`. +pending reconciliation of that attempt. A Todo-bound native admission links +its original receipt to the journal after the Host attempt is durable and +before dispatch. Fresh admission returns the original Turn key for reconciliation +if that attempt is unconfirmed or its linked journal is missing. An admission +that stops before recording an attempt has no journal link. Registry contention +can refuse dispatch with `goal_authority_unavailable`. The journal retains the original capture, identity and saved session recovery for retry or readback. @@ -509,7 +513,9 @@ for retry or readback. `turn/start` 每次非阻塞写入前都检查时限,管道接受完整帧后再检查一次。 确认未派发即到期时,返回 `goal_launch_not_allowed`,不启动模型工作、不写回、 不扣额。完整帧可能在到期后送达时,发送方终止接收进程,原尝试保留在执行中并 -标为 `host_outcome_unknown`。恢复须先核对原尝试,才能再次派发。 +标为 `host_outcome_unknown`。绑定 Todo 的原生执行在尝试落盘后、派发前,将日志索引 +附到原准入回执。结果未确认或已绑定日志缺失时,新准入返回原 Turn 索引供核对,先核查 +原尝试再派发。止于尝试落盘前的准入没有绑定日志索引。 Use a fresh eligible Turn for changed work. Reading detail consumes no quota. Source admission, cancellation, workspace and mutation authority remain with their existing owners. diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index 9a8526f6b8..141586eb7d 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -846,7 +846,7 @@ def settlement_admission(): return request_settlement_lifetime(request_locator, settlement_identity, registry_path=registry_path, runtime_root=runtime_root, workspace=project) - def on_managed_start_admitted() -> None: + def ensure_start_receipt(turn_key: str | None = None) -> None: if settlement_identity.request_subject is not None: require_current_request_operation(request_locator, settlement_identity, registry_path=registry_path, runtime_root=runtime_root, workspace=project) @@ -868,6 +868,7 @@ def on_managed_start_admitted() -> None: else None ), goal_ref=goal_ref, + turn_key=turn_key, ) managed_cadence = managed_cadence_start( @@ -876,7 +877,7 @@ def on_managed_start_admitted() -> None: agent_id=args.agent_id, automation_id=args.automation_id, manual_reason=args.manual_interval_bypass_reason, - on_admitted=on_managed_start_admitted, + on_admitted=ensure_start_receipt, request_id=settlement_identity.effect_id if args.host == "codex-cli" else None, ) @@ -913,6 +914,9 @@ def on_managed_start_admitted() -> None: ) if args.execute else None), admit_start=managed_cadence.admit if args.execute else None, confirm_start=managed_cadence.confirm if args.execute else None, + record_host_attempt=ensure_start_receipt if args.execute and settlement_identity.todo_id and ( + args.host == "codex-app-server" or getattr(args, "codex_operation_tools", False) + ) else None, goal_admission=goal_admission, host_launch_guard=(partial( host_admission.launch_lifetime, diff --git a/loopx/control_plane/quota/heartbeat_receipt.py b/loopx/control_plane/quota/heartbeat_receipt.py index b228abd519..e0389ea962 100644 --- a/loopx/control_plane/quota/heartbeat_receipt.py +++ b/loopx/control_plane/quota/heartbeat_receipt.py @@ -11,6 +11,7 @@ load_rollout_events, rollout_event_log_path, ) +from ..content_digest import ENVELOPED_SHA256_PATTERN from ..todos.contract import normalize_todo_id, normalize_todo_replan_obligation_id from .effect_program import SettlementBindingKind, SettlementIdentity from .error_codes import HeartbeatReceiptIdentityConflictError @@ -208,6 +209,7 @@ def ensure_turn_heartbeat_settlement_receipt( quota_spend_source: str | None = None, goal_ref: Mapping[str, object] | None = None, before_bind: Callable[[], None] | None = None, + turn_key: str | None = None, ) -> dict[str, object]: """Idempotently bind a Turn-created quota guard to its settlement identity. @@ -215,6 +217,8 @@ def ensure_turn_heartbeat_settlement_receipt( obligation was selected. Persist that explicit empty selection so an obligation opened while the host is running cannot retroactively reject the admitted Turn. Old envelopes without the field stay legacy-unscoped. + A journal key links the already bound receipt to a durably recorded native + Host attempt. This revision preserves the original admission facts. """ normalized_semantic_replan_obligation_id = ( @@ -253,6 +257,31 @@ def ensure_turn_heartbeat_settlement_receipt( "Turn heartbeat receipt belongs to another settlement identity" ) previous_details = effective.get("details") if effective is not None else None + if turn_key is not None: + if (identity.binding_kind is not SettlementBindingKind.TODO + or ENVELOPED_SHA256_PATTERN.fullmatch(turn_key) is None + or effective is None or observed != expected or not isinstance(previous_details, Mapping)): + raise HeartbeatReceiptIdentityConflictError( + "Native Host attempt requires a valid journal key and the original bound receipt" + ) + previous_key = previous_details.get("turn_key") + if previous_key == turn_key: + return effective + if "turn_key" in previous_details: + raise HeartbeatReceiptIdentityConflictError( + "Turn heartbeat receipt already identifies another journal" + ) + source_event_id = str(effective["event_id"]) + linked = build_rollout_event( + goal_id=identity.goal_id, goal_ref=goal_ref, event_kind="quota_should_run", + agent_id=identity.agent_id, todo_id=identity.todo_id, run_id=identity.turn_instance_id, + status=str(effective["status"]), recorded_at=str(effective["recorded_at"]), + summary="Original Turn receipt linked to its durable Host attempt", + source_event_id=source_event_id, caused_by=source_event_id, + details={**previous_details, "turn_key": turn_key}, + ) + _append_rollout_event_line(log_path, linked) + return linked previous_source = ( previous_details.get("quota_spend_source") if observed is not None and isinstance(previous_details, Mapping) else None diff --git a/loopx/control_plane/quota/prior_turn_closeout.ts b/loopx/control_plane/quota/prior_turn_closeout.ts index 9f07b8416e..328a1c43f4 100644 --- a/loopx/control_plane/quota/prior_turn_closeout.ts +++ b/loopx/control_plane/quota/prior_turn_closeout.ts @@ -13,7 +13,9 @@ import { jsonObject } from "../runtime_decode.ts"; import { admitsAutonomousReplan, heartbeatReceiptBinding, + heartbeatReceiptDetails, heartbeatReceiptFactFromEvent, + optionalHeartbeatString, selectEffectiveHeartbeatReceipt, type HeartbeatReceiptBinding, type HeartbeatReceiptFact, @@ -32,11 +34,13 @@ export interface PriorHostTurnCloseoutCandidate extends JsonObject { settlement_effect_id: string | null; /** The effective guard admitted a replan under this Todo's settlement identity. */ admitted_todo_replan?: true; + unconfirmed_host_turn_key?: string; } interface CloseoutCandidateCheck { candidate: PriorHostTurnCloseoutCandidate; requireWritebackForRecovery: boolean; + turnKey: string | null; } function candidate( @@ -62,6 +66,8 @@ function candidate( * admission requires a verified outstanding writeback for this binding. * Explicit pending-closeout discovery can also find admitted replans before * writeback, whether their settlement identity is a Todo or an obligation. + * Admission preflight can also inspect the exact journal named by a native + * Todo guard when the receipts do not confirm closeout. * * Every prior Turn is identity-validated, so a conflicting receipt fails the * read instead of being hidden behind a newer Turn. Results are newest first @@ -78,7 +84,7 @@ export function selectCloseoutCandidates( goalId: string, agentId: string, excludeTurnInstanceId: string | null, - { includeUnwrittenReplans } = { includeUnwrittenReplans: false }, + { includeUnwrittenReplans = false, readTurnJournals = false } = {}, ): { candidates: CloseoutCandidateCheck[]; turnsValidated: number } { const perTurn = new Map(); const firstRecorded = new Map(); @@ -120,10 +126,12 @@ export function selectCloseoutCandidates( if (binding === null) continue; const admittedReplan = admitsAutonomousReplan(effective, fact); const admittedTodoReplan = admittedReplan && binding.binding_kind === "todo"; + const turnKey = readTurnJournals && binding.binding_kind === "todo" + ? optionalHeartbeatString(heartbeatReceiptDetails(effective).turn_key) : null; if (!fact.closeout_required && !admittedTodoReplan && - !(includeUnwrittenReplans && admittedReplan)) continue; + !(includeUnwrittenReplans && admittedReplan) && turnKey === null) continue; candidates.push({candidate: candidate(binding, turnInstanceId, fact, admittedTodoReplan), - requireWritebackForRecovery: !fact.closeout_required && !includeUnwrittenReplans}); + requireWritebackForRecovery: !fact.closeout_required && !includeUnwrittenReplans, turnKey}); } return {candidates, turnsValidated: validated}; } diff --git a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts index 8bb66871c0..f392b9c74b 100644 --- a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts +++ b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts @@ -14,8 +14,9 @@ * whose settlement already validates is answered by the preflight, so the * caller reads no bound facts for it. */ -import type { JsonObject } from "../effect_program.ts"; +import { settlementIdentityPayload, type JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; +import { turnJournalHasUnconfirmedHost } from "../turn_driver/turn_journal_query.ts"; import { jsonObject, requireJsonObject, @@ -242,13 +243,15 @@ async function preflightPriorHostTurnCloseoutForOwner( request.goal_id, request.agent_id, request.exclude_turn_instance_id, + {readTurnJournals: true}, ); // Resolve one snapshot on the first candidate. The empty path needs only // the guard log, and unwritten replans may all drop out after validation. let settlementSnapshot: Awaited> | undefined; let newestSettledTurn: string | null = null; let newestAcceptedCloseout: AcceptedCloseout = "validated_writeback_and_quota_spend"; - for (const {candidate: selected, requireWritebackForRecovery} of candidates) { + for (const {candidate, requireWritebackForRecovery, turnKey} of candidates) { + let selected = candidate; settlementSnapshot ??= await readQuotaSettlementSnapshot( request.runtime_root, request.goal_id, rolloutSnapshot, ); @@ -256,8 +259,7 @@ async function preflightPriorHostTurnCloseoutForOwner( settlementReadbackRequest(request, selected), settlementSnapshot, ); - const missingReceipts = missingCloseoutReceipts(readback, requireWritebackForRecovery); - if (missingReceipts === undefined) continue; + let missingReceipts = missingCloseoutReceipts(readback, requireWritebackForRecovery); if (missingReceipts === null) { // A newer settled Turn cannot hide an older missing closeout. Keep // scanning in persisted newest-first order until the first unsettled @@ -271,6 +273,19 @@ async function preflightPriorHostTurnCloseoutForOwner( } continue; } + if (turnKey !== null && await turnJournalHasUnconfirmedHost( + request.runtime_root, + turnKey, + settlementIdentityPayload({ + goal_id: request.goal_id, agent_id: request.agent_id, + ...candidateReadbackBinding(selected), + }), + request.owner_projection.goal_ref, + )) { + selected = {...selected, unconfirmed_host_turn_key: turnKey}; + missingReceipts = missingCloseoutReceipts(readback, false); + } + if (missingReceipts == null) continue; const monitorPoll = selected.binding_kind === "todo" ? committedMonitorPollFromSnapshot(settlementSnapshot, { goal_id: request.goal_id, agent_id: request.agent_id, @@ -335,6 +350,14 @@ function decodeCandidate(value: unknown): PriorHostTurnCloseoutCandidate { ); } const settlementEffectId = optionalHeartbeatString(raw.settlement_effect_id); + const unconfirmedHostTurnKey = optionalHeartbeatString(raw.unconfirmed_host_turn_key); + if (raw.unconfirmed_host_turn_key !== undefined && + (bindingKind !== "todo" || settlementEffectId === null || unconfirmedHostTurnKey === null || + !/^sha256:[a-f0-9]{64}$/u.test(unconfirmedHostTurnKey))) { + throw new EffectRuntimeRequestError( + "prior host Turn recovery candidate has an invalid unconfirmed dispatch locator", + ); + } if (raw.admitted_todo_replan !== undefined && (raw.admitted_todo_replan !== true || bindingKind !== "todo" || settlementEffectId === null)) { throw new EffectRuntimeRequestError( @@ -350,6 +373,7 @@ function decodeCandidate(value: unknown): PriorHostTurnCloseoutCandidate { binding_kind: bindingKind, binding_id: bindingId, settlement_effect_id: settlementEffectId, + ...(unconfirmedHostTurnKey === null ? {} : {unconfirmed_host_turn_key: unconfirmedHostTurnKey}), ...(raw.admitted_todo_replan === true ? {admitted_todo_replan: true as const} : {}), }; } @@ -554,7 +578,9 @@ export function reduceUnsettledHostTurnRecovery(value: unknown): JsonObject { } const replanSpendOwed = selected.admitted_todo_replan === true && awaitingQuotaSpend(missingReceipts); - const lifecycleCloseout = replanSpendOwed ? null : acceptedLifecycleCloseout(todo); + const unconfirmedHostTurnKey = selected.unconfirmed_host_turn_key; + const lifecycleCloseout = replanSpendOwed || unconfirmedHostTurnKey + ? null : acceptedLifecycleCloseout(todo); if (lifecycleCloseout !== null) { return { schema_version: UNSETTLED_HOST_TURN_RECOVERY_RESULT_SCHEMA, @@ -597,9 +623,22 @@ export function reduceUnsettledHostTurnRecovery(value: unknown): JsonObject { recovery.admission_checkpoint = checkpoint; recovery.admissions_without_progress = count; recovery.admission_limit = RECOVERY_ADMISSION_LIMIT; - const attention = count >= RECOVERY_ADMISSION_LIMIT; + const attention = unconfirmedHostTurnKey !== undefined || count >= RECOVERY_ADMISSION_LIMIT; const obligation = recoveryObligation(repair, selected.binding_id); - if (attention) { + if (unconfirmedHostTurnKey) { + recovery.state = "attention_required"; + recovery.reason_code = "host_outcome_unknown"; + recovery.turn_key = unconfirmedHostTurnKey; + Object.assign(obligation, { + must_attempt_work: false, + notify: "NOTIFY", + reason_code: "host_outcome_unknown", + reason: "an earlier Host dispatch has no confirmed outcome", + recommendation_reason: "the original Host dispatch needs reconciliation", + unsettled_reason: "an earlier Host dispatch remains unconfirmed", + recommended_action: "Inspect the original Turn journal. Restore it if unavailable, then reconcile the Host attempt before dispatching a fresh Turn.", + }); + } else if (attention) { recovery.state = "attention_required"; recovery.reason_code = "unsettled_recovery_admission_limit"; Object.assign(obligation, { diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 446c181fa3..d0302346fd 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -1016,6 +1016,7 @@ def _host_result_stage( persist_journal: JournalPersist, effects: dict[str, bool], confirm_start: Callable[[], None] | None = None, + record_host_attempt: Callable[[str], None] | None = None, usage_runtime_root: Path | None = None, usage_goal_id: str = "", host_launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, @@ -1039,6 +1040,8 @@ def _host_result_stage( # reservation. Confirmation failure stops before the host starts. if confirm_start is not None: confirm_start() + if record_host_attempt is not None: + record_host_attempt(str(journal["turn_key"])) from ...extensions.process_runtime import ProcessStdinDispatchUnconfirmed from ...usage_goal import observe_goal_execution @@ -1597,6 +1600,7 @@ def run_loopx_turn_once( settlement_admission: Callable[[], AbstractContextManager] | None = None, admit_start: Callable[[Mapping[str, Any]], dict[str, Any]] | None = None, confirm_start: Callable[[], None] | None = None, + record_host_attempt: Callable[[str], None] | None = None, goal_admission: FirstPartyHostGoalAdmission | None = None, host_launch_guard: Callable[[], AbstractContextManager[int | None]] | None = None, ) -> dict[str, Any]: @@ -1871,6 +1875,7 @@ def finish_recovery(payload: dict[str, Any]) -> dict[str, Any]: persist_journal=persist_journal, effects=effects, confirm_start=confirm_start if reserved_start else None, + record_host_attempt=record_host_attempt, host_launch_guard=host_launch_guard, ) if terminal is not None: diff --git a/loopx/control_plane/turn_driver/turn_journal_query.ts b/loopx/control_plane/turn_driver/turn_journal_query.ts index 6ddf11cdfd..8f18277d85 100644 --- a/loopx/control_plane/turn_driver/turn_journal_query.ts +++ b/loopx/control_plane/turn_driver/turn_journal_query.ts @@ -27,6 +27,7 @@ import { interface JournalEvidence { turn_key: string; + recovery_reason: string; observed_capabilities: string[] | null; } @@ -136,6 +137,7 @@ function evidenceForJournal( } return { turn_key: turnKey, + recovery_reason: inspection.recovery_decision.reason, observed_capabilities: inspection.replay_legal ? observedCapabilities(turnPlanWorkSource(plan ?? {}, sourceNumberTokens(journal, parsedSourceNumberTokens(journal, numbers)).filter(({path}) => path[0] === "plan" && path[1] === "work_contract") @@ -143,19 +145,65 @@ function evidenceForJournal( }; } -async function queryTurnJournal( - runtimeRootValue: unknown, - identity: JournalAddress | BoundSettlementIdentity, - goalRef: ExactGoalRef | null, -): Promise { +async function readJournalSnapshot(path: string): Promise> { + // Journals are atomically replaced. Read a complete snapshot without a + // reader lock or following a link to a different authority. + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + if (!(await handle.stat()).isFile()) journalConflict("Turn journal snapshot is not a regular file"); + try { + return parseJsonSource(new TextDecoder("utf-8", { fatal: true }).decode(await handle.readFile())); + } catch (error) { + if (error instanceof SyntaxError || error instanceof TypeError) { + journalConflict("Turn journal history is unreadable; reconcile it before retrying"); + } + throw error; + } + } finally { + await handle.close(); + } +} + +function turnJournalDirectory(runtimeRootValue: unknown, goalId: string): string { const runtimeRoot = requireNonEmptyString(runtimeRootValue, "runtime_root"); if (!isAbsolute(runtimeRoot) || runtimeRoot.includes("\0")) { throw new EffectRuntimeRequestError("Turn journal runtime_root must be an absolute path"); } - if ([".", ".."].includes(identity.goal_id) || /[/\\\0]/u.test(identity.goal_id)) { + if ([".", ".."].includes(goalId) || /[/\\\0]/u.test(goalId)) { throw new EffectRuntimeRequestError("Turn journal goal_id must be a single path segment"); } - const directory = join(runtimeRoot, "goals", identity.goal_id, "turns"); + return join(runtimeRoot, "goals", goalId, "turns"); +} + +/** An attempt linked by the bound receipt cannot be excluded by a missing journal. */ +export async function turnJournalHasUnconfirmedHost( + runtimeRoot: string, turnKey: string, settlementIdentity: JsonObject, goalRef: unknown, +): Promise { + if (!/^sha256:[a-f0-9]{64}$/u.test(turnKey)) { + journalConflict("The admitted Turn journal locator is invalid"); + } + const identity = decodeIdentity(settlementIdentity); + const directory = turnJournalDirectory(runtimeRoot, identity.goal_id); + let parsed; + try { + parsed = await readJournalSnapshot(join(directory, turnKey.slice("sha256:".length) + ".json")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; + throw error; + } + const evidence = evidenceForJournal( + parsed.value, turnKey, identity, goalRef === undefined ? null : decodeGoalRef(goalRef), parsed.numbers, + ); + if (!evidence) journalConflict("The admitted Turn journal belongs to another settlement identity"); + return evidence.recovery_reason === "host_outcome_unknown"; +} + +async function queryTurnJournal( + runtimeRootValue: unknown, + identity: JournalAddress | BoundSettlementIdentity, + goalRef: ExactGoalRef | null, +): Promise { + const directory = turnJournalDirectory(runtimeRootValue, identity.goal_id); let entries; try { entries = await readdir(directory, { withFileTypes: true }); @@ -167,22 +215,7 @@ async function queryTurnJournal( for (const entry of entries.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) { if (!/^[a-f0-9]{64}\.json$/u.test(entry.name)) continue; if (!entry.isFile()) journalConflict("Turn journal history contains a non-regular snapshot; reconcile it before retrying"); - let parsed: ReturnType; - // Journals are atomic-replace files. Read one complete version without - // creating reader lock files; do not follow a replaced symlink. This is - // not a snapshot of the directory or proof of provider-side absence. - const handle = await open(join(directory, entry.name), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); - try { - if (!(await handle.stat()).isFile()) journalConflict("Turn journal snapshot is not a regular file"); - try { - parsed = parseJsonSource(new TextDecoder("utf-8", { fatal: true }).decode(await handle.readFile())); - } catch (error) { - if (error instanceof SyntaxError || error instanceof TypeError) journalConflict("Turn journal history is unreadable; reconcile it before retrying"); - throw error; - } - } finally { - await handle.close(); - } + const parsed = await readJournalSnapshot(join(directory, entry.name)); const evidence = evidenceForJournal( parsed.value, `sha256:${entry.name.slice(0, -5)}`, diff --git a/tests/control_plane/test_codex_operation_launch.py b/tests/control_plane/test_codex_operation_launch.py index 93ea7959b2..cf34261f70 100644 --- a/tests/control_plane/test_codex_operation_launch.py +++ b/tests/control_plane/test_codex_operation_launch.py @@ -606,7 +606,7 @@ def delayed(chunk): assert count == 1 sent_at.append(time.time()) limit = time.monotonic() + 2 - while not turns.exists() and time.monotonic() < limit: + while not (turns.exists() and turns.stat().st_size) and time.monotonic() < limit: time.sleep(.005) return count @@ -642,6 +642,28 @@ def delayed(chunk): assert replay["recovery_decision"]["reinvoke_host"] is False assert native.read_text().splitlines() == ["native"] and len(turns.read_text().splitlines()) == 1 assert path.read_bytes() == saved and capture.read_bytes() == original_capture + guard_arguments = [ + *arguments[:6], "quota", "should-run", "--goal-id", "loopx-turn-fixture", + "--agent-id", "codex-fixture", + "--context-delivery", "off", "--delta-delivery", "off", + ] + code, guard = _cli([*guard_arguments, "--turn-instance-id", "next-after-unconfirmed-dispatch"]) + assert code == 0, guard + assert guard["should_run"] is False and guard["reason_code"] == "host_outcome_unknown", guard + assert guard["unsettled_host_turn_recovery"]["turn_key"] == key + moved = path.with_suffix(".relocated") + path.rename(moved) + try: + code, guard = _cli([*guard_arguments, "--turn-instance-id", "next-after-original-journal-lost"]) + assert code == 0, guard + assert guard["should_run"] is False and guard["reason_code"] == "host_outcome_unknown", guard + assert guard["unsettled_host_turn_recovery"]["turn_key"] == key + assert not path.exists() + finally: + moved.rename(path) + assert native.read_text().splitlines() == ["native"] and len(turns.read_text().splitlines()) == 1 + assert not index.exists() or "quota_slot_spent" not in index.read_text() + assert path.read_bytes() == saved and capture.read_bytes() == original_capture @pytest.mark.parametrize("provider", ["file", "sqlite"]) diff --git a/tests/test_native_steward_turn_host.py b/tests/test_native_steward_turn_host.py index 3c9b94bc8e..38da8bb668 100644 --- a/tests/test_native_steward_turn_host.py +++ b/tests/test_native_steward_turn_host.py @@ -1169,6 +1169,136 @@ def send_after_expiry(process, frame, **kwargs): assert not index.exists() or "quota_slot_spent" not in index.read_text() +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("journal_available", [True, False], ids=["journal-present", "journal-missing"]) +def test_managed_host_reopen_with_new_request_reconciles_unconfirmed_send( + steward, tmp_path, monkeypatch, provider, journal_available, +): + import time + from types import SimpleNamespace + + from loopx.control_plane.turn_driver.journal_store import turn_journal_path + from loopx.extensions import process_runtime + from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + service, session, actor, todo_id, _, wire = managed_fixture(steward, tmp_path, monkeypatch, provider) + _, runtime, *_ = steward + sid = session["session_id"] + before = wire.stat().st_size + send_frame = process_runtime.write_process_stdin_before_deadline + pipe_writer = process_runtime._nonblocking_pipe_writer + + def expire_during_send(process, frame, **kwargs): + if json.loads(frame).get("method") != "turn/start": + return send_frame(process, frame, **kwargs) + expires = kwargs["lease_expires_at_ms"] + late = False + assert expires > time.time() * 1000 + + def wall_time(): + return (expires + 1) / 1000 if late else time.time() + + @contextlib.contextmanager + def send_last_byte_after_expiry(fd): + with pipe_writer(fd) as write: + def send(chunk): + nonlocal late + if chunk[-1] != ord("\n"): + return write(chunk) + if len(chunk) > 1: + return write(chunk[:-1]) + assert write(chunk) == 1 + late = True + deadline = time.monotonic() + 3 + while time.monotonic() < deadline and b'"method": "turn/start"' not in wire.read_bytes()[before:]: + time.sleep(.005) + return 1 + yield send + + with monkeypatch.context() as patch: + patch.setattr(process_runtime, "time", SimpleNamespace( + time=wall_time, monotonic=time.monotonic, sleep=time.sleep)) + patch.setattr(process_runtime, "_nonblocking_pipe_writer", send_last_byte_after_expiry) + return send_frame(process, frame, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(process_runtime, "write_process_stdin_before_deadline", expire_during_send) + first, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="first-late-frame", todo_id=todo_id) + assert created + original = finish(runtime, first)["response"]["native_loopx"] + + assert original["status"] == "in_progress" and original["error_code"] == "host_outcome_unknown" + key = original["resume_turn_key"] + journal = turn_journal_path(runtime.coordination_runtime_root, goal_id=actor["goal_id"], turn_key=key) + retained_journal = journal.read_bytes() + capture = journal.parent / "decisions" / (key.removeprefix("sha256:") + ".json") + retained_capture = capture.read_bytes() + after_first = wire.read_bytes() + assert [json.loads(line)["method"] for line in after_first[before:].splitlines()].count("turn/start") == 1 + events = load_rollout_events(rollout_event_log_path(runtime.coordination_runtime_root, actor["goal_id"])) + linked, = [event for event in events if event.get("event_kind") == "quota_should_run" + and event.get("details", {}).get("turn_key") == key] + source, = [event for event in events if event["event_id"] == linked["causality"]["source_event_id"]] + assert linked["recorded_at"] == source["recorded_at"] and linked["status"] == source["status"] + assert linked["details"] == {**source["details"], "turn_key": key} + + adapter = runtime.adapters.pop(sid, None) + if adapter is not None: + adapter.close_session() + moved = journal.with_suffix(".relocated") + if not journal_available: + journal.rename(moved) + try: + second, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="new-id-after-reopen", todo_id=todo_id) + assert created + effects = finish(runtime, second)["response"]["native_loopx"]["effects"] + replayed, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="first-late-frame", todo_id=todo_id) + assert not created and replayed["turn_id"] == first["turn_id"] + finally: + if not journal_available: + assert not journal.exists() + moved.rename(journal) + assert not any(effects.values()) + assert all(json.loads(line)["method"] != "turn/start" for line in wire.read_bytes()[len(after_first):].splitlines()) + assert journal.read_bytes() == retained_journal and capture.read_bytes() == retained_capture + assert not cli(runtime, "todo", "list", "--goal-id", actor["goal_id"], "--todo-id", todo_id)["todo"]["done"] + index = runtime.coordination_runtime_root / "goals" / actor["goal_id"] / "runs/index.jsonl" + assert not index.is_file() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_native_receipt_without_durable_attempt_allows_fresh_request(steward, tmp_path, monkeypatch, provider): + from loopx.control_plane.turn_driver import executor + from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + service, session, actor, todo_id, _, wire = managed_fixture(steward, tmp_path, monkeypatch, provider) + _, runtime, *_ = steward + sid = session["session_id"] + before = wire.stat().st_size + + def fail_before_first_checkpoint(path, snapshot, **kwargs): + assert snapshot["host_attempt_count"] == 0 + raise OSError("fixture could not persist the first journal checkpoint") + + with monkeypatch.context() as patch: + patch.setattr(executor, "_write_journal", fail_before_first_checkpoint) + first, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="unrecorded-first", todo_id=todo_id) + assert created + finish(runtime, first) + after_first = wire.read_bytes() + assert all(json.loads(line)["method"] != "turn/start" for line in after_first[before:].splitlines()) + events = load_rollout_events(rollout_event_log_path(runtime.coordination_runtime_root, actor["goal_id"])) + admitted = [event for event in events if event.get("event_kind") == "quota_should_run" + and event.get("todo_id") == todo_id] + assert admitted and all("turn_key" not in event["details"] for event in admitted) + + second, created = service.submit(sid, agent_id=actor["agent_id"], operation_id="new-id-after-no-attempt", todo_id=todo_id) + assert created + effects = finish(runtime, second)["response"]["native_loopx"]["effects"] + assert effects["host_invoked"] is True + assert [json.loads(line)["method"] for line in wire.read_bytes()[len(after_first):].splitlines()].count("turn/start") == 1 + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_managed_host_does_not_borrow_another_execution_lease(steward, tmp_path, monkeypatch, provider): service, session, actor, todo_id, _, capture = managed_fixture(steward, tmp_path, monkeypatch, provider) From 8e6f9805a618db975a00cc6cab5f0f7f04278379 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:17:56 +0800 Subject: [PATCH 09/11] Keep Benchmark provider dispatch bound to its original lease Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- benchmark/runtime/RUNTIME.md | 8 +- .../runtime/provider_request_admission.py | 8 +- .../tests/test_provider_request_admission.py | 147 +++++++++++++++++- .../benchmark_toolkit/provider_gateway.py | 11 +- loopx/cli_commands/turn_decision.py | 7 +- .../goals/first_party_host_runtime.ts | 23 ++- .../first_party_host_runtime.test.ts | 18 +++ 7 files changed, 207 insertions(+), 15 deletions(-) diff --git a/benchmark/runtime/RUNTIME.md b/benchmark/runtime/RUNTIME.md index 91ae2b658d..bae1b45f6b 100644 --- a/benchmark/runtime/RUNTIME.md +++ b/benchmark/runtime/RUNTIME.md @@ -599,9 +599,11 @@ process or admission identity. The request check reuses the existing Stop, quota and canonical admission owners; it does not grant additional work or invent a second quota counter. Inbound parsing, upstream connection setup and waiting for the response do not hold the Goal/registry lock. The admitted POST dispatch has -an absolute two-second deadline, shortened by a smaller remaining phase budget. -An uncertain or partially sent request remains unknown and is not retried by -this transport. +an absolute two-second deadline, shortened by the remaining phase budget or the +active hard lease. A bound checkpoint request holds its original work lease +fence during dispatch. A final write returning after the deadline leaves the +original attempt unknown even if the provider received the body. A request with +an expired or replaced lease for the original work is refused. Per-invocation receipts distinguish requests started, bytes fully sent, pending responses and unknown partial sends. `sent` does not certify provider acceptance diff --git a/benchmark/runtime/provider_request_admission.py b/benchmark/runtime/provider_request_admission.py index e67ce438d8..b3250758cf 100644 --- a/benchmark/runtime/provider_request_admission.py +++ b/benchmark/runtime/provider_request_admission.py @@ -15,6 +15,7 @@ import uuid from contextlib import contextmanager from dataclasses import dataclass +from pathlib import Path from types import SimpleNamespace from urllib.parse import urlsplit from urllib.request import proxy_bypass_environment @@ -59,6 +60,11 @@ def __init__(self, env, session_wake, receipt, wake, *, timeout=None, original_q captured_session = SimpleNamespace(goal_ref=copy.deepcopy(session_wake.goal_ref)) check = heartbeat_launch_check(captured_env, captured_session, original_quota=copy.deepcopy(original_quota)) + preparation = original_quota.get("execution_preparation") if original_quota is not None else None + launch_scope = {} + if isinstance(preparation, dict) and preparation.get("handoff_mode") == "hard_lease": + # Keep the stored work's canonical lease fence through actual POST dispatch. + launch_scope["work_runtime_root"] = Path(captured_env["LOOPX_RUNTIME_ROOT"]) deadline = math.inf if timeout is not None: if not math.isfinite(timeout) or timeout < 0: @@ -72,7 +78,7 @@ def __init__(self, env, session_wake, receipt, wake, *, timeout=None, original_q deadline = min(deadline, time.monotonic() + epoch - reserve - time.time()) self._invocation = _Invocation( - session_wake.admission.with_launch_decision(check), captured_env["LOOPX_TURN"], + session_wake.admission.with_launch_decision(check, **launch_scope), captured_env["LOOPX_TURN"], session_wake.session_id, deadline) self._env = captured_env self._process = None diff --git a/benchmark/tests/test_provider_request_admission.py b/benchmark/tests/test_provider_request_admission.py index 5325dd23bd..b393bff5d2 100644 --- a/benchmark/tests/test_provider_request_admission.py +++ b/benchmark/tests/test_provider_request_admission.py @@ -28,9 +28,12 @@ request_admission_enabled, ) from loopx.capabilities.benchmark_toolkit import provider_gateway +from loopx.control_plane.coordination.shadow_management import shadow_maintenance_lock_target from loopx.control_plane.goals import activation_service, first_party_host_admission from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission from loopx.control_plane.projects.registry_codec import project_registry_transaction +from loopx.control_plane.work_items.task_lease import lease_expires_at +from loopx.file_lock import LockAcquireTimeoutError, exclusive_mutation_file_lock from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime from tests.control_plane import test_quota_settlement_cli as quota_fixture @@ -44,7 +47,7 @@ class _Authority: api_key: str = "fixture-provider-key" def invocation(self, upstream, name, *, timeout=None, receipt=None, wake=None, turn_id=None, - extra_env=None): + extra_env=None, original_quota=None): if wake is None: wake = self.root / name wake.mkdir() @@ -68,7 +71,9 @@ def invocation(self, upstream, name, *, timeout=None, receipt=None, wake=None, t ) if receipt is None: receipt = {} - guard = ProviderRequestAdmission(env, session, receipt, wake, timeout=timeout) + guard = ProviderRequestAdmission( + env, session, receipt, wake, timeout=timeout, original_quota=original_quota, + ) return guard, receipt, wake def stop(self): @@ -463,6 +468,144 @@ def test_real_backpressure_bounds_writer_wait_and_records_unknown_acceptance(aut assert len(upstream.rows) == 1, "unknown acceptance must not retry the POST" +@pytest.fixture(params=["file", "sqlite"]) +def held_provider_task(tmp_path, monkeypatch, request): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = quota_fixture._write_fixture(tmp_path) + quota_fixture._initialize_fixture_authority( + project, runtime, provider=request.param, handoff_mode="hard_lease", + ) + authority = _Authority(project, runtime, registry, tmp_path) + goal, agent, todo, turn = quota_fixture.GOAL_ID, quota_fixture.AGENT_ID, quota_fixture.TODO_ID, quota_fixture.TURN_ID + lease_scope = ["--goal-id", goal, "--todo-id", todo, "--owner", agent] + + def lease(action, version, ttl=None, *, key="provider-original-lease"): + ttl_args = [] if ttl is None else ["--ttl-seconds", str(ttl)] + code, result = quota_fixture._run_cli( + registry, runtime, "task-lease", action, *lease_scope, + "--idempotency-key", key, "--expected-version", str(version), *ttl_args, cwd=project, + ) + assert code == 0 and result["ok"], result + return result["lease"] + + original = lease("acquire", 0, 60) + code, quota = quota_fixture._run_cli( + registry, runtime, "quota", "should-run", "--goal-id", goal, "--agent-id", agent, + "--todo-id", todo, "--turn-instance-id", turn, "--runtime-profile", "generic_cli", + "--complete-turn", cwd=project, + ) + assert code == 0 and quota["should_run"], quota + assert quota["interaction_contract"]["cli_channel"]["execution_preparation"]["handoff_mode"] == "hard_lease" + return authority, quota, original, lease + + +@pytest.mark.parametrize("retry_lease", ["expired", "replacement"]) +def test_final_provider_write_crossing_hard_lease_preserves_unknown_attempt( + held_provider_task, monkeypatch, retry_lease, +): + authority, quota, original, lease = held_provider_task + turn = quota_fixture.TURN_ID + suffix = b"final-provider-body" + body = b"x" * (64 * 1024) + suffix + accepted_write_at = [] + original_connection = provider_gateway._upstream_connection + + def delay_final_write(*args, **kwargs): + connection = original_connection(*args, **kwargs) + send = connection.send + + def send_after_preemption(data): + if accepted_write_at or not isinstance(data, memoryview) or data != suffix: + return send(data) + assert 0 < connection.sock.gettimeout() < 2, "the hard lease must shorten the gateway send limit" + now = time.time() + assert now < expires_at + time.sleep(expires_at - now + .03) + send(data) + accepted_write_at.append(time.time()) + + connection.send = send_after_preemption + return connection + + monkeypatch.setattr(provider_gateway, "_upstream_connection", delay_final_write) + with _upstream() as upstream, _process() as process: + guard, receipt, wake = authority.invocation( + upstream.base_url, "lease-expiry", turn_id=turn, original_quota=quota, + ) + with guard.serve(["native", "-"]): + guard.started(process) + renewed = lease("renew", original["version"], 2) + assert renewed["lease_epoch"] == original["lease_epoch"] + expires_at = lease_expires_at(renewed).timestamp() + status, error = _post(guard.base_url, body) + assert upstream.body.wait(5) and upstream.rows[0]["body"] == body + assert len(accepted_write_at) == 1 and accepted_write_at[0] >= expires_at + assert status == 502 and error["error"]["code"] == "provider_gateway_upstream_unavailable", error + event, = _finished(wake) + summary, = receipt["provider_request_invocations"] + assert event["turn_id"] == turn and event["invocation_id"] == summary["invocation_id"] + assert event["forwarded"] is None and "response_status" not in event and "refusal_code" not in event + assert summary["requests"] == summary["unknown"] == 1 and summary["pending"] == summary["sent"] == 0 + if retry_lease == "replacement": + replacement = lease("acquire", renewed["version"], 60, key="provider-replacement-lease") + assert replacement["lease_epoch"] > renewed["lease_epoch"] + status, denied = _post(guard.base_url, body) + assert status == 409 and denied["error"]["code"] == "goal_launch_not_allowed", denied + _, rejected = _finished(wake, count=2) + assert rejected["invocation_id"] == event["invocation_id"] and rejected["forwarded"] is False + assert len(upstream.rows) == 1 + assert summary["unknown"] == summary["refused"] == 1 and summary["sent"] == 0 + assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1 + assert quota_fixture._spend_run_count(authority.runtime) == 0 + + +def test_bound_provider_send_holds_lease_writer_until_dispatch(held_provider_task, monkeypatch): + authority, quota, original, lease = held_provider_task + waiting, resume = threading.Event(), threading.Event() + original_connection = provider_gateway._upstream_connection + + def connection_at_send(*args, **kwargs): + connection = original_connection(*args, **kwargs) + send = connection.send + + def pause_body(data): + if isinstance(data, memoryview): + waiting.set() + assert resume.wait(4), "test did not release the admitted POST" + return send(data) + + connection.send = pause_body + return connection + + monkeypatch.setattr(provider_gateway, "_upstream_connection", connection_at_send) + with _upstream(hold_response=True) as upstream, _process() as process: + guard, _, wake = authority.invocation( + upstream.base_url, "lease-release", turn_id=quota_fixture.TURN_ID, original_quota=quota, + ) + with guard.serve(["native", "-"]): + guard.started(process) + response = _async(lambda: _post(guard.base_url)) + try: + assert waiting.wait(5) and not upstream.body.is_set() + target = shadow_maintenance_lock_target(authority.runtime, quota_fixture.GOAL_ID) + with pytest.raises(LockAcquireTimeoutError): + with exclusive_mutation_file_lock(target, timeout_seconds=0): + pass + mutation = _async(lambda: lease("release", original["version"])) + resume.set() + released = mutation.result(timeout=5) + assert released["status"] == "released" + assert upstream.body.wait(5) and upstream.rows[0]["body"] == b"{}" + assert not response.done() + finally: + resume.set() + upstream.respond.set() + assert response.result(timeout=5)[0] == 200 + assert _finished(wake)[0]["forwarded"] is True + assert _post(guard.base_url)[0] == 409 + assert len(upstream.rows) == 1 and quota_fixture._spend_run_count(authority.runtime) == 0 + + def test_slow_response_does_not_hold_goal_writer(authority): with _upstream(hold_response=True) as upstream, _process() as process: guard, receipt, wake = authority.invocation(upstream.base_url, "slow-response") diff --git a/loopx/capabilities/benchmark_toolkit/provider_gateway.py b/loopx/capabilities/benchmark_toolkit/provider_gateway.py index 2c8b5d14cf..a0a2247c6e 100644 --- a/loopx/capabilities/benchmark_toolkit/provider_gateway.py +++ b/loopx/capabilities/benchmark_toolkit/provider_gateway.py @@ -126,20 +126,23 @@ def _dispatch_bounded(connection: http.client.HTTPConnection, path: str, body: b for key, value in headers.items(): connection.putheader(key, value) - def remaining() -> None: + def remaining_budget() -> float: budget = deadline - time.monotonic() if not math.isfinite(budget) or budget <= 0: raise TimeoutError("provider request dispatch deadline exceeded") - connected_socket.settimeout(budget) + return budget - remaining() + connected_socket.settimeout(remaining_budget()) connection.endheaders() # Recompute the absolute budget between writes; repeated successful writes # must not renew a socket timeout while a Goal writer is waiting. view = memoryview(body) for start in range(0, len(view), 64 * 1024): - remaining() + connected_socket.settimeout(remaining_budget()) connection.send(view[start:start + 64 * 1024]) + # The sender can be preempted after setting the last relative timeout. + # A late return may have delivered bytes, so leave the attempt unconfirmed. + remaining_budget() @contextmanager diff --git a/loopx/cli_commands/turn_decision.py b/loopx/cli_commands/turn_decision.py index 65a68c9db7..abf77a7f3d 100644 --- a/loopx/cli_commands/turn_decision.py +++ b/loopx/cli_commands/turn_decision.py @@ -495,9 +495,14 @@ def current_heartbeat_host_launch_decision( # Initial heartbeat/planning has not chosen an action yet. Its generic # prompt asks the Host to consume the current decision after launch. A # held continuation already has an action and cannot substitute another. - return {"current": current, "planned_effective_action": ( + launch = {"current": current, "planned_effective_action": ( original_quota.get("effective_action") if original_quota is not None else current.get("effective_action"))} + if selection.receipt_bound_todo_id is not None: + launch["receipt_bound_todo_id"] = selection.receipt_bound_todo_id + if original_quota is not None: + launch["planned_execution_preparation"] = original_quota.get("execution_preparation") + return launch def build_fresh_envelope_for_managed_step( diff --git a/loopx/control_plane/goals/first_party_host_runtime.ts b/loopx/control_plane/goals/first_party_host_runtime.ts index e2b655ccad..d654aafbb8 100644 --- a/loopx/control_plane/goals/first_party_host_runtime.ts +++ b/loopx/control_plane/goals/first_party_host_runtime.ts @@ -1,4 +1,5 @@ import type { JsonObject } from "../effect_program.ts"; +import { decodeTaskLeaseProof } from "../coordination/task_lease_proof.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; import { assertNever, jsonObject } from "../runtime_decode.ts"; import { BARE_SHA256_PATTERN } from "../content_digest.ts"; @@ -185,10 +186,24 @@ function launchDecisionRejection(facts: JsonObject): FirstPartyHostRuntimeDecisi // health, repair, replan or scheduler policy in the Host adapter. // A newly selected repair/replan is not permission to launch the old action. // An already-admitted health repair keeps the quota owner's exception. - return decision?.should_run === true && typeof decision.status_health_ok === "boolean" - && typeof launch?.planned_effective_action === "string" && !!launch.planned_effective_action - && decision.effective_action === launch.planned_effective_action - ? null : {kind: "reject", code: "goal_launch_not_allowed"}; + const rejected = {kind: "reject", code: "goal_launch_not_allowed"} as const; + if (decision?.should_run !== true || typeof decision.status_health_ok !== "boolean" + || typeof launch?.planned_effective_action !== "string" || !launch.planned_effective_action + || decision.effective_action !== launch.planned_effective_action) return rejected; + if (launch.receipt_bound_todo_id === undefined) return null; + + // A Todo-bound receipt requires the live preparation for that same Todo. + const preparation = jsonObject(decision.execution_preparation); + if (typeof launch.receipt_bound_todo_id !== "string" || + preparation?.todo_id !== launch.receipt_bound_todo_id || + preparation.delivery_allowed !== true) return rejected; + const planned = jsonObject(launch.planned_execution_preparation); + if (planned?.handoff_mode !== "hard_lease" || planned.state !== "ready") return null; + const original = decodeTaskLeaseProof(planned.lease_proof); + const current = decodeTaskLeaseProof(preparation.lease_proof); + return preparation.handoff_mode === "hard_lease" && preparation.state === "ready" && + original && current && original.idempotency_key === current.idempotency_key && + current.expected_version >= original.expected_version ? null : rejected; } /** diff --git a/tests/control_plane_ts/first_party_host_runtime.test.ts b/tests/control_plane_ts/first_party_host_runtime.test.ts index fd87ebe30e..ddbc2ea4df 100644 --- a/tests/control_plane_ts/first_party_host_runtime.test.ts +++ b/tests/control_plane_ts/first_party_host_runtime.test.ts @@ -103,6 +103,24 @@ test("legacy native launch consumes the current quota owner without weakening so planned_effective_action: "control_plane_health_repair", current: {should_run: true, status_health_ok: false, effective_action: "control_plane_health_repair"}, }}).kind, "reject"); + + const preparation = {todo_id: "todo-original", handoff_mode: "hard_lease", state: "ready", + delivery_allowed: true, lease_proof: {idempotency_key: "original-lease", expected_version: 2}}; + const boundLaunch = {...admitted, receipt_bound_todo_id: preparation.todo_id, + planned_execution_preparation: preparation}; + const withPreparation = (execution_preparation: unknown) => decideFirstPartyHostRuntime({ + ...legacy, launch_decision: {...boundLaunch, current: {...admitted.current, execution_preparation}}, + }); + assert.notEqual(withPreparation(preparation).kind, "reject"); + assert.notEqual(withPreparation({...preparation, + lease_proof: {...preparation.lease_proof, expected_version: 3}}).kind, "reject"); + for (const current of [null, {...preparation, state: "claim_required", delivery_allowed: false}, + {...preparation, todo_id: "todo-other"}, + {...preparation, handoff_mode: "soft_claim", state: "not_required"}, + {...preparation, lease_proof: {idempotency_key: "replacement-lease", expected_version: 3}}, + {...preparation, lease_proof: {...preparation.lease_proof, expected_version: 1}}]) { + assert.deepEqual(withPreparation(current), {kind: "reject", code: "goal_launch_not_allowed"}); + } assert.deepEqual(decideFirstPartyHostRuntime({...exact, planned_goal_ref: {goal_id: "release"}, launch_decision: admitted}), {kind: "reject", code: "goal_instance_id_missing"}); From ea9b0b647cc739aa1403b14b7e5e47e258b45b50 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:47:56 +0800 Subject: [PATCH 10/11] Keep Benchmark recovery on its original replan and soft claim Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../tests/test_provider_request_admission.py | 28 +++++++++++++++++++ loopx/cli_commands/turn_decision.py | 6 +++- .../goals/first_party_host_runtime.ts | 16 +++++++++-- .../quota/execution_preparation.py | 12 ++++++-- .../test_benchmark_checkpoint_admission.py | 19 ++++++++++--- .../first_party_host_runtime.test.ts | 27 ++++++++++++++++++ 6 files changed, 98 insertions(+), 10 deletions(-) diff --git a/benchmark/tests/test_provider_request_admission.py b/benchmark/tests/test_provider_request_admission.py index b393bff5d2..884a8b50bd 100644 --- a/benchmark/tests/test_provider_request_admission.py +++ b/benchmark/tests/test_provider_request_admission.py @@ -499,6 +499,34 @@ def lease(action, version, ttl=None, *, key="provider-original-lease"): return authority, quota, original, lease +def test_provider_request_keeps_original_soft_claim_turn(authority): + turn = quota_fixture.TURN_ID + with _upstream() as upstream, _process() as process: + guard, _, wake = authority.invocation(upstream.base_url, "soft-claim", turn_id=turn) + with guard.serve(["native", "-"]): + guard.started(process) + # The native session first binds work in its own Turn. The gateway + # was already created and must consume that same live authority. + code, quota = quota_fixture._run_cli( + authority.registry, authority.runtime, "quota", "should-run", + "--goal-id", quota_fixture.GOAL_ID, "--agent-id", quota_fixture.AGENT_ID, + "--todo-id", quota_fixture.TODO_ID, "--turn-instance-id", turn, + "--runtime-profile", "generic_cli", "--complete-turn", cwd=authority.project, + ) + identity = quota["heartbeat_receipt"]["settlement_identity"] + assert code == 0 and quota["normal_delivery_allowed"] is True + assert identity["todo_id"] == quota_fixture.TODO_ID + assert "execution_preparation" not in quota + + status, body = _post(guard.base_url) + assert status == 200, body + event, = _finished(wake) + assert event["turn_id"] == turn and event["forwarded"] is True + assert len(upstream.rows) == 1 + assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1 + assert quota_fixture._spend_run_count(authority.runtime) == 0 + + @pytest.mark.parametrize("retry_lease", ["expired", "replacement"]) def test_final_provider_write_crossing_hard_lease_preserves_unknown_attempt( held_provider_task, monkeypatch, retry_lease, diff --git a/loopx/cli_commands/turn_decision.py b/loopx/cli_commands/turn_decision.py index abf77a7f3d..f637a8cf30 100644 --- a/loopx/cli_commands/turn_decision.py +++ b/loopx/cli_commands/turn_decision.py @@ -473,7 +473,9 @@ def current_heartbeat_host_launch_decision( selection.receipt, turn_instance_id=original_turn, status="replayed", ) if (not held.get("event_id") or current_receipt.get("event_id") != held.get("event_id") - or current_receipt.get("settlement_identity") != held.get("settlement_identity")): + or current_receipt.get("settlement_identity") != held.get("settlement_identity") + or current_receipt.get("semantic_replan_obligation_id") + != held.get("semantic_replan_obligation_id")): raise FirstPartyHostRuntimeRejected("heartbeat_launch_original_binding_changed") current = build_live_quota_should_run_decision( collect_turn_status_payload(args, registry_path=registry_path, runtime_root_arg=str(runtime_root)), @@ -500,6 +502,8 @@ def current_heartbeat_host_launch_decision( else current.get("effective_action"))} if selection.receipt_bound_todo_id is not None: launch["receipt_bound_todo_id"] = selection.receipt_bound_todo_id + if selection.receipt_bound_replan_obligation_id is not None: + launch["receipt_bound_replan_obligation_id"] = selection.receipt_bound_replan_obligation_id if original_quota is not None: launch["planned_execution_preparation"] = original_quota.get("execution_preparation") return launch diff --git a/loopx/control_plane/goals/first_party_host_runtime.ts b/loopx/control_plane/goals/first_party_host_runtime.ts index d654aafbb8..04bd84ee37 100644 --- a/loopx/control_plane/goals/first_party_host_runtime.ts +++ b/loopx/control_plane/goals/first_party_host_runtime.ts @@ -1,6 +1,8 @@ import type { JsonObject } from "../effect_program.ts"; import { decodeTaskLeaseProof } from "../coordination/task_lease_proof.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; +import { EffectiveAction } from "../quota/effective_action.generated.ts"; +import { normalizeHeartbeatReplanObligationId } from "../quota/heartbeat_receipt_identity.ts"; import { assertNever, jsonObject } from "../runtime_decode.ts"; import { BARE_SHA256_PATTERN } from "../content_digest.ts"; import { @@ -191,11 +193,21 @@ function launchDecisionRejection(facts: JsonObject): FirstPartyHostRuntimeDecisi || typeof launch?.planned_effective_action !== "string" || !launch.planned_effective_action || decision.effective_action !== launch.planned_effective_action) return rejected; if (launch.receipt_bound_todo_id === undefined) return null; + const todoId = launch.receipt_bound_todo_id; + if (typeof todoId !== "string") return rejected; + + if (decision.effective_action === EffectiveAction.AUTONOMOUS_REPLAN_REQUIRED) { + // A replan can use a Todo for settlement while delivery is prohibited. + // Its continuation authority is the original semantic obligation. + const obligationId = normalizeHeartbeatReplanObligationId(launch.receipt_bound_replan_obligation_id); + return obligationId !== null && decision.normal_delivery_allowed === false && + jsonObject(decision.replan_action_packet)?.obligation_id === obligationId && + jsonObject(decision.selected_todo)?.todo_id === todoId ? null : rejected; + } // A Todo-bound receipt requires the live preparation for that same Todo. const preparation = jsonObject(decision.execution_preparation); - if (typeof launch.receipt_bound_todo_id !== "string" || - preparation?.todo_id !== launch.receipt_bound_todo_id || + if (preparation?.todo_id !== todoId || preparation.delivery_allowed !== true) return rejected; const planned = jsonObject(launch.planned_execution_preparation); if (planned?.handoff_mode !== "hard_lease" || planned.state !== "ready") return null; diff --git a/loopx/control_plane/quota/execution_preparation.py b/loopx/control_plane/quota/execution_preparation.py index 8972e553e9..988e1a68b6 100644 --- a/loopx/control_plane/quota/execution_preparation.py +++ b/loopx/control_plane/quota/execution_preparation.py @@ -163,10 +163,16 @@ def apply_selected_todo_execution_preparation( or preparation.get("state") == "authority_unavailable" ): return - if preparation is None or preparation.get("state") == "not_required": + if preparation is None: return - payload["execution_preparation"] = preparation state = preparation.get("state") + if state == "not_required": + # The live Host needs the canonical soft-claim result to distinguish + # permission from missing authority. Ordinary quota omits this detail. + if include_launch_lease_deadline: + payload["execution_preparation"] = preparation + return + payload["execution_preparation"] = preparation if state == "ready": return payload["normal_delivery_allowed"] = False @@ -196,7 +202,7 @@ def attach_interaction_execution_preparation( contract: dict[str, Any], payload: dict[str, Any], ) -> None: preparation = payload.get("execution_preparation") - if not isinstance(preparation, dict): + if not isinstance(preparation, dict) or preparation.get("state") == "not_required": return contract["agent_channel"]["execution_preparation"] = preparation contract["cli_channel"]["execution_preparation"] = preparation diff --git a/tests/control_plane/test_benchmark_checkpoint_admission.py b/tests/control_plane/test_benchmark_checkpoint_admission.py index a19613ab93..aeb32e0ceb 100644 --- a/tests/control_plane/test_benchmark_checkpoint_admission.py +++ b/tests/control_plane/test_benchmark_checkpoint_admission.py @@ -167,8 +167,11 @@ def test_checkpoint_resume_rechecks_current_source_with_original_guard(tmp_path, @pytest.mark.parametrize("provider", ["file", "sqlite"]) -def test_checkpoint_fresh_todo_binding_keeps_original_turn(tmp_path, monkeypatch, provider): - _run_checkpoint(tmp_path, monkeypatch, provider, refusal=None, todo_bound=True) +@pytest.mark.parametrize(("handoff_mode", "refusal"), [ + ("soft_claim", None), ("hard_lease", None), ("soft_claim", "original_replan_changed"), +]) +def test_checkpoint_fresh_todo_binding_keeps_original_turn(tmp_path, monkeypatch, provider, handoff_mode, refusal): + _run_checkpoint(tmp_path, monkeypatch, provider, refusal=refusal, todo_bound=True, handoff_mode=handoff_mode) @pytest.mark.parametrize("provider", ["file", "sqlite"]) @@ -206,10 +209,12 @@ def test_checkpoint_request_admission_creates_a_new_invocation_for_original_turn assert all(endpoint.split("=", 1)[1] not in home.read_text() for endpoint in endpoints) -def _run_checkpoint(tmp_path, monkeypatch, provider, *, refusal, todo_bound, unbound=False, workspace_change=False, work_bound=False, provider_url=None): +def _run_checkpoint(tmp_path, monkeypatch, provider, *, refusal, todo_bound, unbound=False, workspace_change=False, + work_bound=False, provider_url=None, handoff_mode="soft_claim"): unbound = unbound or refusal == "original_event_changed" env, project, runtime, source, state = _environment( - tmp_path, monkeypatch, provider, replan="todo" if todo_bound or unbound else not work_bound) + tmp_path, monkeypatch, provider, replan="todo" if todo_bound or unbound else not work_bound, + handoff_mode=handoff_mode) env["CODEX_BIN"] = str(write_python_host_fixture(project / "checkpoint-host", f"#!{sys.executable}\n" + _HOST)) env = process_environment(Path(env["LOOPX_CODEX_HOME"]), base=env) env.update(LOOPX_TURN=OUTER_TURN, FIXTURE_OUTER_TURN=OUTER_TURN, FIXTURE_HELD_TURN=HELD_TURN) @@ -257,6 +262,8 @@ def captured_then_changed(held, deadline): assert held["quota"]["heartbeat_receipt"]["settlement_binding_owed"] is True if todo_bound: assert held["quota"]["heartbeat_receipt"]["settlement_identity"]["todo_id"] == fixture.SELECTED_REPLAN_TODO_ID + assert held["quota"]["normal_delivery_allowed"] is False + assert "execution_preparation" not in held["quota"] env["FIXTURE_HELD_QUOTA"] = str(held["path"]) if workspace_change: _private_workspace(project) @@ -292,6 +299,9 @@ def captured_then_changed(held, deadline): **original_receipt["settlement_identity"], "effect_id": "different-original-effect"}}} elif refusal == "original_action_changed": held["quota"] = {**held["quota"], "effective_action": "skip"} + elif refusal == "original_replan_changed": + held["quota"] = {**held["quota"], "heartbeat_receipt": { + **held["quota"]["heartbeat_receipt"], "semantic_replan_obligation_id": "replan-1000000000000001"}} return captured monkeypatch.setattr(hook, "_capture", captured_then_changed) @@ -330,6 +340,7 @@ def captured_then_changed(held, deadline): "stop":"goal_stopped", "original_event_changed":"heartbeat_launch_original_binding_changed", "original_settlement_changed":"heartbeat_launch_original_binding_changed", "original_action_changed":"goal_launch_not_allowed", + "original_replan_changed":"heartbeat_launch_original_binding_changed", }[refusal] in str(error) assert not (project / "checkpoint-business.json").exists() assert fixture._heartbeat_receipt_events(runtime, HELD_TURN) == observations["events"] diff --git a/tests/control_plane_ts/first_party_host_runtime.test.ts b/tests/control_plane_ts/first_party_host_runtime.test.ts index ddbc2ea4df..fb84a2b1d3 100644 --- a/tests/control_plane_ts/first_party_host_runtime.test.ts +++ b/tests/control_plane_ts/first_party_host_runtime.test.ts @@ -126,6 +126,33 @@ test("legacy native launch consumes the current quota owner without weakening so {kind: "reject", code: "goal_instance_id_missing"}); }); +test("a Todo-bound replan launch requires the same non-delivery obligation", () => { + const witness = {path: "/fixture/registry.json", sha256: "a".repeat(64)}; + const legacy = {profile_id: null, operation: "launch", goal_id: "release", + registry_path: witness.path, planned_registry_witness: witness, registry_witness: witness, + authority: {kind: "present", goal_id: "release", registration_status: "active", activation_state: "active"}}; + const todo = "todo_chain_000000000000"; + const obligation = "replan-1000000000000001"; + const current = { + effective_action: "autonomous_replan_required", should_run: true, + status_health_ok: true, normal_delivery_allowed: false, + selected_todo: {todo_id: todo}, replan_action_packet: {obligation_id: obligation}, + }; + const launch = {receipt_bound_todo_id: todo, + receipt_bound_replan_obligation_id: obligation, + planned_effective_action: current.effective_action, current}; + const decide = (launch_decision: unknown) => decideFirstPartyHostRuntime({...legacy, launch_decision}); + assert.deepEqual(decide(launch), {kind: "legacy"}); + for (const facts of [ + {...launch, receipt_bound_replan_obligation_id: undefined}, + {...launch, current: {...current, replan_action_packet: {obligation_id: "replan-1000000000000002"}}}, + {...launch, current: {...current, selected_todo: {todo_id: "todo_chain_000000000001"}}}, + {...launch, current: {...current, normal_delivery_allowed: true}}, + ]) { + assert.deepEqual(decide(facts), {kind: "reject", code: "goal_launch_not_allowed"}); + } +}); + test("a running legacy Turn requires its original authority and still sees Stop", () => { const entry = "a".repeat(64); const authority = {kind: "present", goal_id: "release", registration_status: "active", From 58c114bcbd743102123077531fccebc7d0608703 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:19:40 +0800 Subject: [PATCH 11/11] Serialize Benchmark worker spawn with canonical lease release Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- benchmark/runtime/worker.py | 6 +- .../tests/test_provider_request_admission.py | 55 +++++++++++++++++++ .../test_benchmark_current_admission.py | 45 ++++++++++++++- 3 files changed, 104 insertions(+), 2 deletions(-) diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index 238eff35b7..69865f6aef 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -145,7 +145,11 @@ def child_process(command: list[str], *, env: dict[str, str], stdout, stderr, if check_current_launch is not None and goal_admission is None: raise ValueError("current Host launch check requires its original Goal admission") if check_current_launch is not None: - goal_admission = goal_admission.with_launch_decision(check_current_launch) + # The live heartbeat can select hard-lease work even on a new Turn. + # Use its canonical writer fence for the decision and actual spawn. + goal_admission = goal_admission.with_launch_decision( + check_current_launch, work_runtime_root=Path(env["LOOPX_RUNTIME_ROOT"]), + ) # Session selection may precede rendering/capture. Serialize only actual # launch with the existing Goal lifetime owner, never the whole model run. with (goal_admission.launch_lifetime(operation="codex_agent_host_launch") diff --git a/benchmark/tests/test_provider_request_admission.py b/benchmark/tests/test_provider_request_admission.py index 884a8b50bd..942f1565d0 100644 --- a/benchmark/tests/test_provider_request_admission.py +++ b/benchmark/tests/test_provider_request_admission.py @@ -23,6 +23,7 @@ import pytest +from benchmark.runtime import worker from benchmark.runtime.provider_request_admission import ( ProviderRequestAdmission, request_admission_enabled, @@ -527,6 +528,60 @@ def test_provider_request_keeps_original_soft_claim_turn(authority): assert quota_fixture._spend_run_count(authority.runtime) == 0 +@pytest.mark.parametrize("released_before_recheck", [False, True]) +def test_bound_native_worker_serializes_original_lease_release(held_provider_task, monkeypatch, released_before_recheck): + authority, quota, original, lease = held_provider_task + turn = quota_fixture.TURN_ID + marker = authority.root / "checkpoint-input" + command = [sys.executable, "-c", + f"import sys; from pathlib import Path; Path({str(marker)!r}).write_text(sys.stdin.read())"] + env = { + "LOOPX_PROJECT": str(authority.project), "LOOPX_RUNTIME_ROOT": str(authority.runtime), + "LOOPX_GOAL_ID": quota_fixture.GOAL_ID, "LOOPX_AGENT_ID": quota_fixture.AGENT_ID, + "LOOPX_TURN": turn, + } + session = SimpleNamespace( + admission=FirstPartyHostGoalAdmission.for_plan( + registry_path=authority.registry, goal_id=quota_fixture.GOAL_ID, planned_goal_ref=None, + ), + goal_ref=None, + session_id="fixture-original", + ) + launch = { + "env": env, "stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL, + "session_wake": session, "receipt": {}, "wake": authority.root, "original_quota": quota, + } + real_popen = subprocess.Popen + mutations = [] + + def start_with_release(args, *extra, **kwargs): + if args == command: + target = shadow_maintenance_lock_target(authority.runtime, quota_fixture.GOAL_ID) + with pytest.raises(LockAcquireTimeoutError): + with exclusive_mutation_file_lock(target, timeout_seconds=0): + pytest.fail("the original lease could be released before the child starts") + mutations.append(_async(lambda: lease("release", original["version"]))) + return real_popen(args, *extra, **kwargs) + + monkeypatch.setattr(worker.subprocess, "Popen", start_with_release) + if released_before_recheck: + assert lease("release", original["version"])["status"] == "released" + with pytest.raises(first_party_host_admission.FirstPartyHostRuntimeRejected, + match="goal_launch_not_allowed"): + with worker.native_child_process(command, **launch): + pytest.fail("the released lease started another worker") + assert not marker.exists() and not mutations + else: + with worker.native_child_process(command, **launch) as process: + process.communicate(input="original checkpoint input", timeout=5) + assert process.returncode == 0 + assert marker.read_text() == "original checkpoint input" + released, = mutations + assert released.result(timeout=10)["status"] == "released" + assert quota_fixture._heartbeat_receipt_count(authority.runtime, turn) == 1 + assert quota_fixture._spend_run_count(authority.runtime) == 0 + + @pytest.mark.parametrize("retry_lease", ["expired", "replacement"]) def test_final_provider_write_crossing_hard_lease_preserves_unknown_attempt( held_provider_task, monkeypatch, retry_lease, diff --git a/tests/control_plane/test_benchmark_current_admission.py b/tests/control_plane/test_benchmark_current_admission.py index c0fddb575a..8728fe2bb8 100644 --- a/tests/control_plane/test_benchmark_current_admission.py +++ b/tests/control_plane/test_benchmark_current_admission.py @@ -1,11 +1,13 @@ """Real heartbeat Hosts consume current source eligibility before native launch.""" from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager import json from pathlib import Path +import subprocess import sys import time -from contextlib import contextmanager import pytest @@ -13,10 +15,12 @@ from benchmark.tests.test_shared_codex_runtime import worker_env from loopx.configure_goal import configure_goal from loopx.cli_commands import turn_decision +from loopx.control_plane.coordination.shadow_management import shadow_maintenance_lock_target from loopx.control_plane.goals import activation_service from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission, FirstPartyHostRuntimeRejected from loopx import contract, status as status_owner from loopx.cli import main +from loopx.file_lock import LockAcquireTimeoutError, exclusive_mutation_file_lock from loopx.global_registry import sync_project_registry_to_global from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime from tests.control_plane.host_process_fixture import write_python_host_fixture @@ -198,6 +202,45 @@ def wait_after_check(admission, *, operation): assert fixture._spend_run_count(runtime) == 0 +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_heartbeat_worker_holds_the_lease_writer_until_native_spawn(tmp_path, monkeypatch, provider): + env, project, runtime, source, _ = _environment(tmp_path, monkeypatch, provider, handoff_mode="hard_lease") + lease_args = ["--goal-id", fixture.GOAL_ID, "--todo-id", fixture.TODO_ID, + "--owner", fixture.AGENT_ID, "--idempotency-key", "benchmark-worker-spawn-lease"] + code, acquired = fixture._run_cli(source, runtime, "task-lease", "acquire", + *lease_args, "--expected-version", "0", "--ttl-seconds", "60", cwd=project) + assert code == 0 and acquired["ok"], acquired + real_popen = subprocess.Popen + mutations = [] + + def release(): + code, result = fixture._run_cli(source, runtime, "task-lease", "release", + *lease_args, "--expected-version", str(acquired["lease"]["version"]), cwd=project) + assert code == 0 and result["ok"], result + return result["lease"] + + with ThreadPoolExecutor(max_workers=1) as writer: + def start_with_release(args, *extra, **kwargs): + if isinstance(args, list) and args and args[0] == env["CODEX_BIN"]: + target = shadow_maintenance_lock_target(runtime, fixture.GOAL_ID) + with pytest.raises(LockAcquireTimeoutError): + with exclusive_mutation_file_lock(target, timeout_seconds=0): + pytest.fail("canonical lease writer could enter before Popen") + mutations.append(writer.submit(release)) + return real_popen(args, *extra, **kwargs) + + monkeypatch.setattr(worker.subprocess, "Popen", start_with_release) + result = worker.run_once(env) + assert result["ok"], result + released, = mutations + assert released.result(timeout=10)["status"] == "released" + + assert (project / "native-starts").read_text().splitlines() == ["native"] + assert (project / "model-starts").read_text().splitlines() == ["model"] + assert fixture._heartbeat_receipt_count(runtime, result["turn_id"]) == 0 + assert fixture._spend_run_count(runtime) == 0 + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize("mode", ["heartbeat", "turn"]) def test_initial_benchmark_planner_can_launch_without_a_prior_heartbeat_receipt(tmp_path, monkeypatch, provider, mode):