From 83ea68a998a6e1dc0300001bbd733f90711ba28c Mon Sep 17 00:00:00 2001 From: xiaobaifly7 <196511160+xiaobaifly7@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:14:22 +0800 Subject: [PATCH 1/4] fix(quota): bound unchanged prior-turn recovery admissions Signed-off-by: xiaobaifly7 <196511160+xiaobaifly7@users.noreply.github.com> --- loopx/control_plane/quota/settlement_cli.py | 5 + .../quota/unsettled_host_turn.py | 56 ++++++++--- .../quota/unsettled_host_turn_recovery.ts | 92 ++++++++++++++++++- loopx/semantics/vocabulary_v0.json | 2 +- ...test_unsettled_recovery_admission_bound.py | 81 ++++++++++++++++ .../unsettled_host_turn_recovery.test.ts | 53 +++++++++++ 6 files changed, 273 insertions(+), 16 deletions(-) create mode 100644 tests/control_plane/test_unsettled_recovery_admission_bound.py diff --git a/loopx/control_plane/quota/settlement_cli.py b/loopx/control_plane/quota/settlement_cli.py index c57e084a1f..77f179f741 100644 --- a/loopx/control_plane/quota/settlement_cli.py +++ b/loopx/control_plane/quota/settlement_cli.py @@ -403,6 +403,11 @@ def quota_rollout_details( # readiness and an absent timer remain explicit null facts here. details["admission_resume_when"] = selected_todo.get("resume_when") details["admission_resume_ready"] = selected_todo.get("resume_ready") + recovery = payload.get("unsettled_host_turn_recovery") + if isinstance(recovery, Mapping) and recovery.get("state") == "recovery_required": + checkpoint = recovery.get("admission_checkpoint") + if isinstance(checkpoint, Mapping): + details.update(checkpoint) replan_packet = payload.get("replan_action_packet") obligation = payload.get("autonomous_replan_obligation") if semantic_replan_obligation_id and isinstance(obligation, Mapping) and obligation.get("obligation_id") == semantic_replan_obligation_id: diff --git a/loopx/control_plane/quota/unsettled_host_turn.py b/loopx/control_plane/quota/unsettled_host_turn.py index 7eb53a7579..6da04e1fc0 100644 --- a/loopx/control_plane/quota/unsettled_host_turn.py +++ b/loopx/control_plane/quota/unsettled_host_turn.py @@ -20,10 +20,11 @@ effect_runtime_result, ) from ..scheduler.execution_context import SchedulerExecutionContextResolution +from ..agents.agent_scope_frontier import AgentScopeFrontierAction from ..todos.todo_semantics import todo_item_task_class from ..work_items.interaction_contract import ( _scoped_cli_args, - build_interaction_contract, + _build_interaction_contract_projection, ) from .error_codes import ( CloseoutQueryUnavailableError, @@ -171,7 +172,8 @@ def _prior_closeout_preflight( current_turn_instance_id: str | None, registry_path: Path | None = None, goal_ref: Mapping[str, Any] | None = None, -) -> tuple[dict[str, Any], list[str], dict[str, Any]] | None: + owner_action_id: str | None = None, +) -> tuple[dict[str, Any], list[str], dict[str, Any], dict[str, Any]] | None: """Ask the typed owner which prior Turn must still be closed out. The preflight reads the goal's persisted guards and the selected Turn's @@ -203,6 +205,7 @@ def _prior_closeout_preflight( "goal_id": goal_id, "agent_id": agent_id, "exclude_turn_instance_id": current_turn_instance_id, + "owner_action_id": owner_action_id, **( {"goal_ref": dict(goal_ref)} if goal_ref is not None @@ -253,6 +256,7 @@ def _prior_closeout_preflight( return ( dict(candidate), [str(name) for name in missing_receipts], dict(monitor_poll) if isinstance(monitor_poll, Mapping) else {}, + dict(result["recovery_admissions"]), ) @@ -264,6 +268,7 @@ def _unsettled_host_turn_recovery( agent_id: str | None, current_turn_instance_id: str | None, goal_ref: Mapping[str, Any] | None = None, + owner_action_id: str | None = None, ) -> dict[str, Any] | None: if not agent_id or not current_turn_instance_id: return None @@ -274,10 +279,11 @@ def _unsettled_host_turn_recovery( current_turn_instance_id=current_turn_instance_id, registry_path=registry_path, goal_ref=goal_ref, + owner_action_id=owner_action_id, ) if preflight is None: return None - selected, missing_receipts, monitor_poll = preflight + selected, missing_receipts, monitor_poll, recovery_admissions = preflight # A candidate carries exactly one binding: the Todo it must read, or the # autonomous replan obligation that has no Todo to read. todo_id = ( @@ -307,6 +313,7 @@ def _unsettled_host_turn_recovery( "candidate": selected, "missing_receipts": missing_receipts, "binding_facts": binding_facts, + "recovery_admissions": recovery_admissions, }, ) if not isinstance(verdict, Mapping) or ( @@ -316,7 +323,7 @@ def _unsettled_host_turn_recovery( status = verdict.get("status") if status == "none": return None - if status != "recovery_required": + if status not in ("recovery_required", "attention_required"): raise RuntimeError("TypeScript recovery result shape mismatch") recovery = verdict.get("recovery") obligation = verdict.get("obligation") @@ -343,6 +350,11 @@ def apply_unsettled_host_turn_recovery_if_required( ) -> bool: """Preempt ordinary selection when the preceding host Turn lacks closeout.""" + from ...agent_registry import load_goal_from_registry + + goal = load_goal_from_registry(registry_path, goal_id) or {} + activation = goal.get("activation") or {} + verdict = _unsettled_host_turn_recovery( registry_path=registry_path, runtime_root=runtime_root, @@ -350,6 +362,8 @@ def apply_unsettled_host_turn_recovery_if_required( agent_id=agent_id, current_turn_instance_id=current_turn_instance_id, goal_ref=goal_ref, + owner_action_id=(activation.get("updated_at") + if activation.get("actor_kind") == "owner" else None), ) if verdict is None: return False @@ -449,16 +463,21 @@ def _apply_recovery_projection( ) -> bool: recovery = verdict["recovery"] obligation = verdict["obligation"] + attention = recovery.get("state") == "attention_required" + if attention: + from .settlement_precedence import clear_quota_action_projections + + clear_quota_action_projections(payload) payload.pop("selected_todo", None) payload.pop("todo_id", None) payload.pop("action_portfolio", None) payload.update( { - "decision": "unsettled_host_turn_recovery", - "should_run": True, - "state": "eligible", + "decision": "blocked_wait" if attention else "unsettled_host_turn_recovery", + "should_run": obligation["must_attempt_work"], + "state": "waiting" if attention else "eligible", "effective_action": EffectiveAction.UNSETTLED_HOST_TURN_RECOVERY.value, - "actionable_by_codex": True, + "actionable_by_codex": obligation["must_attempt_work"], "normal_delivery_allowed": False, "recovery_delivery_allowed": False, "self_repair_allowed": False, @@ -468,6 +487,7 @@ def _apply_recovery_projection( "safe_bypass_kind": None, "safe_bypass_policy": None, "reason": obligation["reason"], + **({"reason_code": obligation["reason_code"], "requires_user_action": True} if attention else {}), "recommended_action": obligation["recommended_action"], "unsettled_host_turn_recovery": recovery, "heartbeat_recommendation": { @@ -476,10 +496,10 @@ def _apply_recovery_projection( "notify": obligation["notify"], "spend_policy": obligation["spend_policy"], "reason": obligation["recommendation_reason"], - "agent_must_attempt": True, + "agent_must_attempt": obligation["must_attempt_work"], }, "execution_obligation": { - "must_attempt_work": True, + "must_attempt_work": obligation["must_attempt_work"], "kind": "unsettled_host_turn_recovery", "contract": obligation["contract"], "contract_obligation": obligation["contract_obligation"], @@ -500,15 +520,18 @@ def _apply_recovery_projection( "automation_liveness": { "schema_version": "automation_liveness_v0", "keep_active": True, - "pause_allowed": False, - "automation_action": "execute_bounded_recovery", + "pause_allowed": attention, + "automation_action": "wait_for_owner" if attention else "execute_bounded_recovery", "reason": obligation["unsettled_reason"], "spend_policy": obligation["spend_policy"], }, } ) - interaction_contract = build_interaction_contract( + if attention: + payload["effective_action"] = EffectiveAction.BLOCKED_WAIT.value + interaction_contract, arbitration = _build_interaction_contract_projection( payload, + agent_scope_frontier_actions=[action.value for action in AgentScopeFrontierAction], available_capabilities=available_capabilities, scheduler_execution_context=scheduler_execution_context, turn_instance_id=current_turn_instance_id, @@ -526,4 +549,11 @@ def _apply_recovery_projection( cli_channel["spend_allowed_now"] = obligation.get("spend_allowed_now") is True cli_channel["recovery_ref"] = "$.unsettled_host_turn_recovery" payload["interaction_contract"] = interaction_contract + if attention: + from ..scheduler.scheduler_hint import build_scheduler_hint + + payload["scheduler_hint"] = build_scheduler_hint( + payload, arbitration=arbitration, scheduler_execution_context=scheduler_execution_context, + available_capabilities=available_capabilities, + ) return True diff --git a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts index 9cb4f9b55d..7adfff290c 100644 --- a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts +++ b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts @@ -62,6 +62,19 @@ export const UNSETTLED_HOST_TURN_RECOVERY_RESULT_SCHEMA = "loopx_prior_host_turn_recovery_result_v0"; export const UNSETTLED_HOST_TURN_RECOVERY_SCHEMA_VERSION = "unsettled_host_turn_recovery_v0"; +export const RECOVERY_ADMISSION_LIMIT = 3; +export const RECOVERY_ADMISSION_CHECKPOINT_SCHEMA = "unsettled_recovery_admission_v0"; + +export interface RecoveryAdmissionCheckpoint extends JsonObject { + recovery_checkpoint_schema: typeof RECOVERY_ADMISSION_CHECKPOINT_SCHEMA; + recovery_owed_turn_id: string; + recovery_binding_kind: PriorHostTurnCloseoutCandidate["binding_kind"]; + recovery_binding_id: string; + recovery_effect_id: string | null; + recovery_missing_writeback: boolean; + recovery_missing_spend: boolean; + recovery_owner_action_id: string | null; +} const MONITOR_TASK_CLASS = "continuous_monitor"; const SETTLED_LIFECYCLE_STATUSES = ["done", "blocked", "deferred"] as const; @@ -83,6 +96,7 @@ interface PreflightRequest { exclude_turn_instance_id: string | null; owner: QuotaAccountingOwner; owner_projection: JsonObject; + owner_action_id: string | null; } function decodePreflightRequest(value: unknown): PreflightRequest { @@ -107,6 +121,7 @@ function decodePreflightRequest(value: unknown): PreflightRequest { exclude_turn_instance_id: optionalHeartbeatString( request.exclude_turn_instance_id, ), + owner_action_id: optionalHeartbeatString(request.owner_action_id), owner, owner_projection: owner.kind === "alias" ? {} @@ -120,6 +135,53 @@ function decodePreflightRequest(value: unknown): PreflightRequest { }; } +function admissionCheckpoint( + selected: PriorHostTurnCloseoutCandidate, + missingReceipts: string[], + ownerActionId: string | null, +): RecoveryAdmissionCheckpoint { + return { + recovery_checkpoint_schema: RECOVERY_ADMISSION_CHECKPOINT_SCHEMA, + recovery_owed_turn_id: selected.prior_turn_instance_id, + recovery_binding_kind: selected.binding_kind, + recovery_binding_id: selected.binding_id, + recovery_effect_id: selected.settlement_effect_id, + recovery_missing_writeback: missingReceipts.includes(MISSING_RECEIPT_NAMES[0]), + recovery_missing_spend: missingReceipts.includes(MISSING_RECEIPT_NAMES[1]), + recovery_owner_action_id: ownerActionId, + }; +} + +function recoveryAdmissions( + receipts: readonly JsonObject[], + selected: PriorHostTurnCloseoutCandidate, + missingReceipts: string[], + request: PreflightRequest, +): JsonObject { + const checkpoint = admissionCheckpoint(selected, missingReceipts, request.owner_action_id); + const admittedTurns = new Set(); + for (const event of [...receipts].reverse()) { + const details = jsonObject(event.details); + if (details?.recovery_checkpoint_schema !== RECOVERY_ADMISSION_CHECKPOINT_SCHEMA || + details.should_run !== true || event.run_id === request.exclude_turn_instance_id) continue; + if (details.recovery_owed_turn_id !== selected.prior_turn_instance_id || + details.recovery_binding_kind !== selected.binding_kind || + details.recovery_binding_id !== selected.binding_id || + details.recovery_effect_id !== selected.settlement_effect_id) continue; + if (typeof details.recovery_missing_writeback !== "boolean" || + typeof details.recovery_missing_spend !== "boolean") { + throw new EffectRuntimeRequestError("recovery admission checkpoint has no missing-receipt snapshot"); + } + if (details.recovery_owner_action_id !== request.owner_action_id) break; + // Only receipt progress resets this debt; other owed identities and fresh + // host Turn ids cannot erase its unchanged admissions. + if ((details.recovery_missing_writeback && !checkpoint.recovery_missing_writeback) || + (details.recovery_missing_spend && !checkpoint.recovery_missing_spend)) break; + admittedTurns.add(requireNonEmptyString(event.run_id, "recovery admission run_id")); + } + return {count: admittedTurns.size, owner_action_id: request.owner_action_id}; +} + function settlementReadbackRequest( request: PreflightRequest, candidate: PriorHostTurnCloseoutCandidate, @@ -213,6 +275,7 @@ async function preflightPriorHostTurnCloseoutForOwner( turns_validated: turnsValidated, candidate: selected, missing_receipts: missingReceipts, + recovery_admissions: recoveryAdmissions(receipts ?? [], selected, missingReceipts, request), }; } return { @@ -498,11 +561,36 @@ export function reduceUnsettledHostTurnRecovery(value: unknown): JsonObject { // The typed repair lane lets the CLI renderer project its command list // without re-deriving which closeout family the Turn belongs to. recovery.repair = repair; + const admissions = jsonObject(request.recovery_admissions); + const count = admissions?.count ?? 0; + if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0) { + throw new EffectRuntimeRequestError("recovery admission count must be a nonnegative integer"); + } + const checkpoint = admissionCheckpoint(selected, missingReceipts, + optionalHeartbeatString(admissions?.owner_action_id)); + recovery.admission_checkpoint = checkpoint; + recovery.admissions_without_progress = count; + recovery.admission_limit = RECOVERY_ADMISSION_LIMIT; + const attention = count >= RECOVERY_ADMISSION_LIMIT; + const obligation = recoveryObligation(repair, selected.binding_id); + if (attention) { + recovery.state = "attention_required"; + recovery.reason_code = "unsettled_recovery_admission_limit"; + Object.assign(obligation, { + must_attempt_work: false, + notify: "NOTIFY", + reason_code: "unsettled_recovery_admission_limit", + reason: "prior host Turn recovery exhausted its unchanged admission allowance", + recommendation_reason: "recovery needs receipt progress or explicit owner resume", + unsettled_reason: "unchanged prior host Turn recovery requires attention", + recommended_action: "Repair the named original Turn's missing receipts, or explicitly stop and resume the Goal as owner to renew the recovery allowance; do not create a new debt or invent a closeout", + }); + } return { schema_version: UNSETTLED_HOST_TURN_RECOVERY_RESULT_SCHEMA, - status: "recovery_required", + status: attention ? "attention_required" : "recovery_required", prior_turn_instance_id: selected.prior_turn_instance_id, recovery, - obligation: recoveryObligation(repair, selected.binding_id), + obligation, }; } diff --git a/loopx/semantics/vocabulary_v0.json b/loopx/semantics/vocabulary_v0.json index 659f4a17c2..96719a1763 100644 --- a/loopx/semantics/vocabulary_v0.json +++ b/loopx/semantics/vocabulary_v0.json @@ -476,7 +476,7 @@ "terminal_no_followup": "The Goal is terminal with no follow-up. The controller reads it as the terminal_action partition, so with no prior receipt the disposition is terminal, and the scheduler stops.", "throttled_skip": "Existing result of quotaEffectiveAction; previously missed because the literal scan did not inspect declared return functions.", "todo_decision_scope_projection_repair": "A Todo user-gate scope is missing from the decision-scope projection; trigger user_gate_scope_projection_drift.", - "unsettled_host_turn_recovery": "A host Turn is unsettled and must be recovered before anything else: the selected Todo and action portfolio are dropped, should_run is set, and normal, recovery and self-repair delivery are all refused." + "unsettled_host_turn_recovery": "A host Turn is unsettled and must be recovered before anything else: the selected Todo and action portfolio are dropped, and normal, recovery and self-repair delivery are refused. Three unchanged admissions for the same owed identity exhaust the allowance; the typed recovery owner then projects blocked_wait until receipt progress or explicit owner resume, without erasing the original debt." }, "producers": [ "loopx/control_plane/work_items/action_selection_contract.py::build_action_selection_recovery_fields", diff --git a/tests/control_plane/test_unsettled_recovery_admission_bound.py b/tests/control_plane/test_unsettled_recovery_admission_bound.py new file mode 100644 index 0000000000..a07a3284b7 --- /dev/null +++ b/tests/control_plane/test_unsettled_recovery_admission_bound.py @@ -0,0 +1,81 @@ +"""Recovery admissions are bounded per original debt, never treated as spend.""" +from pathlib import Path + +import pytest + +from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from tests.control_plane.test_quota_settlement_cli import ( + AGENT_ID, GOAL_ID, TODO_ID, _run_cli, _spend_run_count, _write_fixture, +) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("reset", ["writeback", "owner_resume", "blocked_closeout"]) +def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, reset: str, +) -> None: + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path, provider=provider) + guard = ("quota", "should-run", "--codex-app", "--goal-id", GOAL_ID, + "--agent-id", AGENT_ID) + original = ("--turn-instance-id", "owed-original", "--todo-id", TODO_ID) + rc, first = _run_cli(registry, runtime, *guard, *original) + assert rc == 0 and first["heartbeat_receipt"]["closeout_required"], first + effect = first["heartbeat_receipt"]["settlement_identity"]["effect_id"] + for index in range(3): + rc, recovery = _run_cli(registry, runtime, *guard, + "--turn-instance-id", f"recovery-{index}") + assert rc == 0 and recovery["should_run"] is True, recovery + debt = recovery["unsettled_host_turn_recovery"] + assert debt["prior_turn_instance_id"] == "owed-original" + assert debt["settlement_effect_id"] == effect + assert debt["admissions_without_progress"] == index + # Repeating a committed recovery identity is not a fresh admission. + rc, replay = _run_cli(registry, runtime, *guard, "--turn-instance-id", "recovery-2") + assert rc == 0 and replay["should_run"] is True, replay + for turn in ("wait-1", "wait-2"): + rc, wait = _run_cli(registry, runtime, *guard, "--turn-instance-id", turn) + assert rc == 0 and wait["should_run"] is False, wait + assert wait["reason_code"] == "unsettled_recovery_admission_limit" + assert wait["unsettled_host_turn_recovery"]["state"] == "attention_required" + assert wait["unsettled_host_turn_recovery"]["admissions_without_progress"] == 3 + assert wait["scheduler_hint"]["action"] != "run_now" + channel = wait["interaction_contract"]["agent_channel"] + assert channel["must_attempt"] is False and channel["delivery_allowed"] is False + assert wait["interaction_contract"]["cli_channel"]["spend_allowed_now"] is False + assert _spend_run_count(runtime) == 0 + + if reset == "owner_resume": + for operation in ("stop", "resume"): + rc, result = _run_cli(registry, runtime, "goal-lifecycle", "--goal-id", GOAL_ID, + "--operation", operation, "--actor-kind", "owner", "--execute") + assert rc == 0, result + elif reset == "blocked_closeout": + rc, result = _run_cli(registry, runtime, "todo", "update", "--goal-id", GOAL_ID, + "--agent-id", AGENT_ID, "--todo-id", TODO_ID, + "--status", "blocked", "--reason", "Await independent evidence.") + assert rc == 0, result + else: + # The original identity remains available even after attention. A real + # writeback is progress; recovery grants themselves never fabricate it. + rc, reopened = _run_cli(registry, runtime, *guard, *original) + assert rc == 0 and reopened["should_run"] is True, reopened + assert reopened["heartbeat_receipt"]["settlement_identity"]["effect_id"] == effect + rc, result = _run_cli(registry, runtime, "refresh-state", "--goal-id", GOAL_ID, + "--agent-id", AGENT_ID, *original, + "--classification", "validated_work", "--delivery-batch-scale", "implementation", + "--delivery-outcome", "outcome_progress", "--delivery-boundary", "in_flight_continuation", + "--no-global-sync", "--suppress-external-sinks", cwd=project) + assert rc == 0, result + rc, after = _run_cli(registry, runtime, *guard, "--turn-instance-id", "after-progress") + assert rc == 0, after + if reset == "blocked_closeout": + assert after.get("unsettled_host_turn_recovery") is None, after + else: + debt = after["unsettled_host_turn_recovery"] + assert after["should_run"] is True and debt["admissions_without_progress"] == 0, after + assert debt["prior_turn_instance_id"] == "owed-original" + assert debt["settlement_effect_id"] == effect + if reset == "writeback": + assert debt["missing_receipts"] == ["quota_spend_receipt"] + assert _spend_run_count(runtime) == 0 diff --git a/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts b/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts index 8287c5fc62..5578c7bf78 100644 --- a/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts +++ b/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts @@ -16,6 +16,7 @@ import { } from "../../loopx/control_plane/effect_runtime_io.ts"; import { ACCEPTED_CLOSEOUTS, + RECOVERY_ADMISSION_LIMIT, PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, UNSETTLED_HOST_TURN_RECOVERY_REQUEST_SCHEMA, preflightPriorHostTurnCloseout, @@ -248,6 +249,46 @@ const ADVANCEMENT_OPEN = { cadence: null, }; +test("the admission bound keeps debt identity, deduplicates replays, and accepts owner resume and legal closeout", async () => { + const original = receipt("owed", closeoutRequired("owed", "todo_alpha")); + const runtime = await runtimeWith([original]); + try { + const candidate = candidateFrom(await preflight(runtime.root)); + const initial = reduce(candidate, undefined, READ_TODO(ADVANCEMENT_OPEN)); + const checkpoint = (initial.recovery as JsonObject).admission_checkpoint as JsonObject; + const admitted = (id: string, patch: JsonObject = {}) => receipt(id, + {...checkpoint, should_run: true, ...patch}); + const history = [original, + admitted("recovery-a"), admitted("recovery-a"), + admitted("other-debt", {recovery_owed_turn_id: "different"}), + admitted("recovery-b"), admitted("recovery-c")]; + const log = join(runtime.root, "goals", GOAL, "rollout-event-log.jsonl"); + await writeFile(log, history.map(event => JSON.stringify(event)).join("\n") + "\n"); + const observed = await preflight(runtime.root); + assert.deepEqual(observed.recovery_admissions, {count: RECOVERY_ADMISSION_LIMIT, owner_action_id: null}); + const decide = (facts: JsonObject, missing = observed.missing_receipts) => reduceUnsettledHostTurnRecovery({ + schema_version: UNSETTLED_HOST_TURN_RECOVERY_REQUEST_SCHEMA, + goal_id: GOAL, agent_id: AGENT, candidate, missing_receipts: missing, + recovery_admissions: observed.recovery_admissions, binding_facts: facts, + }); + const attention = decide(READ_TODO(ADVANCEMENT_OPEN)); + assert.equal(attention.status, "attention_required"); + assert.equal((attention.obligation as JsonObject).must_attempt_work, false); + assert.equal((attention.recovery as JsonObject).settlement_effect_id, candidate.settlement_effect_id); + assert.equal(decide(READ_TODO({...ADVANCEMENT_OPEN, status: "blocked"})).status, "none"); + const owner = await preflightPriorHostTurnCloseout({ + schema_version: PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, + runtime_root: runtime.root, goal_id: GOAL, agent_id: AGENT, + exclude_turn_instance_id: "current", owner_action_id: "explicit-owner-resume", + }); + assert.deepEqual(owner.recovery_admissions, {count: 0, owner_action_id: "explicit-owner-resume"}); + const replay = await preflight(runtime.root, "recovery-c"); + assert.equal((replay.recovery_admissions as JsonObject).count, 2); + } finally { + await runtime.close(); + } +}); + test("the preflight reads the persisted receipts and names the newest required closeout", async () => { const runtime = await runtimeWith([ receipt("turn-old", closeoutRequired("turn-old", "todo_older")), @@ -887,6 +928,18 @@ test("an unsettled Turn keeps the exact public recovery payload", async () => { binding_target_key: "key-1", binding_cadence: "1h", repair: "resume_prior_turn", + admissions_without_progress: 0, + admission_limit: 3, + admission_checkpoint: { + recovery_checkpoint_schema: "unsettled_recovery_admission_v0", + recovery_owed_turn_id: "turn-a", + recovery_binding_kind: "todo", + recovery_binding_id: "todo_alpha", + recovery_effect_id: `${GOAL}:${AGENT}:todo_alpha:turn-a`, + recovery_missing_writeback: true, + recovery_missing_spend: true, + recovery_owner_action_id: null, + }, }); const obligation = verdict.obligation as JsonObject; assert.equal(obligation.lane, "control_plane_recovery"); From 7498e2ae7e2feca54faa51288197fac6fc2fa38a Mon Sep 17 00:00:00 2001 From: xiaobaifly7 <196511160+xiaobaifly7@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:25:28 +0800 Subject: [PATCH 2/4] fix(quota): retain recovery epoch across controller activation Signed-off-by: xiaobaifly7 <196511160+xiaobaifly7@users.noreply.github.com> --- .../quota/unsettled_host_turn_recovery.ts | 10 +++++++-- ...test_unsettled_recovery_admission_bound.py | 15 ++++++++++--- .../unsettled_host_turn_recovery.test.ts | 22 +++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts index 7adfff290c..639a12e4e5 100644 --- a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts +++ b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts @@ -160,6 +160,8 @@ function recoveryAdmissions( ): JsonObject { const checkpoint = admissionCheckpoint(selected, missingReceipts, request.owner_action_id); const admittedTurns = new Set(); + let ownerActionId = request.owner_action_id; + let ownerEpochObserved = ownerActionId !== null; for (const event of [...receipts].reverse()) { const details = jsonObject(event.details); if (details?.recovery_checkpoint_schema !== RECOVERY_ADMISSION_CHECKPOINT_SCHEMA || @@ -172,14 +174,18 @@ function recoveryAdmissions( typeof details.recovery_missing_spend !== "boolean") { throw new EffectRuntimeRequestError("recovery admission checkpoint has no missing-receipt snapshot"); } - if (details.recovery_owner_action_id !== request.owner_action_id) break; + if (!ownerEpochObserved) { + ownerActionId = optionalHeartbeatString(details.recovery_owner_action_id); + ownerEpochObserved = true; + } + if (details.recovery_owner_action_id !== ownerActionId) break; // Only receipt progress resets this debt; other owed identities and fresh // host Turn ids cannot erase its unchanged admissions. if ((details.recovery_missing_writeback && !checkpoint.recovery_missing_writeback) || (details.recovery_missing_spend && !checkpoint.recovery_missing_spend)) break; admittedTurns.add(requireNonEmptyString(event.run_id, "recovery admission run_id")); } - return {count: admittedTurns.size, owner_action_id: request.owner_action_id}; + return {count: admittedTurns.size, owner_action_id: ownerActionId}; } function settlementReadbackRequest( diff --git a/tests/control_plane/test_unsettled_recovery_admission_bound.py b/tests/control_plane/test_unsettled_recovery_admission_bound.py index a07a3284b7..04d020e39d 100644 --- a/tests/control_plane/test_unsettled_recovery_admission_bound.py +++ b/tests/control_plane/test_unsettled_recovery_admission_bound.py @@ -10,12 +10,17 @@ @pytest.mark.parametrize("provider", ["file", "sqlite"]) -@pytest.mark.parametrize("reset", ["writeback", "owner_resume", "blocked_closeout"]) +@pytest.mark.parametrize("reset", ["writeback", "owner_resume", "blocked_closeout", "controller_resume"]) def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, reset: str, ) -> None: isolate_sqlite_runtime(tmp_path, monkeypatch) project, runtime, registry = _write_fixture(tmp_path, provider=provider) + if reset == "controller_resume": + for operation in ("stop", "resume"): + rc, result = _run_cli(registry, runtime, "goal-lifecycle", "--goal-id", GOAL_ID, + "--operation", operation, "--actor-kind", "owner", "--execute") + assert rc == 0, result guard = ("quota", "should-run", "--codex-app", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID) original = ("--turn-instance-id", "owed-original", "--todo-id", TODO_ID) @@ -45,10 +50,11 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( assert wait["interaction_contract"]["cli_channel"]["spend_allowed_now"] is False assert _spend_run_count(runtime) == 0 - if reset == "owner_resume": + if reset in ("owner_resume", "controller_resume"): for operation in ("stop", "resume"): rc, result = _run_cli(registry, runtime, "goal-lifecycle", "--goal-id", GOAL_ID, - "--operation", operation, "--actor-kind", "owner", "--execute") + "--operation", operation, "--actor-kind", + "owner" if reset == "owner_resume" else "controller", "--execute") assert rc == 0, result elif reset == "blocked_closeout": rc, result = _run_cli(registry, runtime, "todo", "update", "--goal-id", GOAL_ID, @@ -71,6 +77,9 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( assert rc == 0, after if reset == "blocked_closeout": assert after.get("unsettled_host_turn_recovery") is None, after + elif reset == "controller_resume": + assert after["should_run"] is False, after + assert after["unsettled_host_turn_recovery"]["admissions_without_progress"] == 3 else: debt = after["unsettled_host_turn_recovery"] assert after["should_run"] is True and debt["admissions_without_progress"] == 0, after diff --git a/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts b/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts index 5578c7bf78..26edb674ac 100644 --- a/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts +++ b/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts @@ -289,6 +289,28 @@ test("the admission bound keeps debt identity, deduplicates replays, and accepts } }); +test("losing an owner activation projection cannot renew its exhausted allowance", async () => { + const original = receipt("owed", closeoutRequired("owed", "todo_alpha")); + const runtime = await runtimeWith([original]); + try { + const candidate = candidateFrom(await preflight(runtime.root)); + const recovery = reduce(candidate, undefined, READ_TODO(ADVANCEMENT_OPEN)).recovery as JsonObject; + const checkpoint = recovery.admission_checkpoint as JsonObject; + const history = [original, ...["old-owner", "current-owner", "current-owner", "current-owner"].map( + (owner, index) => receipt(`admission-${index}`, {...checkpoint, + should_run: true, recovery_owner_action_id: owner}), + )]; + await writeFile(join(runtime.root, "goals", GOAL, "rollout-event-log.jsonl"), + history.map(event => JSON.stringify(event)).join("\n") + "\n"); + // Controller activation has no owner id. Keep the latest recorded owner + // epoch, without counting admissions before that owner's explicit reset. + assert.deepEqual((await preflight(runtime.root)).recovery_admissions, + {count: 3, owner_action_id: "current-owner"}); + } finally { + await runtime.close(); + } +}); + test("the preflight reads the persisted receipts and names the newest required closeout", async () => { const runtime = await runtimeWith([ receipt("turn-old", closeoutRequired("turn-old", "todo_older")), From 9a30e684d92f562425638806cb4b1c9da1fa659f Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:25:16 +0800 Subject: [PATCH 3/4] Retain the latest owner resume through controller lifecycle transitions Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/quota-allocation.md | 13 ++++ loopx/control_plane/goals/activation.py | 22 ++++++- .../control_plane/goals/activation_service.py | 23 ++++++- loopx/control_plane/quota/live_decision.py | 3 + .../quota/unsettled_host_turn.py | 9 +-- tests/control_plane/test_goal_activation.py | 21 +++++++ ...test_unsettled_recovery_admission_bound.py | 62 +++++++++++++++++++ 7 files changed, 143 insertions(+), 10 deletions(-) diff --git a/docs/quota-allocation.md b/docs/quota-allocation.md index 57d8ae49ec..88d9bc2069 100644 --- a/docs/quota-allocation.md +++ b/docs/quota-allocation.md @@ -226,6 +226,19 @@ instead sets `closeout_kind=typed_blocked_writeback_no_spend` and settles the Turn without a quota debit. Todo completion and Goal acceptance retain their separate checks in both cases. +Repeated recovery admissions for the same bound prior Turn are limited when +the missing receipts remain unchanged. The limit counts distinct committed +guard admissions. Replaying a guard or receiving a host-completed event leaves +that count as it was. Verified receipt progress or an explicit Owner resume +renews the allowance. The Goal activation record retains the last Owner +resume through later Controller Stop and Resume transitions. Resuming a +Goal that is already active leaves the recorded allowance in place. + +同一原始 Turn 的缺失回执没有减少时,恢复准入有次数上限,按已提交的不同 guard +准入计数。相同 guard 重放和宿主完成事件均保持原计数。验证到回执进展或 Owner +显式恢复 Goal 后重新计算。Goal 激活记录会保留最近一次 Owner 恢复,后续 +Controller 停止或恢复均沿用该记录。对已启用的 Goal 再次恢复会保留既有额度。 + An admitted advancement Todo can also become persistently `blocked` through its ordinary owner-authorized lifecycle. Its original Turn then offers the same blocked writeback command. The settlement owner freezes the current diff --git a/loopx/control_plane/goals/activation.py b/loopx/control_plane/goals/activation.py index 26cf1947dd..2a66dc971a 100644 --- a/loopx/control_plane/goals/activation.py +++ b/loopx/control_plane/goals/activation.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Mapping -from ..actor_identity import normalize_owner_controller_actor +from ..actor_identity import OwnerControllerActorKind, normalize_owner_controller_actor GOAL_ACTIVATION_SCHEMA_VERSION = "loopx_goal_activation_v1" @@ -84,5 +84,25 @@ def build_goal_activation( return activation +def goal_owner_resume_id(goal: Mapping[str, Any] | None) -> str | None: + """Read the last owner resume from the canonical activation record.""" + activation = goal.get("activation") if goal is not None else None + if not isinstance(activation, Mapping): + return None + resume_id = activation.get("owner_resume_id") + if isinstance(resume_id, str) and resume_id.strip(): + return resume_id.strip() + # Older records can prove an owner resume only when it is still the + # recorded transition. A controller transition carries no such proof. + if ( + activation.get("state") == GoalActivationState.ACTIVE.value + and activation.get("actor_kind") == OwnerControllerActorKind.OWNER.value + ): + updated_at = activation.get("updated_at") + if isinstance(updated_at, str): + return updated_at.strip() or None + return None + + def goal_is_stopped(goal: Mapping[str, Any] | None) -> bool: return goal_activation_state(goal) is GoalActivationState.STOPPED diff --git a/loopx/control_plane/goals/activation_service.py b/loopx/control_plane/goals/activation_service.py index 3f50def680..e1e8ffff8f 100644 --- a/loopx/control_plane/goals/activation_service.py +++ b/loopx/control_plane/goals/activation_service.py @@ -6,6 +6,7 @@ import hashlib from pathlib import Path from typing import Any +from uuid import uuid4 from ..projects.registry_codec import ( activation_upgrade_registry_transaction, @@ -20,12 +21,13 @@ from ...history import load_registry from ...registry import registry_goals from ...registry_writability import probe_registry_write_path -from ..actor_identity import normalize_owner_controller_actor +from ..actor_identity import OwnerControllerActorKind, normalize_owner_controller_actor from ..runtime.time import now_local_iso from .activation import ( GoalActivationState, build_goal_activation, goal_activation_state, + goal_owner_resume_id, normalize_goal_activation_state, ) from ..content_digest import BARE_SHA256_PATTERN @@ -231,12 +233,18 @@ def _readback( target_registry: Path, goal_id: str, expected_state: GoalActivationState, + expected_owner_resume_id: str | None, ) -> dict[str, Any]: source_goal = _goal(load_registry(source_registry), goal_id) target_goal = _goal(load_registry(target_registry), goal_id) source_state = goal_activation_state(source_goal) target_state = goal_activation_state(target_goal) - verified = source_state is expected_state and target_state is expected_state + verified = ( + source_state is expected_state + and target_state is expected_state + and goal_owner_resume_id(source_goal) == expected_owner_resume_id + and goal_owner_resume_id(target_goal) == expected_owner_resume_id + ) return { "schema_version": GOAL_ACTIVATION_READBACK_SCHEMA_VERSION, "status": "verified" if verified else "mismatch", @@ -461,7 +469,17 @@ def set_goal_activation_state( } ) return payload + owner_resume_id = goal_owner_resume_id(locked_goal) if changed: + # Preserve the owner resume across later lifecycle writes. + # Mint it inside the source lock for an actual owner resume. + if ( + target_state is GoalActivationState.ACTIVE + and actor is OwnerControllerActorKind.OWNER + ): + owner_resume_id = uuid4().hex + if owner_resume_id is not None: + proposed_activation["owner_resume_id"] = owner_resume_id locked_goal["activation"] = proposed_activation transaction.commit(source_payload) payload["written"] = True @@ -488,6 +506,7 @@ def set_goal_activation_state( target_registry=target_registry, goal_id=normalized_goal_id, expected_state=target_state, + expected_owner_resume_id=owner_resume_id, ) ) payload["readback"] = readback diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index c7f652833b..4495716bdb 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -10,6 +10,7 @@ from ...agent_registry import load_goal_from_registry, registered_agent_ids_for_goal from ..agent_context import project_agent_context, project_goal_agent_context from ..runtime.time import now_utc_iso +from ..goals.activation import goal_owner_resume_id from ..goals.goal_frontier.fallback_source import live_fallback_authority_items from ..capability_hooks import ( InteractionProjectionHookRegistration, @@ -921,6 +922,7 @@ def build_live_quota_should_run_decision( available_capabilities=available_capabilities, scheduler_execution_context=resolved_context, goal_ref=goal_ref, + owner_action_id=goal_owner_resume_id(registered_goal), ) if recovery_todo_id := _receipt_bound_recovery_todo( payload, receipt_bound_todo_id, agent_id=agent_id, turn_instance_id=turn_instance_id, @@ -1059,6 +1061,7 @@ def _build_remote_authority_wait_decision( agent_id=agent_id, current_turn_instance_id=turn_instance_id, available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, goal_ref=goal_ref, + owner_action_id=goal_owner_resume_id(configured_goal), ): return payload diff --git a/loopx/control_plane/quota/unsettled_host_turn.py b/loopx/control_plane/quota/unsettled_host_turn.py index 0c82e44584..aeefb57dd1 100644 --- a/loopx/control_plane/quota/unsettled_host_turn.py +++ b/loopx/control_plane/quota/unsettled_host_turn.py @@ -348,17 +348,13 @@ def apply_unsettled_host_turn_recovery_if_required( Mapping[str, Any] | SchedulerExecutionContextResolution | None ), goal_ref: Mapping[str, Any] | None = None, + owner_action_id: str | None = None, ) -> bool: """Preempt ordinary selection when the preceding host Turn lacks closeout.""" if quota_item_is_paused(payload): return False - from ...agent_registry import load_goal_from_registry - - goal = load_goal_from_registry(registry_path, goal_id) or {} - activation = goal.get("activation") or {} - verdict = _unsettled_host_turn_recovery( registry_path=registry_path, runtime_root=runtime_root, @@ -366,8 +362,7 @@ def apply_unsettled_host_turn_recovery_if_required( agent_id=agent_id, current_turn_instance_id=current_turn_instance_id, goal_ref=goal_ref, - owner_action_id=(activation.get("updated_at") - if activation.get("actor_kind") == "owner" else None), + owner_action_id=owner_action_id, ) if verdict is None: return False diff --git a/tests/control_plane/test_goal_activation.py b/tests/control_plane/test_goal_activation.py index e7459477e1..b72f4ab98a 100644 --- a/tests/control_plane/test_goal_activation.py +++ b/tests/control_plane/test_goal_activation.py @@ -385,6 +385,27 @@ def test_stop_and_resume_sync_source_global_and_quota( assert resumed_quota["allowed_slots"] == 4 +def test_controller_transitions_retain_legacy_owner_resume( + connected_registries: tuple[Path, Path], +) -> None: + source_registry, global_registry = connected_registries + prior_resume = "2026-06-01T12:00:00Z" + source = load_registry(source_registry) + registry_goals(source)[0]["activation"] = { + "schema_version": "loopx_goal_activation_v1", "state": "active", + "updated_at": prior_resume, "reason": "Resumed by owner", "actor_kind": "owner", + } + _write_json(source_registry, source) + for state in ("stopped", "active"): + result = set_goal_activation_state( + registry_path=global_registry, goal_id="goal-one", state=state, + actor_kind="controller", execute=True, + ) + assert result["ok"] and result["written"] and result["readback"]["verified"], result + assert _goal(source_registry)["activation"]["owner_resume_id"] == prior_resume + assert _goal(global_registry)["activation"]["owner_resume_id"] == prior_resume + + def test_activation_and_agent_registration_share_source_to_global_lock_order( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/control_plane/test_unsettled_recovery_admission_bound.py b/tests/control_plane/test_unsettled_recovery_admission_bound.py index 04d020e39d..88ecb300bc 100644 --- a/tests/control_plane/test_unsettled_recovery_admission_bound.py +++ b/tests/control_plane/test_unsettled_recovery_admission_bound.py @@ -88,3 +88,65 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( if reset == "writeback": assert debt["missing_receipts"] == ["quota_spend_receipt"] assert _spend_run_count(runtime) == 0 + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("intermediate_guard", [False, True]) +def test_owner_resume_survives_controller_lifecycle( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, intermediate_guard: bool, +) -> None: + isolate_sqlite_runtime(tmp_path, monkeypatch) + _, runtime, registry = _write_fixture(tmp_path, provider=provider) + guard = ("quota", "should-run", "--codex-app", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID) + lifecycle = ("goal-lifecycle", "--goal-id", GOAL_ID) + + rc, original = _run_cli(registry, runtime, *guard, + "--turn-instance-id", "owed-original", "--todo-id", TODO_ID) + assert rc == 0 and original["heartbeat_receipt"]["closeout_required"], original + effect_id = original["heartbeat_receipt"]["settlement_identity"]["effect_id"] + for attempt in range(3): + rc, admitted = _run_cli(registry, runtime, *guard, + "--turn-instance-id", f"recovery-{attempt}") + assert rc == 0 and admitted["should_run"] is True, admitted + rc, limited = _run_cli(registry, runtime, *guard, "--turn-instance-id", "limited") + assert rc == 0 and limited["should_run"] is False, limited + + registry_before_preview = registry.read_bytes() + rc, preview = _run_cli(registry, runtime, *lifecycle, "--operation", "stop", + "--actor-kind", "owner") + assert rc == 0 and preview["dry_run"] is True and preview["written"] is False, preview + assert registry.read_bytes() == registry_before_preview + + for actor in ("owner", "controller"): + for operation in ("stop", "resume"): + rc, transition = _run_cli(registry, runtime, *lifecycle, "--operation", operation, + "--actor-kind", actor, "--execute") + assert rc == 0 and transition["written"] is True, transition + if actor == "owner" and intermediate_guard: + rc, interim = _run_cli(registry, runtime, *guard, "--turn-instance-id", "interim") + assert rc == 0 and interim["should_run"] is True, interim + assert interim["unsettled_host_turn_recovery"]["admissions_without_progress"] == 0 + + rc, after = _run_cli(registry, runtime, *guard, "--turn-instance-id", "after-controller") + assert rc == 0 and after["should_run"] is True, { + "should_run": after.get("should_run"), + "debt_state": (after.get("unsettled_host_turn_recovery") or {}).get("state"), + "admissions_without_progress": (after.get("unsettled_host_turn_recovery") or {}).get( + "admissions_without_progress"), + } + debt = after["unsettled_host_turn_recovery"] + assert debt["admissions_without_progress"] == int(intermediate_guard) + assert debt["prior_turn_instance_id"] == "owed-original" + assert debt["settlement_effect_id"] == effect_id + + rc, replay = _run_cli(registry, runtime, *guard, "--turn-instance-id", "after-controller") + assert rc == 0 and replay["should_run"] is True, replay + assert replay["heartbeat_receipt"]["event_id"] == after["heartbeat_receipt"]["event_id"] + + rc, noop = _run_cli(registry, runtime, *lifecycle, "--operation", "resume", + "--actor-kind", "owner", "--execute") + assert rc == 0 and noop["written"] is False, noop + rc, next_guard = _run_cli(registry, runtime, *guard, "--turn-instance-id", "after-noop") + assert rc == 0 and next_guard["should_run"] is True, next_guard + assert next_guard["unsettled_host_turn_recovery"]["admissions_without_progress"] == 1 + int(intermediate_guard) + assert _spend_run_count(runtime) == 0 From b958bc4f899c2a78bb5af14136b7c80f3655ad79 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 05:22:46 +0800 Subject: [PATCH 4/4] Commit bounded recovery grants against current receipts Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/quota-allocation.md | 9 + loopx/cli_commands/quota.py | 230 +++++++++++------- loopx/cli_rollout.py | 7 +- .../quota/unsettled_host_turn.py | 138 ++++++++--- .../quota/unsettled_host_turn_recovery.ts | 15 +- loopx/rollout_event_log.py | 4 + ...test_unsettled_recovery_admission_bound.py | 83 ++++++- .../unsettled_host_turn_recovery.test.ts | 35 ++- 8 files changed, 385 insertions(+), 136 deletions(-) diff --git a/docs/quota-allocation.md b/docs/quota-allocation.md index 88d9bc2069..e04242209f 100644 --- a/docs/quota-allocation.md +++ b/docs/quota-allocation.md @@ -234,11 +234,20 @@ renews the allowance. The Goal activation record retains the last Owner resume through later Controller Stop and Resume transitions. Resuming a Goal that is already active leaves the recorded allowance in place. +Each new recovery grant checks its admission basis under the receipt append +lock. A contender with a stale basis reevaluates the same Turn. When an unbound +waiting Turn becomes eligible after resume, its first positive recovery +receipt is appended under the same lock and subsequent replays reuse it. + 同一原始 Turn 的缺失回执没有减少时,恢复准入有次数上限,按已提交的不同 guard 准入计数。相同 guard 重放和宿主完成事件均保持原计数。验证到回执进展或 Owner 显式恢复 Goal 后重新计算。Goal 激活记录会保留最近一次 Owner 恢复,后续 Controller 停止或恢复均沿用该记录。对已启用的 Goal 再次恢复会保留既有额度。 +新的恢复准入会在持有回执追加锁时复核准入依据。依据已变化的并发请求用同一 Turn +重新判定。未绑定的等待 Turn 在恢复后取得资格时,在同一把锁内追加首次准入回执, +后续重放复用该回执。 + An admitted advancement Todo can also become persistently `blocked` through its ordinary owner-authorized lifecycle. Its original Turn then offers the same blocked writeback command. The settlement owner freezes the current diff --git a/loopx/cli_commands/quota.py b/loopx/cli_commands/quota.py index 0516351b53..053b0d5778 100644 --- a/loopx/cli_commands/quota.py +++ b/loopx/cli_commands/quota.py @@ -26,6 +26,8 @@ project_captured_context, ) from ..usage_goal import observe_quota_result +from ..file_lock import LockAcquireTimeoutError +from ..rollout_event_log import RolloutAppendPreconditionChanged from ..control_plane.coordination.local_authority import CanonicalAuthorityRequired from ..control_plane.quota.effective_action import EffectiveAction @@ -40,6 +42,7 @@ attach_uncommitted_heartbeat_receipt, fail_heartbeat_receipt, find_heartbeat_receipt, + heartbeat_receipt_request_subject, heartbeat_receipt_view, ) from .quota_registration import ( @@ -671,6 +674,123 @@ def execute() -> int: return run_with_quota_failure_reader(args, limit, print_payload=print_payload, run=execute) +def _append_quota_heartbeat_receipt( + payload: dict[str, object], + args: argparse.Namespace, + *, + registry_path: Path, + runtime_root_arg: str | None, + runtime_root: Path, + goal_ref: dict[str, str] | None, + append_cli_rollout_event: RolloutEventAppender, + turn_instance_id: str, + stall_observation: str, + rollout_todo_id: str | None, + rollout_replan_obligation_id: str | None, + rollout_details: dict[str, object], + upgrade_recovery_wait: bool, + requested_todo_id: str | None, +) -> dict[str, object]: + from ..control_plane.quota.unsettled_host_turn import recovery_admission_append_precondition + + settlement_identity = ( + SettlementIdentity( + goal_id=args.goal_id, + agent_id=args.agent_id, + todo_id=rollout_todo_id, + turn_instance_id=turn_instance_id, + replan_obligation_id=rollout_replan_obligation_id, + ) + if rollout_todo_id or rollout_replan_obligation_id + else None + ) + rollout_details.update( + { + "turn_instance_id": turn_instance_id, + "stall_observation": stall_observation, + "settlement_effect_id": settlement_identity.effect_id if settlement_identity is not None else "", + } + ) + try: + with recovery_admission_append_precondition( + payload, registry_path=registry_path, runtime_root=runtime_root, + goal_id=args.goal_id, agent_id=args.agent_id, + turn_instance_id=turn_instance_id, goal_ref=goal_ref, + ) as precondition: + append_cli_rollout_event( + payload, + registry_path=registry_path, + runtime_root_arg=runtime_root_arg, + event_kind="quota_should_run", + goal_ref=goal_ref, + agent_id=args.agent_id, + run_id=turn_instance_id, + status=str(payload.get("effective_action") or payload.get("decision") or "should-run"), + summary=f"heartbeat quota receipt committed for turn={turn_instance_id} stall={stall_observation}", + details=rollout_details, + allow_failed=True, + idempotency_fields=[ + "goal_id", + "event_kind", + "agent_id", + "run_id", + *(["goal_ref"] if goal_ref is not None else []), + *(["status"] if upgrade_recovery_wait else []), + ], + **({"precondition": precondition} if precondition is not None else {}), + ) + receipt = find_heartbeat_receipt( + runtime_root, goal_id=args.goal_id, agent_id=args.agent_id, + turn_instance_id=turn_instance_id, goal_ref=goal_ref, + ) + except LockAcquireTimeoutError as exc: + failure = _export("quota_failure_payload")( + args, registry_path=registry_path, runtime_root_arg=runtime_root_arg, error=exc, + ) + fail_heartbeat_receipt( + failure, turn_instance_id=turn_instance_id, stall_observation=stall_observation, + reason="recovery receipt append could not acquire the quota authority lock", + ) + return failure + if receipt is None: + fail_heartbeat_receipt( + payload, turn_instance_id=turn_instance_id, stall_observation=stall_observation, + reason="heartbeat receipt append could not be read back; retry quota should-run with the same --turn-instance-id", + ) + return payload + if precondition is not None and receipt.get("status") != EffectiveAction.UNSETTLED_HOST_TURN_RECOVERY.value: + raise RolloutAppendPreconditionChanged("A different guard was recorded for this Turn.") + + rollout_event = payload.get("rollout_event") + status = "replayed" + if isinstance(rollout_event, Mapping) and rollout_event.get("appended"): + status = "upgraded" if upgrade_recovery_wait else "committed" + payload["heartbeat_receipt"] = heartbeat_receipt_view(receipt, turn_instance_id=turn_instance_id, status=status) + _export("commit_requested_action_selection")(payload, requested_todo_id=requested_todo_id) + return payload + + +def _can_upgrade_recovery_wait( + payload: Mapping[str, object], selection: RequestedQuotaActionSelection, +) -> bool: + receipt = selection.receipt + if receipt is None or payload.get("effective_action") != EffectiveAction.UNSETTLED_HOST_TURN_RECOVERY.value: + return False + details = receipt.get("details") + if payload.get("should_run") is not True or not isinstance(details, Mapping) or details.get("should_run") is not False: + return False + if ( + receipt.get("status") != EffectiveAction.BLOCKED_WAIT.value + or selection.receipt_bound_todo_id + or selection.receipt_bound_replan_obligation_id + or heartbeat_receipt_request_subject(receipt) is not None + ): + raise HeartbeatReceiptIdentityConflictError( + "The existing Turn has no recovery grant; use a fresh Turn for recovery." + ) + return True + + def _execute_quota_command( args: argparse.Namespace, *, @@ -680,6 +800,7 @@ def _execute_quota_command( append_cli_rollout_event: RolloutEventAppender, capture_directory: Path | None = None, retained_action_selection_todo_id: str | None = None, + recovery_refreshes_remaining: int = 3, ) -> int: usage_quota_started = time.time_ns() // 1_000_000 heartbeat_turn_id: str | None = None @@ -687,6 +808,7 @@ def _execute_quota_command( heartbeat_receipt_existing_status = "replayed" heartbeat_receipt_existing_appended = False heartbeat_receipt_ready = False + upgrade_recovery_wait = False action_selection_preflight_failed = False closeout_query_unavailable = False action_selection: RequestedQuotaActionSelection | None = None @@ -818,7 +940,10 @@ def _execute_quota_command( action_selection_preflight.receipt_appended ) if heartbeat_turn_id and reentry is None and not action_selection_preflight_failed: - if heartbeat_receipt_existing: + # A wait receipt grants no recovery allowance. After a valid + # resume, bind its first grant under the ordinary append fence. + upgrade_recovery_wait = _can_upgrade_recovery_wait(payload, action_selection) + if heartbeat_receipt_existing and not upgrade_recovery_wait: ( heartbeat_receipt_existing, heartbeat_receipt_existing_status, @@ -961,6 +1086,7 @@ def _execute_quota_command( reentry_args, registry_path=registry_path, runtime_root_arg=runtime_root_arg, print_payload=print_payload, append_cli_rollout_event=append_cli_rollout_event, capture_directory=capture_directory, retained_action_selection_todo_id=retained_todo_id, + recovery_refreshes_remaining=recovery_refreshes_remaining, ) preview = args.quota_command == "should-run" and getattr(args, "dry_run", False) if not preview and _export("should_log_quota")(args.quota_command, payload): @@ -1003,7 +1129,7 @@ def _execute_quota_command( + (f": {prior_reason}" if prior_reason else "") ), ) - elif heartbeat_receipt_existing: + elif heartbeat_receipt_existing and not upgrade_recovery_wait: _export("render_existing_heartbeat_receipt_payload")( payload, receipt=heartbeat_receipt_existing, @@ -1020,93 +1146,29 @@ def _execute_quota_command( ), ) else: - settlement_identity = ( - SettlementIdentity( - goal_id=args.goal_id, - agent_id=args.agent_id, - todo_id=rollout_todo_id, - turn_instance_id=heartbeat_turn_id, - replan_obligation_id=rollout_replan_obligation_id, - ) - if rollout_todo_id or rollout_replan_obligation_id - else None - ) - rollout_details.update( - { - "turn_instance_id": heartbeat_turn_id, - "stall_observation": heartbeat_stall_observation, - "settlement_effect_id": ( - settlement_identity.effect_id - if settlement_identity is not None - else "" - ), - } - ) - append_cli_rollout_event( - payload, - registry_path=registry_path, - runtime_root_arg=runtime_root_arg, - event_kind="quota_should_run", - goal_ref=goal_ref, - agent_id=args.agent_id, - run_id=heartbeat_turn_id, - status=str( - payload.get("effective_action") - or payload.get("decision") - or "should-run" - ), - summary=( - "heartbeat quota receipt committed for " - f"turn={heartbeat_turn_id} stall={heartbeat_stall_observation}" - ), - details=rollout_details, - allow_failed=True, - idempotency_fields=[ - "goal_id", - "event_kind", - "agent_id", - "run_id", - *(["goal_ref"] if goal_ref is not None else []), - ], - ) - receipt = find_heartbeat_receipt( - runtime_root, - goal_id=args.goal_id, - agent_id=args.agent_id, - turn_instance_id=heartbeat_turn_id, - goal_ref=goal_ref, - ) - if receipt: - rollout_event_value = payload.get("rollout_event") - rollout_event: Mapping[str, object] = ( - rollout_event_value - if isinstance(rollout_event_value, Mapping) - else {} - ) - payload["heartbeat_receipt"] = heartbeat_receipt_view( - receipt, - turn_instance_id=heartbeat_turn_id, - status="committed" - if rollout_event.get("appended") - else "replayed", + try: + payload = _append_quota_heartbeat_receipt( + payload, args, registry_path=registry_path, runtime_root_arg=runtime_root_arg, + runtime_root=runtime_root, goal_ref=goal_ref, append_cli_rollout_event=append_cli_rollout_event, + turn_instance_id=heartbeat_turn_id, stall_observation=heartbeat_stall_observation, + rollout_todo_id=rollout_todo_id, rollout_replan_obligation_id=rollout_replan_obligation_id, + rollout_details=rollout_details, upgrade_recovery_wait=upgrade_recovery_wait, + requested_todo_id=action_selection.requested_todo_id if action_selection is not None else None, ) - _export("commit_requested_action_selection")( - payload, - requested_todo_id=( - action_selection.requested_todo_id - if action_selection is not None - else None - ), - ) - else: + except RolloutAppendPreconditionChanged: + if recovery_refreshes_remaining: + return _execute_quota_command( + args, registry_path=registry_path, runtime_root_arg=runtime_root_arg, + print_payload=print_payload, append_cli_rollout_event=append_cli_rollout_event, + capture_directory=capture_directory, + retained_action_selection_todo_id=retained_action_selection_todo_id, + recovery_refreshes_remaining=recovery_refreshes_remaining - 1, + ) fail_heartbeat_receipt( payload, turn_instance_id=heartbeat_turn_id, stall_observation=heartbeat_stall_observation, - reason=( - "heartbeat receipt append could not be read back; retry " - "quota should-run with the same --turn-instance-id" - ), + reason="recovery admission changed repeatedly before append; retry the same Turn", ) else: _record_non_guard_quota_event( diff --git a/loopx/cli_rollout.py b/loopx/cli_rollout.py index e275fa182f..ca176c0a66 100644 --- a/loopx/cli_rollout.py +++ b/loopx/cli_rollout.py @@ -1,11 +1,12 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path from .control_plane.projects.registry_codec import load_registry from .paths import resolve_runtime_root from .rollout_event_log import ( + RolloutAppendPreconditionChanged, append_rollout_event, append_rollout_event_once, build_rollout_event, @@ -31,6 +32,7 @@ def append_cli_rollout_event( details: dict[str, object] | None = None, allow_failed: bool = False, idempotency_fields: list[str] | None = None, + precondition: Callable[[], None] | None = None, ) -> dict[str, object]: """Append a compact rollout event for core CLI lifecycle commands. @@ -73,6 +75,7 @@ def append_cli_rollout_event( log_path, event, identity_fields=idempotency_fields, + precondition=precondition, ) else: appended = append_rollout_event(log_path, event) @@ -87,6 +90,8 @@ def append_cli_rollout_event( if idempotency_fields: rollout_event_view["appended"] = newly_appended payload["rollout_event"] = rollout_event_view + except RolloutAppendPreconditionChanged: + raise except Exception as exc: payload["rollout_event_log_error"] = { "recorded": False, diff --git a/loopx/control_plane/quota/unsettled_host_turn.py b/loopx/control_plane/quota/unsettled_host_turn.py index aeefb57dd1..44ee6885f7 100644 --- a/loopx/control_plane/quota/unsettled_host_turn.py +++ b/loopx/control_plane/quota/unsettled_host_turn.py @@ -11,10 +11,12 @@ from .effective_action import EffectiveAction from .states import quota_item_is_paused -from collections.abc import Mapping +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager from pathlib import Path from typing import Any +from ...rollout_event_log import RolloutAppendPreconditionChanged from ..effect_runtime import ( EffectRuntimeRejected, EffectRuntimeResponseAmbiguous, @@ -182,44 +184,57 @@ def _prior_closeout_preflight( than a megabyte log, and a settled prior Turn never causes a bound-fact read. """ - try: - with quota_accounting_admission( + with quota_accounting_admission( + runtime_root=runtime_root, + registry_path=registry_path, + goal_id=goal_id, + goal_ref=goal_ref, + operation="prior-host-turn-closeout-preflight", + lock_legacy_index=False, + ) as source_admission: + return _read_prior_closeout_preflight( runtime_root=runtime_root, - registry_path=registry_path, goal_id=goal_id, + agent_id=agent_id, + current_turn_instance_id=current_turn_instance_id, goal_ref=goal_ref, - operation="prior-host-turn-closeout-preflight", - lock_legacy_index=False, - ) as source_admission: - request_runtime_root = ( - runtime_root.expanduser().resolve() - if source_admission is not None - else runtime_root.expanduser() - ) - result = effect_runtime_result( - PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_METHOD, - { - "schema_version": ( - PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA - ), - "runtime_root": str(request_runtime_root), - "goal_id": goal_id, - "agent_id": agent_id, - "exclude_turn_instance_id": current_turn_instance_id, - "owner_action_id": owner_action_id, - **( - {"goal_ref": dict(goal_ref)} - if goal_ref is not None - else {} - ), - **( - {"source_admission": dict(source_admission)} - if source_admission is not None - else {} - ), - }, - timeout=PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_TIMEOUT_SECONDS, - ) + owner_action_id=owner_action_id, + source_admission=source_admission, + ) + + +def _read_prior_closeout_preflight( + *, + runtime_root: Path, + goal_id: str, + agent_id: str, + current_turn_instance_id: str | None, + goal_ref: Mapping[str, Any] | None, + owner_action_id: str | None, + source_admission: Mapping[str, Any] | None, + borrow_source_admission: bool = False, +) -> tuple[dict[str, Any], list[str], dict[str, Any], dict[str, Any]] | None: + try: + request_runtime_root = ( + runtime_root.expanduser().resolve() + if source_admission is not None + else runtime_root.expanduser() + ) + result = effect_runtime_result( + PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_METHOD, + { + "schema_version": PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, + "runtime_root": str(request_runtime_root), + "goal_id": goal_id, + "agent_id": agent_id, + "exclude_turn_instance_id": current_turn_instance_id, + "owner_action_id": owner_action_id, + **({"goal_ref": dict(goal_ref)} if goal_ref is not None else {}), + **({"source_admission": dict(source_admission)} if source_admission is not None else {}), + **({"borrow_source_admission": True} if borrow_source_admission else {}), + }, + timeout=PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_TIMEOUT_SECONDS, + ) except EffectRuntimeResponseAmbiguous as exc: # This method only reads receipts. A lost query response is not a # possibly committed mutation, and must not send the operator hunting @@ -261,6 +276,57 @@ def _prior_closeout_preflight( ) +@contextmanager +def recovery_admission_append_precondition( + payload: Mapping[str, Any], + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + agent_id: str, + turn_instance_id: str, + goal_ref: Mapping[str, Any] | None, +) -> Iterator[Callable[[], None] | None]: + recovery = payload.get("unsettled_host_turn_recovery") + if (payload.get("should_run") is not True or not isinstance(recovery, Mapping) + or recovery.get("state") != "recovery_required"): + yield None + return + + from ...agent_registry import load_goal_from_registry + from ..goals.activation import goal_is_stopped, goal_owner_resume_id + + # The append owns the rollout lock. Acquire any source-lifetime locks + # first, as quota settlement does, and reuse those locks for the read. + with quota_accounting_admission( + runtime_root=runtime_root, + registry_path=registry_path, + goal_id=goal_id, + goal_ref=goal_ref, + operation="recovery_admission_append", + lock_legacy_index=False, + ) as source_admission: + def check() -> None: + goal = load_goal_from_registry(registry_path, goal_id) + if goal is None or goal_is_stopped(goal): + raise RolloutAppendPreconditionChanged("The Goal is unavailable or stopped before recovery admission.") + current = _read_prior_closeout_preflight( + runtime_root=runtime_root, + goal_id=goal_id, + agent_id=agent_id, + current_turn_instance_id=turn_instance_id, + goal_ref=goal_ref, + owner_action_id=goal_owner_resume_id(goal), + source_admission=source_admission, + borrow_source_admission=source_admission is not None, + ) + if (current is None or current[3]["checkpoint"] != recovery["admission_checkpoint"] + or current[3]["count"] != recovery["admissions_without_progress"]): + raise RolloutAppendPreconditionChanged("The recovery admission basis changed before append.") + + yield check + + def _unsettled_host_turn_recovery( *, registry_path: Path, diff --git a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts index a09ebf65c6..8bb66871c0 100644 --- a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts +++ b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts @@ -49,6 +49,7 @@ import { import { parseQuotaAccountingOwner, quotaOwnerOwnsProjection, + withBorrowedQuotaAccountingOwner, withQuotaAccountingOwner, type QuotaAccountingOwner, } from "./source_admission.ts"; @@ -98,6 +99,7 @@ interface PreflightRequest { owner: QuotaAccountingOwner; owner_projection: JsonObject; owner_action_id: string | null; + borrow_source_admission: boolean; } function decodePreflightRequest(value: unknown): PreflightRequest { @@ -115,6 +117,12 @@ function decodePreflightRequest(value: unknown): PreflightRequest { runtimeRoot, goalId, }); + if (request.borrow_source_admission !== undefined && typeof request.borrow_source_admission !== "boolean") { + throw new EffectRuntimeRequestError("borrow_source_admission must be a boolean"); + } + if (request.borrow_source_admission === true && owner.kind !== "exact_source") { + throw new EffectRuntimeRequestError("borrow_source_admission requires exact source admission"); + } return { runtime_root: runtimeRoot, goal_id: goalId, @@ -123,6 +131,7 @@ function decodePreflightRequest(value: unknown): PreflightRequest { request.exclude_turn_instance_id, ), owner_action_id: optionalHeartbeatString(request.owner_action_id), + borrow_source_admission: request.borrow_source_admission === true, owner, owner_projection: owner.kind === "alias" ? {} @@ -186,7 +195,8 @@ function recoveryAdmissions( (details.recovery_missing_spend && !checkpoint.recovery_missing_spend)) break; admittedTurns.add(requireNonEmptyString(event.run_id, "recovery admission run_id")); } - return {count: admittedTurns.size, owner_action_id: ownerActionId}; + return {count: admittedTurns.size, owner_action_id: ownerActionId, + checkpoint: {...checkpoint, recovery_owner_action_id: ownerActionId}}; } function settlementReadbackRequest( @@ -295,7 +305,8 @@ export async function preflightPriorHostTurnCloseout( value: unknown, ): Promise { const request = decodePreflightRequest(value); - return await withQuotaAccountingOwner( + const withOwner = request.borrow_source_admission ? withBorrowedQuotaAccountingOwner : withQuotaAccountingOwner; + return await withOwner( request.owner, async () => preflightPriorHostTurnCloseoutForOwner(request), ); diff --git a/loopx/rollout_event_log.py b/loopx/rollout_event_log.py index 7019853c0a..c0b0b6dc9d 100644 --- a/loopx/rollout_event_log.py +++ b/loopx/rollout_event_log.py @@ -463,6 +463,10 @@ def append_rollout_event(log_path: Path, event: Mapping[str, Any]) -> dict[str, return payload +class RolloutAppendPreconditionChanged(RuntimeError): + """The proposed transition needs to be recomputed before append.""" + + def append_rollout_event_once( log_path: Path, event: Mapping[str, Any], diff --git a/tests/control_plane/test_unsettled_recovery_admission_bound.py b/tests/control_plane/test_unsettled_recovery_admission_bound.py index 88ecb300bc..2f1dad56b7 100644 --- a/tests/control_plane/test_unsettled_recovery_admission_bound.py +++ b/tests/control_plane/test_unsettled_recovery_admission_bound.py @@ -1,8 +1,11 @@ """Recovery admissions are bounded per original debt, never treated as spend.""" +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from threading import Barrier import pytest +from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime from tests.control_plane.test_quota_settlement_cli import ( AGENT_ID, GOAL_ID, TODO_ID, _run_cli, _spend_run_count, _write_fixture, @@ -73,6 +76,20 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( "--delivery-outcome", "outcome_progress", "--delivery-boundary", "in_flight_continuation", "--no-global-sync", "--suppress-external-sinks", cwd=project) assert rc == 0, result + if reset == "owner_resume": + rc, upgraded = _run_cli(registry, runtime, *guard, "--turn-instance-id", "wait-2") + assert rc == 0 and upgraded["should_run"] is True, upgraded + assert upgraded["heartbeat_receipt"]["status"] == "upgraded" + assert upgraded["heartbeat_receipt"]["event_id"] != wait["heartbeat_receipt"]["event_id"] + assert upgraded["unsettled_host_turn_recovery"]["admissions_without_progress"] == 0 + rc, replay = _run_cli(registry, runtime, *guard, "--turn-instance-id", "wait-2") + assert rc == 0 and replay["should_run"] is True, replay + assert replay["heartbeat_receipt"]["event_id"] == upgraded["heartbeat_receipt"]["event_id"] + rows = [event for event in load_rollout_events(rollout_event_log_path(runtime, GOAL_ID)) + if event["run_id"] == "wait-2"] + assert [event["details"]["should_run"] for event in rows] == [False, True] + assert rows[-1]["details"]["recovery_effect_id"] == effect + rc, after = _run_cli(registry, runtime, *guard, "--turn-instance-id", "after-progress") assert rc == 0, after if reset == "blocked_closeout": @@ -82,7 +99,8 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( assert after["unsettled_host_turn_recovery"]["admissions_without_progress"] == 3 else: debt = after["unsettled_host_turn_recovery"] - assert after["should_run"] is True and debt["admissions_without_progress"] == 0, after + assert after["should_run"] is True, after + assert debt["admissions_without_progress"] == int(reset == "owner_resume"), after assert debt["prior_turn_instance_id"] == "owed-original" assert debt["settlement_effect_id"] == effect if reset == "writeback": @@ -150,3 +168,66 @@ def test_owner_resume_survives_controller_lifecycle( assert rc == 0 and next_guard["should_run"] is True, next_guard assert next_guard["unsettled_host_turn_recovery"]["admissions_without_progress"] == 1 + int(intermediate_guard) assert _spend_run_count(runtime) == 0 + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_only_one_process_can_take_the_last_recovery_admission( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +) -> None: + isolate_sqlite_runtime(tmp_path, monkeypatch) + _, runtime, registry = _write_fixture(tmp_path, provider=provider) + guard = ("quota", "should-run", "--codex-app", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID) + + def read(turn: str, *scoping: str) -> dict: + rc, payload = _run_cli(registry, runtime, *guard, "--turn-instance-id", turn, *scoping) + assert rc == 0, payload + return payload + + original = read("owed-original", "--todo-id", TODO_ID) + assert original["heartbeat_receipt"]["closeout_required"] is True + effect = original["heartbeat_receipt"]["settlement_identity"]["effect_id"] + for index in range(2): + granted = read(f"prior-recovery-{index}") + assert granted["should_run"] is True + assert granted["unsettled_host_turn_recovery"]["admissions_without_progress"] == index + + log = rollout_event_log_path(runtime, GOAL_ID) + before_preview = log.read_bytes() + rc, preview = _run_cli(registry, runtime, *guard, "--turn-instance-id", "preview-recovery", "--dry-run") + assert rc == 1 and preview["reason"] == "turn-scoped `quota should-run` cannot use --dry-run", preview + assert log.read_bytes() == before_preview + + start = Barrier(3) + + def claim_last(turn: str) -> tuple[str, dict]: + start.wait(timeout=15) + return turn, read(turn) + + with ThreadPoolExecutor(max_workers=3) as workers: + results = dict(workers.map(claim_last, (f"racing-recovery-{i}" for i in range(3)))) + granted_turns = [turn for turn, payload in results.items() if payload["should_run"] is True] + assert len(granted_turns) == 1, { + turn: (payload.get("should_run"), payload.get("reason_code")) for turn, payload in results.items() + } + winner, = granted_turns + for turn, payload in results.items(): + debt = payload["unsettled_host_turn_recovery"] + assert debt["settlement_effect_id"] == effect + if turn == winner: + assert debt["admissions_without_progress"] == 2 + else: + assert debt["admissions_without_progress"] == 3 + assert payload["reason_code"] == "unsettled_recovery_admission_limit" + + replay = read(winner) + assert replay["heartbeat_receipt"]["event_id"] == results[winner]["heartbeat_receipt"]["event_id"] + exhausted = read("after-race") + assert exhausted["should_run"] is False + assert exhausted["unsettled_host_turn_recovery"]["admissions_without_progress"] == 3 + grants = [ + event["run_id"] for event in load_rollout_events(log) + if event.get("details", {}).get("recovery_effect_id") == effect + and event["details"].get("should_run") is True + ] + assert sorted(grants) == sorted(["prior-recovery-0", "prior-recovery-1", winner]) + assert _spend_run_count(runtime) == 0 diff --git a/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts b/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts index 231be993ec..4bd8ef4448 100644 --- a/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts +++ b/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts @@ -265,7 +265,8 @@ test("the admission bound keeps debt identity, deduplicates replays, and accepts const log = join(runtime.root, "goals", GOAL, "rollout-event-log.jsonl"); await writeFile(log, history.map(event => JSON.stringify(event)).join("\n") + "\n"); const observed = await preflight(runtime.root); - assert.deepEqual(observed.recovery_admissions, {count: RECOVERY_ADMISSION_LIMIT, owner_action_id: null}); + assert.deepEqual(observed.recovery_admissions, + {count: RECOVERY_ADMISSION_LIMIT, owner_action_id: null, checkpoint}); const decide = (facts: JsonObject, missing = observed.missing_receipts) => reduceUnsettledHostTurnRecovery({ schema_version: UNSETTLED_HOST_TURN_RECOVERY_REQUEST_SCHEMA, goal_id: GOAL, agent_id: AGENT, candidate, missing_receipts: missing, @@ -281,7 +282,10 @@ test("the admission bound keeps debt identity, deduplicates replays, and accepts runtime_root: runtime.root, goal_id: GOAL, agent_id: AGENT, exclude_turn_instance_id: "current", owner_action_id: "explicit-owner-resume", }); - assert.deepEqual(owner.recovery_admissions, {count: 0, owner_action_id: "explicit-owner-resume"}); + assert.deepEqual(owner.recovery_admissions, { + count: 0, owner_action_id: "explicit-owner-resume", + checkpoint: {...checkpoint, recovery_owner_action_id: "explicit-owner-resume"}, + }); const replay = await preflight(runtime.root, "recovery-c"); assert.equal((replay.recovery_admissions as JsonObject).count, 2); } finally { @@ -304,8 +308,10 @@ test("losing an owner activation projection cannot renew its exhausted allowance history.map(event => JSON.stringify(event)).join("\n") + "\n"); // Controller activation has no owner id. Keep the latest recorded owner // epoch, without counting admissions before that owner's explicit reset. - assert.deepEqual((await preflight(runtime.root)).recovery_admissions, - {count: 3, owner_action_id: "current-owner"}); + assert.deepEqual((await preflight(runtime.root)).recovery_admissions, { + count: 3, owner_action_id: "current-owner", + checkpoint: {...checkpoint, recovery_owner_action_id: "current-owner"}, + }); } finally { await runtime.close(); } @@ -418,14 +424,19 @@ test("exact preflight filters stale owners before selecting the newest Turn", as const result = await withSourceAdmission( runtime.root, instanceB, - async (binding) => await preflightPriorHostTurnCloseout({ - schema_version: PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, - runtime_root: runtime.root, - goal_id: GOAL, - agent_id: AGENT, - exclude_turn_instance_id: "current-turn", - ...binding, - }), + async (binding) => { + const request = { + schema_version: PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, + runtime_root: runtime.root, + goal_id: GOAL, + agent_id: AGENT, + exclude_turn_instance_id: "current-turn", + ...binding, + }; + const borrowed = await preflightPriorHostTurnCloseout({...request, borrow_source_admission: true}); + assert.equal(candidateFrom(borrowed).prior_turn_instance_id, "turn-b"); + return await preflightPriorHostTurnCloseout(request); + }, ); assert.equal(candidateFrom(result).prior_turn_instance_id, "turn-b");