diff --git a/loopx/cli_commands/quota.py b/loopx/cli_commands/quota.py index 213e79a511..0516351b53 100644 --- a/loopx/cli_commands/quota.py +++ b/loopx/cli_commands/quota.py @@ -612,6 +612,13 @@ def _record_non_guard_quota_event( ), details=rollout_details, allow_failed=args.quota_command == "should-run", + idempotency_fields=( + ["goal_id", "event_kind", "agent_id", "run_id"] + + (["todo_id"] if rollout_todo_id else []) + + (["goal_ref"] if goal_ref is not None else []) + if args.quota_command == "spend-slot" and spend_turn_instance_id + else None + ), ) if args.quota_command == "spend-slot" and spend_turn_instance_id: assert context is not None # A settlement requires a prepared authority context. diff --git a/loopx/cli_commands/quota_action_selection.py b/loopx/cli_commands/quota_action_selection.py index 7b59cfc3cb..52e4be56e0 100644 --- a/loopx/cli_commands/quota_action_selection.py +++ b/loopx/cli_commands/quota_action_selection.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from pathlib import Path +from ..control_plane.quota.effective_action import EffectiveAction from ..control_plane.quota.error_codes import ( HeartbeatReceiptIdentityConflictError, QuotaActionSelectionConflictError, @@ -166,6 +167,10 @@ def _requested_quota_action_selection_preflight( "current selected Todo: explicitly requested Todo differs" ) return None + if payload.get("effective_action") == EffectiveAction.UNSETTLED_HOST_TURN_RECOVERY.value: + # Recovery preempts the choice. Keep the exact recovery actions in the + # response; the rollout owner leaves this Turn unbound until reentry. + return None interaction = payload.get("interaction_contract") agent_channel = ( interaction.get("agent_channel") if isinstance(interaction, Mapping) else None diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index b93a5df82f..9cbf8136d9 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -872,6 +872,11 @@ def on_managed_start_admitted() -> None: semantic_replan_guard_scoped=replan_guard_scoped, semantic_replan_obligation_id=replan_obligation_id, semantic_replan_capability_guard=capability_guard, + quota_spend_source=( + quota_spend_source_for_execution_context(payload.get("scheduler_execution_context")) + if settlement_identity.replan_obligation_id or settlement_identity.request_subject is not None + else None + ), goal_ref=goal_ref, ) diff --git a/loopx/control_plane/quota/accounting_artifact_transaction.ts b/loopx/control_plane/quota/accounting_artifact_transaction.ts index 576deb4323..54867f699d 100644 --- a/loopx/control_plane/quota/accounting_artifact_transaction.ts +++ b/loopx/control_plane/quota/accounting_artifact_transaction.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import { access, lstat, readFile } from "node:fs/promises"; -import { basename, dirname, extname, join, resolve } from "node:path"; +import { access, lstat, readFile, realpath } from "node:fs/promises"; +import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path"; import type { JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; @@ -109,6 +109,7 @@ export interface QuotaAccountingArtifactCommitRequest { expectedIndexDigest: string | null; goalRef?: JsonObject; indexLockHeld?: boolean; + matchesRetainedRequest?: (record: JsonObject) => boolean; prepare: ( context: QuotaAccountingArtifactPrepareContext, ) => @@ -507,6 +508,59 @@ export function resolveQuotaAccountingEffect( return { kind: "absent" }; } +export async function readQuotaSpendEventFromRun( + runsDir: string, + run: JsonObject, + goalId: string, +): Promise { + const inline = jsonObject(run.quota_event); + if (inline) return inline; + if (typeof run.json_path !== "string" || !run.json_path.trim()) return null; + let targetPath: string; + try { + targetPath = await realpath(resolve(run.json_path.trim())); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return null; + } + throw error; + } + const relativePath = relative(await realpath(runsDir), targetPath); + if ( + !relativePath || + relativePath === ".." || + relativePath.startsWith(`..${sep}`) || + isAbsolute(relativePath) + ) { + throw new EffectRuntimeRequestError( + "quota spend json_path must stay inside the goal runs directory", + ); + } + let content: string; + try { + content = await readFile(targetPath, "utf8"); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return null; + throw error; + } + let value: unknown; + try { + value = JSON.parse(content); + } catch { + throw new EffectRuntimeRequestError("quota spend JSON artifact is malformed"); + } + const record = requiredObject(value, "quota spend JSON artifact"); + if ( + record.classification !== contractFor("spend").classification || + (record.goal_id !== undefined && record.goal_id !== goalId) + ) { + throw new EffectRuntimeRequestError( + "quota spend JSON artifact identity does not match its index row", + ); + } + return jsonObject(record.quota_event); +} + export async function lookupQuotaAccountingReplay( kind: QuotaAccountingArtifactKind, indexPath: string, @@ -920,7 +974,8 @@ export async function commitQuotaAccountingArtifactTransaction( } if ( existingReceipt.effect_id !== request.effectId || - existingReceipt.request_digest !== request.requestDigest + (existingReceipt.request_digest !== request.requestDigest && + request.matchesRetainedRequest?.(existingReceipt.record) !== true) ) { return { status: "conflict", diff --git a/loopx/control_plane/quota/heartbeat_receipt.py b/loopx/control_plane/quota/heartbeat_receipt.py index 8eaf717a38..b228abd519 100644 --- a/loopx/control_plane/quota/heartbeat_receipt.py +++ b/loopx/control_plane/quota/heartbeat_receipt.py @@ -17,6 +17,7 @@ from .settlement_workspace_causality import ( delivery_workspace_causality_from_event_details, ) +from .spend_sources import TURN_SCOPED_SLOT_SPEND_SOURCES HEARTBEAT_RECEIPT_SCHEMA_VERSION = "heartbeat_quota_receipt_v0" @@ -204,6 +205,7 @@ def ensure_turn_heartbeat_settlement_receipt( semantic_replan_guard_scoped: bool, semantic_replan_obligation_id: str | None, semantic_replan_capability_guard: Mapping[str, object] | None = None, + quota_spend_source: str | None = None, goal_ref: Mapping[str, object] | None = None, before_bind: Callable[[], None] | None = None, ) -> dict[str, object]: @@ -224,6 +226,8 @@ def ensure_turn_heartbeat_settlement_receipt( ): raise ValueError("Turn semantic replan obligation id is malformed") + if quota_spend_source is not None and quota_spend_source not in TURN_SCOPED_SLOT_SPEND_SOURCES: + raise ValueError("Turn quota spend source must be heartbeat or visible-goal") log_path = rollout_event_log_path(runtime_root, identity.goal_id) log_path.parent.mkdir(parents=True, exist_ok=True) with exclusive_cross_runtime_file_lock( @@ -248,6 +252,15 @@ def ensure_turn_heartbeat_settlement_receipt( raise HeartbeatReceiptIdentityConflictError( "Turn heartbeat receipt belongs to another settlement identity" ) + previous_details = effective.get("details") if effective is not None else None + previous_source = ( + previous_details.get("quota_spend_source") + if observed is not None and isinstance(previous_details, Mapping) else None + ) + if previous_source is not None and quota_spend_source not in (None, previous_source): + raise HeartbeatReceiptIdentityConflictError( + "Turn heartbeat receipt belongs to another quota spend source" + ) if before_bind is not None: # The existing cross-runtime binding lock serializes the competing # identity before transport-specific start admission. Its owner may @@ -311,6 +324,8 @@ def ensure_turn_heartbeat_settlement_receipt( details["semantic_replan_capability_id"] = semantic_replan_capability_guard["capability_id"] details["semantic_replan_gap_id"] = semantic_replan_capability_guard["gap_id"] details["semantic_replan_frontier_revision"] = semantic_replan_capability_guard["frontier_revision"] + if (source := previous_source or quota_spend_source) is not None: + details["quota_spend_source"] = source source_event_id = ( str(effective.get("event_id") or "").strip() if effective is not None diff --git a/loopx/control_plane/quota/prior_turn_closeout.ts b/loopx/control_plane/quota/prior_turn_closeout.ts index 723f548e78..67841d77f8 100644 --- a/loopx/control_plane/quota/prior_turn_closeout.ts +++ b/loopx/control_plane/quota/prior_turn_closeout.ts @@ -14,6 +14,7 @@ import { heartbeatReceiptBinding, heartbeatReceiptDetails, heartbeatReceiptFactFromEvent, + normalizeHeartbeatReplanObligationId, selectEffectiveHeartbeatReceipt, type HeartbeatReceiptBinding, type HeartbeatReceiptFact, @@ -30,12 +31,20 @@ export interface PriorHostTurnCloseoutCandidate extends JsonObject { /** The Todo id or autonomous replan obligation id, per `binding_kind`. */ binding_id: string; settlement_effect_id: string | null; + /** The effective guard admitted a replan under this Todo's settlement identity. */ + admitted_todo_replan?: true; +} + +interface CloseoutCandidateCheck { + candidate: PriorHostTurnCloseoutCandidate; + requireWritebackForRecovery: boolean; } function candidate( binding: HeartbeatReceiptBinding, turnInstanceId: string, fact: HeartbeatReceiptFact, + admittedTodoReplan: boolean, ): PriorHostTurnCloseoutCandidate { return { prior_turn_instance_id: turnInstanceId, @@ -43,21 +52,26 @@ function candidate( binding_kind: binding.binding_kind, binding_id: binding.binding_id, settlement_effect_id: binding.settlement_effect_id, + ...(admittedTodoReplan ? {admitted_todo_replan: true as const} : {}), }; } function admitsAutonomousReplan(event: JsonObject, fact: HeartbeatReceiptFact): boolean { - if (fact.replan_obligation_id === null || fact.settlement_effect_id === null) return false; const guard = heartbeatReceiptDetails(event); - return guard.semantic_replan_obligation_id === fact.replan_obligation_id + const obligationId = normalizeHeartbeatReplanObligationId(guard.semantic_replan_obligation_id); + return fact.settlement_effect_id !== null && obligationId !== null + && (fact.todo_id !== null || obligationId === fact.replan_obligation_id) && guard.interaction_mode === "autonomous_replan" && guard.ok === true && guard.should_run === true && guard.must_attempt_work === true && guard.delivery_allowed === false && guard.quiet_noop_allowed === false; } /** - * Select prior Turns eligible for closeout readback. Guard preflight requires - * an explicit closeout marker. Refresh recovery can also discover an admitted - * replan-only binding without making it a later Turn's admission prerequisite. + * Select prior Turns eligible for closeout readback. A Todo-bound replan + * carries the Todo settlement identity and a separate semantic obligation. + * It did not deliver a Todo, so its closeout marker can be false. Fresh + * admission requires a verified outstanding writeback for this binding. + * Explicit pending-closeout discovery can also find admitted replans before + * writeback, whether their settlement identity is a Todo or an obligation. * * Every prior Turn is identity-validated, so a conflicting receipt fails the * read instead of being hidden behind a newer Turn. Results are newest first @@ -74,8 +88,8 @@ export function selectCloseoutCandidates( goalId: string, agentId: string, excludeTurnInstanceId: string | null, - { includeAdmittedReplans } = { includeAdmittedReplans: false }, -): { candidates: PriorHostTurnCloseoutCandidate[]; turnsValidated: number } { + { includeUnwrittenReplans } = { includeUnwrittenReplans: false }, +): { candidates: CloseoutCandidateCheck[]; turnsValidated: number } { const perTurn = new Map(); const firstRecorded = new Map(); for (const event of receipts) { @@ -100,7 +114,7 @@ export function selectCloseoutCandidates( } } - const candidates: PriorHostTurnCloseoutCandidate[] = []; + const candidates: CloseoutCandidateCheck[] = []; const currentPosition = excludeTurnInstanceId === null ? undefined : firstRecorded.get(excludeTurnInstanceId); let validated = 0; @@ -112,10 +126,14 @@ export function selectCloseoutCandidates( // must not conceal a conflicting persisted settlement binding. if (currentPosition !== undefined && firstRecorded.get(turnInstanceId)! >= currentPosition) continue; const fact = heartbeatReceiptFactFromEvent(effective); - if (!fact.closeout_required && !(includeAdmittedReplans && admitsAutonomousReplan(effective, fact))) continue; const binding = heartbeatReceiptBinding(goalId, agentId, fact); if (binding === null) continue; - candidates.push(candidate(binding, turnInstanceId, fact)); + const admittedReplan = admitsAutonomousReplan(effective, fact); + const admittedTodoReplan = admittedReplan && binding.binding_kind === "todo"; + if (!fact.closeout_required && !admittedTodoReplan && + !(includeUnwrittenReplans && admittedReplan)) continue; + candidates.push({candidate: candidate(binding, turnInstanceId, fact, admittedTodoReplan), + requireWritebackForRecovery: !fact.closeout_required && !includeUnwrittenReplans}); } return {candidates, turnsValidated: validated}; } @@ -147,14 +165,18 @@ function bundleFailed(readback: JsonObject, step: string): boolean { /** * The closeout receipts a candidate's settlement readback still lacks, or - * null when that readback already validates the Turn's settlement. + * null when the Turn's settlement validates, or undefined when a candidate + * without a closeout marker has no verified writeback. */ -export function missingCloseoutReceipts(readback: JsonObject): string[] | null { +export function missingCloseoutReceipts( + readback: JsonObject, requireWriteback: boolean, +): string[] | null | undefined { if (readback.found === true && !bundleFailed(readback, "settlement")) { return null; } const missingReceipts: string[] = []; if (readback.found !== true || bundleFailed(readback, "writeback")) { + if (requireWriteback) return undefined; missingReceipts.push(WRITEBACK_RECEIPT); } if (readback.found !== true || bundleFailed(readback, "spend")) { @@ -162,3 +184,8 @@ export function missingCloseoutReceipts(readback: JsonObject): string[] | null { } return missingReceipts; } + +/** Only an exact writeback can leave the original debit as the sole debt. */ +export function awaitingQuotaSpend(missing: readonly string[]): boolean { + return missing.length === 1 && missing[0] === SPEND_RECEIPT; +} diff --git a/loopx/control_plane/quota/settlement.py b/loopx/control_plane/quota/settlement.py index 389bda1c0f..f0563d3fe5 100644 --- a/loopx/control_plane/quota/settlement.py +++ b/loopx/control_plane/quota/settlement.py @@ -152,6 +152,7 @@ class QuotaSpendPrecheck: settlement: SettlementResult[dict[str, Any]] progress: dict[str, Any] | None spend_run_present: bool + spend_request_rejection: SettlementResult[dict[str, Any]] | None @dataclass(frozen=True, slots=True) @@ -424,6 +425,7 @@ def read_quota_spend_precheck( allow_unbound_binding: bool = False, registry_path: Path | None = None, goal_ref: Mapping[str, Any] | None = None, + spend_request: Mapping[str, Any] | None = None, ) -> QuotaSpendPrecheck | None: """Read the identity and replay controls before building a fresh preview.""" @@ -438,6 +440,7 @@ def read_quota_spend_precheck( "infer_turn_instance_id": infer_turn_instance_id, "allow_unbound_binding": allow_unbound_binding, "spend_precheck": True, + **({"spend_request": dict(spend_request)} if spend_request is not None else {}), }, registry_path=registry_path, goal_ref=goal_ref, @@ -452,6 +455,10 @@ def read_quota_spend_precheck( settlement=_readback_result(payload.get("settlement")), progress=_optional_readback_record(payload.get("progress")), spend_run_present=spend_run_present, + spend_request_rejection=( + _readback_result(payload["spend_request_rejection"]) + if "spend_request_rejection" in payload else None + ), ) diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index 827f1b0ee4..10bd891ae3 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -22,6 +22,7 @@ import { } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; import { hasMonitorPollTransaction } from "./monitor_poll_commit.ts"; +import { readQuotaSpendEventFromRun } from "./accounting_artifact_transaction.ts"; import { jsonObject, requireJsonObject, @@ -1110,7 +1111,7 @@ function owedPriorTurnFromSnapshot( snapshot: QuotaSettlementReadbackSnapshot, agentId: string, turnInstanceId: string | null, - options?: { includeAdmittedReplans: boolean }, + options?: { includeUnwrittenReplans: boolean }, ): OwedPriorTurn | null { const receipts = heartbeatReceiptsFromEvents(snapshot.events, request.goal_id, agentId) .filter((event) => quotaOwnerOwnsProjection(request.owner, event.goal_ref)); @@ -1121,7 +1122,7 @@ function owedPriorTurnFromSnapshot( turnInstanceId, options, ); - for (const selected of candidates) { + for (const {candidate: selected, requireWritebackForRecovery} of candidates) { // A committed monitor poll is its own exact no-spend closeout. This is the // same acceptance rule the next-Turn preflight applies; using only the // advancement writeback/spend chain here incorrectly made every later @@ -1149,9 +1150,9 @@ function owedPriorTurnFromSnapshot( }, snapshot, false, - )) + ), requireWritebackForRecovery) : null; - if (missingReceipts !== null) { + if (missingReceipts != null) { return { ...selected, missing_receipts: missingReceipts }; } } @@ -1169,7 +1170,7 @@ function readPendingCloseoutFromRequest( } const notFound = {schema_version: QUOTA_SETTLEMENT_READBACK_RESULT_SCHEMA, found: false}; const turn = request.turn_instance_id ?? owedPriorTurnFromSnapshot( - request, snapshot, explicitAgentId, null, { includeAdmittedReplans: true }, + request, snapshot, explicitAgentId, null, { includeUnwrittenReplans: true }, )?.prior_turn_instance_id; if (!turn) return notFound; // Selection, original binding validation and receipt progress share this @@ -1489,10 +1490,17 @@ export async function readQuotaSettlement(value: unknown): Promise { return prepareBlockedWait(value); } const request = decodeRequest(value); - const spendPrecheck = jsonObject(value)?.spend_precheck; + const input = requireJsonObject(value, "quota settlement readback request"); + const spendPrecheck = input.spend_precheck; if (spendPrecheck !== undefined && typeof spendPrecheck !== "boolean") { throw new EffectRuntimeRequestError("spend_precheck must be a boolean"); } + const spendRequest = input.spend_request === undefined ? null + : requireJsonObject(input.spend_request, "spend_request"); + if (spendRequest !== null && (!spendPrecheck || + !Number.isSafeInteger(spendRequest.slots) || typeof spendRequest.source !== "string")) { + throw new EffectRuntimeRequestError("spend_request requires spend_precheck, integer slots and a source"); + } if (spendPrecheck) request.include_record_bodies = true; const withOwner = request.borrow_source_admission ? withBorrowedQuotaAccountingOwner @@ -1518,14 +1526,60 @@ export async function readQuotaSettlement(value: unknown): Promise { // only its receipts and whether a spend Run already needs receipt repair. const settlement = requireJsonObject(result.settlement, "quota settlement bundle"); const outcome = requireJsonObject(settlement.result, "quota settlement result"); + const spendRun = jsonObject(result.spend_run); + const spendRejection = spendRequest && identity + ? await originalSpendRejection(request, result, spendRequest, spendRun) + : null; return { schema_version: result.schema_version, found: true, identity: result.identity, settlement: {...settlement, result: {...outcome, value: null}}, progress: result.progress, - spend_run_present: jsonObject(result.spend_run) !== null, + spend_run_present: spendRun !== null, + ...(spendRejection === null ? {} : {spend_request_rejection: bundle(spendRejection)}), }; }, ); } + +async function originalSpendRejection( + request: ReadbackRequest, readback: JsonObject, proposed: JsonObject, + spendRun: JsonObject | null, +): Promise | null> { + const progress = requireJsonObject(readback.progress, "quota settlement progress"); + const guard = jsonObject(readback.heartbeat_receipt); + const authorizedSource = guard ? details(guard).quota_spend_source : null; + // Older ledgers can have a rollout receipt but no readable artifact. Keep + // their receipt facts available and never repair an already mismatched debit. + const spendEvent = jsonObject(readback.spend_event); + const receipt = spendEvent ? details(spendEvent) : {}; + const retained = spendRun === null ? null : await readQuotaSpendEventFromRun( + join(request.runtime_root, "goals", request.goal_id, "runs"), spendRun, request.goal_id, + ); + const recordedSlots = retained?.slots ?? (receipt.slots === "" ? undefined : receipt.slots); + const recordedSource = retained?.source ?? (receipt.source === "" ? undefined : receipt.source); + const fallbackSource = recordedSource === "heartbeat" || recordedSource === "visible-goal" + ? recordedSource : progress.quota_spend_source; + const source = authorizedSource ?? fallbackSource; + // Every admitted CLI settlement plan authorizes one slot and the original + // source. The same check gates initial spend, receipt repair, and replay. + if (proposed.slots !== 1 || proposed.source !== source) { + return settlementFailed({ + kind: "quota_spend_rejected", step_kind: "quota_spend", + reason: `original Turn settlement requires --slots 1 --source ${String(source)}`, + }); + } + const slotsConflict = recordedSlots !== undefined && recordedSlots !== proposed.slots; + const sourceConflict = recordedSource !== undefined && recordedSource !== source; + // Historical complete receipts may lack the amount projection. A partial + // debit needs the original charge facts before we can certify its receipt. + const unverifiableRepair = spendRun !== null && spendEvent === null && + (recordedSlots === undefined || recordedSource === undefined); + return slotsConflict || sourceConflict || unverifiableRepair ? settlementFailed({ + kind: "quota_spend_rejected", step_kind: "quota_spend", + reason: unverifiableRepair + ? "the recorded debit has no verifiable billing terms for receipt repair" + : "the recorded debit differs from the original Turn settlement plan", + }) : null; +} diff --git a/loopx/control_plane/quota/spend_commit.ts b/loopx/control_plane/quota/spend_commit.ts index 7a4f7eab4a..4f201a0190 100644 --- a/loopx/control_plane/quota/spend_commit.ts +++ b/loopx/control_plane/quota/spend_commit.ts @@ -361,6 +361,43 @@ function requestDigest(request: QuotaSpendCommitRequest): string { })); } +function matchesRetainedSettlementSpend( + request: QuotaSpendCommitRequest, + proposedRecord: JsonObject, + retainedRecord: JsonObject, +): boolean { + const identity = jsonObject(proposedRecord.settlement_identity); + const turnId = optionalString(identity?.turn_instance_id, "settlement_identity.turn_instance_id"); + const effectId = optionalString(identity?.effect_id, "settlement_identity.effect_id"); + if (!turnId || !effectId || + (request.source !== "heartbeat" && request.source !== "visible-goal") || + request.preview.effect_ref != null || request.preview.delivery_completion_spend !== true || + request.effect_id !== `${effectId}#quota_spend` || + identity?.goal_id !== request.goal_id || identity?.agent_id !== request.resolved_agent_id || + identity?.turn_instance_id !== proposedRecord.turn_instance_id || + (identity?.todo_id || null) !== (proposedRecord.todo_id ?? null) || + (identity?.replan_obligation_id ?? null) !== (proposedRecord.replan_obligation_id ?? null)) { + return false; + } + const proposed = requiredObject(proposedRecord.quota_event, "proposed spend event"); + const retained = requiredObject(retainedRecord.quota_event, "retained spend event"); + // The live quota projection can change between two attempts to settle one + // validated writeback. The original identity and accounting facts cannot. + return retainedRecord.classification === QUOTA_SLOT_SPENT_CLASSIFICATION && + retainedRecord.goal_id === request.goal_id && + retainedRecord.agent_id === request.resolved_agent_id && + canonicalJson(retainedRecord.settlement_identity) === canonicalJson(identity) && + retained.source === proposed.source && retained.slots === proposed.slots && + typeof proposed.delivery_run_generated_at === "string" && + typeof proposed.delivery_run_classification === "string" && + proposed.delivery_run_agent_id === identity.agent_id && + retained.delivery_run_generated_at === proposed.delivery_run_generated_at && + retained.delivery_run_classification === proposed.delivery_run_classification && + retained.delivery_run_agent_id === proposed.delivery_run_agent_id && + canonicalJson(retained.delivery_workspace) === canonicalJson(proposed.delivery_workspace) && + retained.delivery_workspace_validated === proposed.delivery_workspace_validated; +} + function positiveSlots(preview: JsonObject): number { const value = preview.slots; if (typeof value !== "number" || !Number.isInteger(value) || value < 1) { @@ -851,6 +888,7 @@ export async function evaluateQuotaSpendCommit( expectedIndexDigest: request.expected_index_digest, ...(goalRef === null ? {} : { goalRef }), indexLockHeld, + matchesRetainedRequest: (retained) => matchesRetainedSettlementSpend(request, record, retained), prepare: ({ jsonPath, markdownPath, indexPath: lockedIndexPath }) => { const payload = payloadFor( request, @@ -915,6 +953,7 @@ export async function evaluateQuotaSpendCommit( appended: repaired, idempotent_replay: !repaired, transaction_repaired: repaired, + ...(outcome.receipt.record.settlement_identity ? {receipt_repair_required: true} : {}), reason: repaired ? "quota spend commit repaired its prepared durable transaction" : "quota spend commit replayed for the same effect identity", diff --git a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts index 9cb4f9b55d..935729ec1f 100644 --- a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts +++ b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts @@ -33,6 +33,7 @@ import { type HeartbeatReceiptBinding, } from "./heartbeat_receipt_identity.ts"; import { + awaitingQuotaSpend, candidateReadbackBinding, MISSING_RECEIPT_NAMES, missingCloseoutReceipts, @@ -164,27 +165,21 @@ async function preflightPriorHostTurnCloseoutForOwner( request.agent_id, request.exclude_turn_instance_id, ); - if (candidates.length === 0) { - return { - schema_version: PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_RESULT_SCHEMA, - status: "none", - reason: "no_prior_turn_requires_closeout", - turns_validated: turnsValidated, - }; - } - const settlementSnapshot = await readQuotaSettlementSnapshot( - request.runtime_root, - request.goal_id, - rolloutSnapshot, - ); + // Resolve one snapshot on the first candidate. The empty path needs only + // the guard log, and unwritten replans may all drop out after validation. + let settlementSnapshot: Awaited> | undefined; let newestSettledTurn: string | null = null; let newestAcceptedCloseout: AcceptedCloseout = "validated_writeback_and_quota_spend"; - for (const selected of candidates) { + for (const {candidate: selected, requireWritebackForRecovery} of candidates) { + settlementSnapshot ??= await readQuotaSettlementSnapshot( + request.runtime_root, request.goal_id, rolloutSnapshot, + ); const readback = readAdmittedQuotaSettlementFromSnapshot( settlementReadbackRequest(request, selected), settlementSnapshot, ); - const missingReceipts = missingCloseoutReceipts(readback); + const missingReceipts = missingCloseoutReceipts(readback, requireWritebackForRecovery); + if (missingReceipts === undefined) continue; if (missingReceipts === null) { // A newer settled Turn cannot hide an older missing closeout. Keep // scanning in persisted newest-first order until the first unsettled @@ -218,10 +213,12 @@ async function preflightPriorHostTurnCloseoutForOwner( return { schema_version: PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_RESULT_SCHEMA, status: "none", - reason: "prior_turn_settlement_validated", turns_validated: turnsValidated, - prior_turn_instance_id: newestSettledTurn, - accepted_closeout: newestAcceptedCloseout, + ...(newestSettledTurn === null ? {reason: "no_prior_turn_requires_closeout"} : { + reason: "prior_turn_settlement_validated", + prior_turn_instance_id: newestSettledTurn, + accepted_closeout: newestAcceptedCloseout, + }), }; } @@ -257,6 +254,13 @@ function decodeCandidate(value: unknown): PriorHostTurnCloseoutCandidate { "prior host Turn recovery candidate autonomous replan binding is malformed", ); } + const settlementEffectId = optionalHeartbeatString(raw.settlement_effect_id); + if (raw.admitted_todo_replan !== undefined && + (raw.admitted_todo_replan !== true || bindingKind !== "todo" || settlementEffectId === null)) { + throw new EffectRuntimeRequestError( + "prior host Turn recovery candidate has an invalid Todo-bound replan fact", + ); + } return { prior_turn_instance_id: requireNonEmptyString( raw.prior_turn_instance_id, @@ -265,7 +269,8 @@ function decodeCandidate(value: unknown): PriorHostTurnCloseoutCandidate { event_id: optionalHeartbeatString(raw.event_id), binding_kind: bindingKind, binding_id: bindingId, - settlement_effect_id: optionalHeartbeatString(raw.settlement_effect_id), + settlement_effect_id: settlementEffectId, + ...(raw.admitted_todo_replan === true ? {admitted_todo_replan: true as const} : {}), }; } @@ -368,8 +373,9 @@ function recoveryRepair( bindingKind: HeartbeatReceiptBinding["binding_kind"], todo: CandidateTodoFacts | null, ): RecoveryRepair { - if (bindingKind !== "todo" || todo === null) return "lifecycle"; - if (todo.task_class === MONITOR_TASK_CLASS) return "monitor_poll"; + if (bindingKind !== "todo") return "lifecycle"; + if (todo?.task_class === MONITOR_TASK_CLASS) return "monitor_poll"; + if (todo === null) return "lifecycle"; // Missing receipts do not prove an external dependency. Re-enter // the original guard to recheck eligibility and retain its settlement id. // This is a recovery route, never an accepted closeout or delivery grant. @@ -420,7 +426,9 @@ function recoveryObligation( * * The accepted closeout order is policy: a validated settlement wins, then an * exactly committed monitor-poll closeout, then a typed lifecycle transition. - * Anything else is a recovery obligation with the exact missing receipts named. + * For a Todo-bound autonomous replan, a verified writeback keeps its debit + * obligation even when the Todo has changed. Ordinary lifecycle closeouts + * retain their existing meaning. */ export function reduceUnsettledHostTurnRecovery(value: unknown): JsonObject { const request = requireJsonObject(value, "prior host Turn recovery request"); @@ -465,7 +473,8 @@ export function reduceUnsettledHostTurnRecovery(value: unknown): JsonObject { }; } - const lifecycleCloseout = acceptedLifecycleCloseout(todo); + const replanSpendOwed = selected.admitted_todo_replan === true && awaitingQuotaSpend(missingReceipts); + const lifecycleCloseout = replanSpendOwed ? null : acceptedLifecycleCloseout(todo); if (lifecycleCloseout !== null) { return { schema_version: UNSETTLED_HOST_TURN_RECOVERY_RESULT_SCHEMA, @@ -494,7 +503,7 @@ export function reduceUnsettledHostTurnRecovery(value: unknown): JsonObject { recovery.binding_target_key = todo.target_key; recovery.binding_cadence = todo.cadence; } - const repair = recoveryRepair(selected.binding_kind, todo); + const repair = replanSpendOwed ? "resume_prior_turn" : recoveryRepair(selected.binding_kind, todo); // The typed repair lane lets the CLI renderer project its command list // without re-deriving which closeout family the Turn belongs to. recovery.repair = repair; diff --git a/loopx/control_plane/quota/void_commit.ts b/loopx/control_plane/quota/void_commit.ts index abb056decf..ba49176d59 100644 --- a/loopx/control_plane/quota/void_commit.ts +++ b/loopx/control_plane/quota/void_commit.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import { readFile, realpath } from "node:fs/promises"; -import { basename, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { readFile } from "node:fs/promises"; +import { basename, isAbsolute, join } from "node:path"; import type { JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; @@ -9,6 +9,7 @@ import { nextQuotaAccountingArtifactPaths, parseQuotaAccountingIndex, quotaAccountingIndexDigest, + readQuotaSpendEventFromRun, renderQuotaSlotMarkdown, type QuotaAccountingArtifactPrepareContext, type QuotaAccountingArtifactPreparation, @@ -365,59 +366,6 @@ function compactDecision(value: unknown): JsonObject { }; } -async function readTargetEvent( - runsDir: string, - run: JsonObject, - goalId: string, -): Promise { - const inline = jsonObject(run.quota_event); - if (inline) return inline; - if (typeof run.json_path !== "string" || !run.json_path.trim()) return null; - let targetPath: string; - try { - targetPath = await realpath(resolve(run.json_path.trim())); - } catch (error) { - if (error instanceof Error && "code" in error && error.code === "ENOENT") { - return null; - } - throw error; - } - const relativePath = relative(await realpath(runsDir), targetPath); - if ( - !relativePath || - relativePath === ".." || - relativePath.startsWith(`..${sep}`) || - isAbsolute(relativePath) - ) { - throw new EffectRuntimeRequestError( - "quota spend json_path must stay inside the goal runs directory", - ); - } - let content: string; - try { - content = await readFile(targetPath, "utf8"); - } catch (error) { - if (error instanceof Error && "code" in error && error.code === "ENOENT") return null; - throw error; - } - let value: unknown; - try { - value = JSON.parse(content); - } catch { - throw new EffectRuntimeRequestError("quota spend JSON artifact is malformed"); - } - const record = requiredObject(value, "quota spend JSON artifact"); - if ( - record.classification !== QUOTA_SLOT_SPENT_CLASSIFICATION || - (record.goal_id !== undefined && record.goal_id !== goalId) - ) { - throw new EffectRuntimeRequestError( - "quota spend JSON artifact identity does not match its index row", - ); - } - return jsonObject(record.quota_event); -} - async function findTargetSpend( runsDir: string, records: readonly JsonObject[], @@ -429,7 +377,7 @@ async function findTargetSpend( if (String(run.goal_id || goalId) !== goalId) continue; if (String(run.generated_at ?? "") !== generatedAt) continue; if (run.classification !== QUOTA_SLOT_SPENT_CLASSIFICATION) continue; - const event = await readTargetEvent(runsDir, run, goalId); + const event = await readQuotaSpendEventFromRun(runsDir, run, goalId); if (event?.event_type !== QUOTA_SLOT_SPENT_CLASSIFICATION) continue; requireQuotaOwnerProjection(owner, run.goal_ref, "quota void target"); requireQuotaOwnerProjection( diff --git a/loopx/quota.py b/loopx/quota.py index 450115daaf..3836b4b82d 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1642,7 +1642,7 @@ def spend_quota_slot( "goal_id": safe_goal_id, "reason": "effect_ref must be a non-empty string when provided", } - if normalized_effect_ref: + if normalized_effect_ref and not turn_instance_id: if not raw_runtime_root: return { "ok": False, @@ -1709,6 +1709,7 @@ def spend_quota_slot( and not replan_obligation_id ) settlement_readback = None + spend_request = {"slots": slots, "source": source} if ( not turn_instance_id and raw_runtime_root @@ -1727,6 +1728,7 @@ def spend_quota_slot( allow_unbound_binding=recover_unbound_visible_goal, registry_path=registry_path, goal_ref=goal_ref, + spend_request=spend_request, ) inferred_result = ( settlement_readback.identity @@ -1765,6 +1767,7 @@ def spend_quota_slot( replan_obligation_id=replan_obligation_id, registry_path=registry_path, goal_ref=goal_ref, + spend_request=spend_request, ) if settlement_readback is None: raise RuntimeError("exact settlement readback unexpectedly returned not-found") @@ -1784,6 +1787,18 @@ def spend_quota_slot( "settlement_result": settlement_result_payload(guard_result), } identity = guard_result.value + refusal = settlement_readback.spend_request_rejection + if refusal is not None: + return { + "ok": False, + "mode": "spend-slot", + "dry_run": not execute, + "appended": False, + "goal_id": safe_goal_id, + "settlement_identity": identity.as_dict(), + "settlement_result": settlement_result_payload(refusal), + "reason": refusal.failure.reason if refusal.failure else "invalid original Turn spend", + } spent_result = settlement_readback.settlement if spent_result.failure is None: return { diff --git a/tests/control_plane/test_quota_settlement_cli.py b/tests/control_plane/test_quota_settlement_cli.py index e747385d8c..e5a420ed67 100644 --- a/tests/control_plane/test_quota_settlement_cli.py +++ b/tests/control_plane/test_quota_settlement_cli.py @@ -7136,8 +7136,9 @@ def test_unbound_visible_goal_todoless_replan_reenters_through_guided_turn( assert {path: path.read_bytes() for path in source_paths} == original_files +@pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_autonomous_replan_semantic_delta_keeps_accountable_receipt_chain( - tmp_path: Path, + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, ) -> None: """Regression for #3528's real Todo-bound replan path. @@ -7147,12 +7148,15 @@ def test_autonomous_replan_semantic_delta_keeps_accountable_receipt_chain( instead of being mislabeled as a noop after it discharges the obligation. """ + from canonical_authority_fixture import isolate_sqlite_runtime + + isolate_sqlite_runtime(tmp_path, monkeypatch) project, runtime, registry_path = _write_fixture(tmp_path) - _configure_selected_todo_replan_fixture(project, registry_path) + _configure_selected_todo_replan_fixture(project, registry_path, stride_strategy="off") _initialize_git_checkout(project) turn_instance_id = "turn-autonomous-replan-semantic-1" - _initialize_fixture_authority(project, runtime) + _initialize_fixture_authority(project, runtime, provider=provider) guard_rc, guard = _run_cli( registry_path, runtime, @@ -7235,6 +7239,9 @@ def test_autonomous_replan_semantic_delta_keeps_accountable_receipt_chain( assert [ receipt["step_kind"] for receipt in refresh["settlement_result"]["receipts"] ] == ["validation", "durable_writeback"] + original = guard["heartbeat_receipt"]["settlement_identity"] + assert refresh["settlement_identity"] == original + assert _spend_run_count(runtime) == 0 replay_rc, replay = _run_cli( registry_path, @@ -7246,23 +7253,98 @@ def test_autonomous_replan_semantic_delta_keeps_accountable_receipt_chain( assert replay["appended"] is False assert _classification_count(runtime, "validated_progress") == 1 - next_guard_rc, next_guard = _run_cli( - registry_path, - runtime, - "quota", - "should-run", - "--codex-app", - "--goal-id", - GOAL_ID, - "--agent-id", - AGENT_ID, - "--turn-instance-id", - "turn-after-autonomous-replan-semantic-1", - ) + # A new guard can see the replan has ended before the original debit is + # durable. It must still expose recovery instead of another execution. + next_turn = "turn-after-autonomous-replan-semantic-1" + next_args = ("quota", "should-run", "--codex-app", "--goal-id", GOAL_ID, + "--agent-id", AGENT_ID, "--turn-instance-id", next_turn) + next_guard_rc, next_guard = _run_cli(registry_path, runtime, *next_args) assert next_guard_rc == 0, next_guard - assert next_guard.get("autonomous_replan_obligation") is None - assert next_guard.get("decision") != "autonomous_replan_required" - assert obligation_id not in json.dumps(next_guard, sort_keys=True) + assert next_guard["effective_action"] == "unsettled_host_turn_recovery", next_guard + assert next_guard["normal_delivery_allowed"] is False + recovery = next_guard["unsettled_host_turn_recovery"] + assert recovery["prior_turn_instance_id"] == turn_instance_id + assert recovery["binding_id"] == SELECTED_REPLAN_TODO_ID + assert recovery["settlement_effect_id"] == original["effect_id"] + assert recovery["missing_receipts"] == ["quota_spend_receipt"] + assert all(not row["details"].get("settlement_effect_id") + for row in _heartbeat_receipt_events(runtime, next_turn)) + + explicit_rc, explicit = _run_cli(registry_path, runtime, *next_args, "--todo-id", SELECTED_REPLAN_TODO_ID) + assert explicit_rc == 0 and explicit["effective_action"] == "unsettled_host_turn_recovery", explicit + assert explicit["unsettled_host_turn_recovery"] == recovery + assert "settlement_identity" not in explicit["heartbeat_receipt"] + actions = explicit["interaction_contract"]["cli_channel"]["next_cli_actions"] + original_guard = next(command for command in actions if turn_instance_id in shlex.split(command)) + rc, recovering = _run_generated_cli(original_guard, registry_path=registry_path, cwd=project) + assert rc == 0 and recovering["normal_delivery_allowed"] is False, recovering + owed = recovering["settlement_owed"] + assert owed["command"] == refresh["settlement_owed"]["command"] + assert owed["effect_id"] == original["effect_id"] + + def reject_changed_spend() -> None: + for flag, replacement in (("--slots", "2"), ("--source", "visible-goal")): + tokens = shlex.split(owed["command"]) + tokens[tokens.index(flag) + 1] = replacement + refused_rc, refused = _run_generated_cli( + shlex.join(tokens), registry_path=registry_path, cwd=project, + ) + assert refused_rc == 1 and refused["ok"] is False, refused + assert refused["appended"] is False + assert refused["settlement_result"]["failure"]["kind"] == "quota_spend_rejected" + + reject_changed_spend() + assert _spend_run_count(runtime) == 0 + from concurrent.futures import ThreadPoolExecutor + + def settle() -> dict[str, Any]: + rc, result = _run_generated_cli(owed["command"], registry_path=registry_path, cwd=project) + if rc != 0: + pytest.fail(f"Original settlement failed: {result.get('error_code')}: {result.get('reason') or result.get('error')}") + assert result["settlement_identity"] == original + return result + + with ThreadPoolExecutor(max_workers=2) as pool: + results = [pool.submit(settle), pool.submit(settle)] + attempts = [result.result() for result in results] + assert sorted(attempt["appended"] for attempt in attempts) == [False, True] + assert all( + attempt["settlement_result"]["ok"] + and [receipt["step_kind"] for receipt in attempt["settlement_result"]["receipts"]] + == ["validation", "durable_writeback", "quota_spend"] + for attempt in attempts + ) + rollout_log = runtime / "goals" / GOAL_ID / "rollout-event-log.jsonl" + events = [json.loads(line) for line in rollout_log.read_text(encoding="utf-8").splitlines()] + assert sum( + event["event_kind"] == "quota_spend" and event.get("run_id") == turn_instance_id + for event in events + ) == 1 + assert _spend_run_count(runtime) == 1 + reject_changed_spend() + # Simulate the debit committing before its rollout receipt. Refused + # requests must not fill the missing receipt with different billing terms. + partial_log = "".join(json.dumps(event) + "\n" for event in events if event["event_kind"] != "quota_spend") + rollout_log.write_text(partial_log, encoding="utf-8") + reject_changed_spend() + assert rollout_log.read_text(encoding="utf-8") == partial_log + repaired = settle() + assert repaired["appended"] is False and repaired["receipt_repaired"] is True + assert repaired["settlement_result"]["ok"] is True + repaired_events = [json.loads(line) for line in rollout_log.read_text(encoding="utf-8").splitlines()] + assert sum( + event["event_kind"] == "quota_spend" + and event["details"]["settlement_effect_id"] == original["effect_id"] + for event in repaired_events + ) == 1 + assert _spend_run_count(runtime) == 1 + assert _classification_count(runtime, "validated_progress") == 1 + rc, admitted = _run_cli(registry_path, runtime, *next_args, "--todo-id", SELECTED_REPLAN_TODO_ID) + assert rc == 0 and admitted["normal_delivery_allowed"], admitted + assert admitted["heartbeat_receipt"]["settlement_identity"]["turn_instance_id"] == next_turn + assert admitted["heartbeat_receipt"]["settlement_identity"]["todo_id"] == SELECTED_REPLAN_TODO_ID + assert admitted["heartbeat_receipt"]["settlement_identity"]["effect_id"] != original["effect_id"] + assert obligation_id not in json.dumps(admitted, sort_keys=True) def test_open_replan_rejects_missing_semantic_delta_before_durable_write( diff --git a/tests/control_plane_ts/quota_spend_commit.test.ts b/tests/control_plane_ts/quota_spend_commit.test.ts index 553c25ed95..dac67a0493 100644 --- a/tests/control_plane_ts/quota_spend_commit.test.ts +++ b/tests/control_plane_ts/quota_spend_commit.test.ts @@ -23,6 +23,7 @@ import { EffectRuntimeRequestError, } from "../../loopx/control_plane/effect_runtime_errors.ts"; import { requireJsonObject } from "../../loopx/control_plane/runtime_decode.ts"; +import { settlementIdentityPayload } from "../../loopx/control_plane/effect_program.ts"; import { parseQuotaAccountingOwner, withQuotaAccountingOwner, @@ -1087,6 +1088,52 @@ test("effect identity and index CAS reject drift and racing writers", async (t) assert.equal(fresh.status, "written"); }); +test("concurrent settlement retries share one debit across quota observation changes", async (t) => { + const runtimeRoot = await tempRuntime(t); + const identity = settlementIdentityPayload({ + goal_id: goalId, agent_id: "codex-main-control", todo_id: "todo_original", + turn_instance_id: "turn-original", + }); + const stable = { + settlement_identity: identity, + turn_instance_id: identity.turn_instance_id, + todo_id: identity.todo_id, + delivery_completion_spend: true, + delivery_run_generated_at: "2026-08-25T11:59:00+08:00", + delivery_run_classification: "validated_progress", + delivery_run_agent_id: identity.agent_id, + delivery_workspace: {workspace_identity: "git:github.com/example/fixture"}, + delivery_workspace_validated: true, + }; + const spendRequest = (spentSlots: number, slots = 1) => request(runtimeRoot, { + effect_id: null, + preview: preview({...stable, slots, would_throttle: spentSlots > 0}), + before: decision(spentSlots), + after: decision(spentSlots + slots), + }); + const outcomes = await Promise.all([ + evaluateQuotaSpendCommit(spendRequest(0)), + evaluateQuotaSpendCommit(spendRequest(1)), + ]); + assert.deepEqual(outcomes.map((result) => result.status).sort(), ["replayed", "written"]); + const committed = outcomes.find((result) => result.written)!; + const indexPath = String(committed.payload.index_path); + const originalRows = await readFile(indexPath, "utf8"); + assert.equal(originalRows.trim().split("\n").length, 1); + assert.deepEqual(outcomes[0]?.record, outcomes[1]?.record); + + const changedSpend = await evaluateQuotaSpendCommit(spendRequest(1, 2)); + assert.equal(changedSpend.reason_code, "effect_id_conflict"); + const changedSource = await evaluateQuotaSpendCommit({...spendRequest(1), source: "visible-goal"}); + assert.equal(changedSource.reason_code, "effect_id_conflict"); + const replacedWriteback = await evaluateQuotaSpendCommit({ + ...spendRequest(1), + preview: preview({...stable, delivery_run_generated_at: "2026-08-25T11:58:00+08:00"}), + }); + assert.equal(replacedWriteback.reason_code, "effect_id_conflict"); + assert.equal(await readFile(indexPath, "utf8"), originalRows); +}); + test("concurrent exact-effect retries serialize to one spend", async (t) => { const runtimeRoot = await tempRuntime(t); const params = request(runtimeRoot); diff --git a/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts b/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts index 8287c5fc62..e787055478 100644 --- a/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts +++ b/tests/control_plane_ts/unsettled_host_turn_recovery.test.ts @@ -413,6 +413,27 @@ test("a receipt that does not opt in never becomes a recovery obligation", async } }); +test("an admitted Todo-bound replan without verified writeback does not hold a later guard", async () => { + const admitted = { + ...closeoutRequired("turn-replan", "todo_alpha"), closeout_required: false, + semantic_replan_obligation_id: `replan-${"b".repeat(16)}`, + ok: true, should_run: true, interaction_mode: "autonomous_replan", + must_attempt_work: true, delivery_allowed: false, quiet_noop_allowed: false, + }; + const runtime = await runtimeWith([ + receipt("turn-normal", closeoutRequired("turn-normal", "todo_beta")), + receipt("turn-replan", admitted), + ]); + const replanOnly = await runtimeWith([receipt("turn-replan", admitted)]); + try { + assert.equal(candidateFrom(await preflight(runtime.root)).prior_turn_instance_id, "turn-normal"); + assert.equal((await preflight(replanOnly.root)).reason, "no_prior_turn_requires_closeout"); + } finally { + await runtime.close(); + await replanOnly.close(); + } +}); + test("the current Turn, other goals, other agents, and other event kinds are ignored", async () => { const runtime = await runtimeWith([ receipt("current-turn", closeoutRequired("current-turn", "todo_current")), @@ -852,6 +873,17 @@ test("typed lifecycle closeouts are accepted and other open Todos are not", asyn assert.equal(verdict.status, "none"); assert.equal(verdict.accepted_closeout, accepted); } + const ordinaryPending = reduce(candidate, ["quota_spend_receipt"], READ_TODO(todo)); + assert.equal(ordinaryPending.status, verdict.status); + assert.equal(ordinaryPending.accepted_closeout, verdict.accepted_closeout); + + const replanPending = reduce( + {...candidate, admitted_todo_replan: true}, + ["quota_spend_receipt"], + READ_TODO(todo), + ); + assert.equal(replanPending.status, "recovery_required"); + assert.equal((replanPending.recovery as JsonObject).repair, "resume_prior_turn"); } } finally { await runtime.close();