Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions loopx/cli_commands/quota.py
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,13 @@ def _record_non_guard_quota_event(
),
details=rollout_details,
allow_failed=args.quota_command == "should-run",
idempotency_fields=(
["goal_id", "event_kind", "agent_id", "run_id"]
+ (["todo_id"] if rollout_todo_id else [])
+ (["goal_ref"] if goal_ref is not None else [])
if args.quota_command == "spend-slot" and spend_turn_instance_id
else None
),
)
if args.quota_command == "spend-slot" and spend_turn_instance_id:
assert context is not None # A settlement requires a prepared authority context.
Expand Down
5 changes: 5 additions & 0 deletions loopx/cli_commands/quota_action_selection.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from dataclasses import dataclass
from pathlib import Path

from ..control_plane.quota.effective_action import EffectiveAction
from ..control_plane.quota.error_codes import (
HeartbeatReceiptIdentityConflictError,
QuotaActionSelectionConflictError,
Expand Down Expand Up @@ -166,6 +167,10 @@ def _requested_quota_action_selection_preflight(
"current selected Todo: explicitly requested Todo differs"
)
return None
if payload.get("effective_action") == EffectiveAction.UNSETTLED_HOST_TURN_RECOVERY.value:
# Recovery preempts the choice. Keep the exact recovery actions in the
# response; the rollout owner leaves this Turn unbound until reentry.
return None
interaction = payload.get("interaction_contract")
agent_channel = (
interaction.get("agent_channel") if isinstance(interaction, Mapping) else None
Expand Down
5 changes: 5 additions & 0 deletions loopx/cli_commands/turn_run_once.py
Original file line number Diff line number Diff line change
Expand Up @@ -872,6 +872,11 @@ def on_managed_start_admitted() -> None:
semantic_replan_guard_scoped=replan_guard_scoped,
semantic_replan_obligation_id=replan_obligation_id,
semantic_replan_capability_guard=capability_guard,
quota_spend_source=(
quota_spend_source_for_execution_context(payload.get("scheduler_execution_context"))
if settlement_identity.replan_obligation_id or settlement_identity.request_subject is not None
else None
),
goal_ref=goal_ref,
)

Expand Down
61 changes: 58 additions & 3 deletions loopx/control_plane/quota/accounting_artifact_transaction.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createHash } from "node:crypto";
import { access, lstat, readFile } from "node:fs/promises";
import { basename, dirname, extname, join, resolve } from "node:path";
import { access, lstat, readFile, realpath } from "node:fs/promises";
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";

import type { JsonObject } from "../effect_program.ts";
import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts";
Expand Down Expand Up @@ -109,6 +109,7 @@ export interface QuotaAccountingArtifactCommitRequest {
expectedIndexDigest: string | null;
goalRef?: JsonObject;
indexLockHeld?: boolean;
matchesRetainedRequest?: (record: JsonObject) => boolean;
prepare: (
context: QuotaAccountingArtifactPrepareContext,
) =>
Expand Down Expand Up @@ -507,6 +508,59 @@ export function resolveQuotaAccountingEffect(
return { kind: "absent" };
}

export async function readQuotaSpendEventFromRun(
runsDir: string,
run: JsonObject,
goalId: string,
): Promise<JsonObject | null> {
const inline = jsonObject(run.quota_event);
if (inline) return inline;
if (typeof run.json_path !== "string" || !run.json_path.trim()) return null;
let targetPath: string;
try {
targetPath = await realpath(resolve(run.json_path.trim()));
} catch (error) {
if (error instanceof Error && "code" in error && error.code === "ENOENT") {
return null;
}
throw error;
}
const relativePath = relative(await realpath(runsDir), targetPath);
if (
!relativePath ||
relativePath === ".." ||
relativePath.startsWith(`..${sep}`) ||
isAbsolute(relativePath)
) {
throw new EffectRuntimeRequestError(
"quota spend json_path must stay inside the goal runs directory",
);
}
let content: string;
try {
content = await readFile(targetPath, "utf8");
} catch (error) {
if (error instanceof Error && "code" in error && error.code === "ENOENT") return null;
throw error;
}
let value: unknown;
try {
value = JSON.parse(content);
} catch {
throw new EffectRuntimeRequestError("quota spend JSON artifact is malformed");
}
const record = requiredObject(value, "quota spend JSON artifact");
if (
record.classification !== contractFor("spend").classification ||
(record.goal_id !== undefined && record.goal_id !== goalId)
) {
throw new EffectRuntimeRequestError(
"quota spend JSON artifact identity does not match its index row",
);
}
return jsonObject(record.quota_event);
}

export async function lookupQuotaAccountingReplay(
kind: QuotaAccountingArtifactKind,
indexPath: string,
Expand Down Expand Up @@ -920,7 +974,8 @@ export async function commitQuotaAccountingArtifactTransaction(
}
if (
existingReceipt.effect_id !== request.effectId ||
existingReceipt.request_digest !== request.requestDigest
(existingReceipt.request_digest !== request.requestDigest &&
request.matchesRetainedRequest?.(existingReceipt.record) !== true)
) {
return {
status: "conflict",
Expand Down
15 changes: 15 additions & 0 deletions loopx/control_plane/quota/heartbeat_receipt.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from .settlement_workspace_causality import (
delivery_workspace_causality_from_event_details,
)
from .spend_sources import TURN_SCOPED_SLOT_SPEND_SOURCES

HEARTBEAT_RECEIPT_SCHEMA_VERSION = "heartbeat_quota_receipt_v0"

Expand Down Expand Up @@ -204,6 +205,7 @@ def ensure_turn_heartbeat_settlement_receipt(
semantic_replan_guard_scoped: bool,
semantic_replan_obligation_id: str | None,
semantic_replan_capability_guard: Mapping[str, object] | None = None,
quota_spend_source: str | None = None,
goal_ref: Mapping[str, object] | None = None,
before_bind: Callable[[], None] | None = None,
) -> dict[str, object]:
Expand All @@ -224,6 +226,8 @@ def ensure_turn_heartbeat_settlement_receipt(
):
raise ValueError("Turn semantic replan obligation id is malformed")

if quota_spend_source is not None and quota_spend_source not in TURN_SCOPED_SLOT_SPEND_SOURCES:
raise ValueError("Turn quota spend source must be heartbeat or visible-goal")
log_path = rollout_event_log_path(runtime_root, identity.goal_id)
log_path.parent.mkdir(parents=True, exist_ok=True)
with exclusive_cross_runtime_file_lock(
Expand All @@ -248,6 +252,15 @@ def ensure_turn_heartbeat_settlement_receipt(
raise HeartbeatReceiptIdentityConflictError(
"Turn heartbeat receipt belongs to another settlement identity"
)
previous_details = effective.get("details") if effective is not None else None
previous_source = (
previous_details.get("quota_spend_source")
if observed is not None and isinstance(previous_details, Mapping) else None
)
if previous_source is not None and quota_spend_source not in (None, previous_source):
raise HeartbeatReceiptIdentityConflictError(
"Turn heartbeat receipt belongs to another quota spend source"
)
if before_bind is not None:
# The existing cross-runtime binding lock serializes the competing
# identity before transport-specific start admission. Its owner may
Expand Down Expand Up @@ -311,6 +324,8 @@ def ensure_turn_heartbeat_settlement_receipt(
details["semantic_replan_capability_id"] = semantic_replan_capability_guard["capability_id"]
details["semantic_replan_gap_id"] = semantic_replan_capability_guard["gap_id"]
details["semantic_replan_frontier_revision"] = semantic_replan_capability_guard["frontier_revision"]
if (source := previous_source or quota_spend_source) is not None:
details["quota_spend_source"] = source
source_event_id = (
str(effective.get("event_id") or "").strip()
if effective is not None
Expand Down
51 changes: 39 additions & 12 deletions loopx/control_plane/quota/prior_turn_closeout.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
heartbeatReceiptBinding,
heartbeatReceiptDetails,
heartbeatReceiptFactFromEvent,
normalizeHeartbeatReplanObligationId,
selectEffectiveHeartbeatReceipt,
type HeartbeatReceiptBinding,
type HeartbeatReceiptFact,
Expand All @@ -30,34 +31,47 @@ export interface PriorHostTurnCloseoutCandidate extends JsonObject {
/** The Todo id or autonomous replan obligation id, per `binding_kind`. */
binding_id: string;
settlement_effect_id: string | null;
/** The effective guard admitted a replan under this Todo's settlement identity. */
admitted_todo_replan?: true;
}

interface CloseoutCandidateCheck {
candidate: PriorHostTurnCloseoutCandidate;
requireWritebackForRecovery: boolean;
}

function candidate(
binding: HeartbeatReceiptBinding,
turnInstanceId: string,
fact: HeartbeatReceiptFact,
admittedTodoReplan: boolean,
): PriorHostTurnCloseoutCandidate {
return {
prior_turn_instance_id: turnInstanceId,
event_id: fact.event_id,
binding_kind: binding.binding_kind,
binding_id: binding.binding_id,
settlement_effect_id: binding.settlement_effect_id,
...(admittedTodoReplan ? {admitted_todo_replan: true as const} : {}),
};
}

function admitsAutonomousReplan(event: JsonObject, fact: HeartbeatReceiptFact): boolean {
if (fact.replan_obligation_id === null || fact.settlement_effect_id === null) return false;
const guard = heartbeatReceiptDetails(event);
return guard.semantic_replan_obligation_id === fact.replan_obligation_id
const obligationId = normalizeHeartbeatReplanObligationId(guard.semantic_replan_obligation_id);
return fact.settlement_effect_id !== null && obligationId !== null
&& (fact.todo_id !== null || obligationId === fact.replan_obligation_id)
&& guard.interaction_mode === "autonomous_replan" && guard.ok === true && guard.should_run === true
&& guard.must_attempt_work === true && guard.delivery_allowed === false && guard.quiet_noop_allowed === false;
}

/**
* Select prior Turns eligible for closeout readback. Guard preflight requires
* an explicit closeout marker. Refresh recovery can also discover an admitted
* replan-only binding without making it a later Turn's admission prerequisite.
* Select prior Turns eligible for closeout readback. A Todo-bound replan
* carries the Todo settlement identity and a separate semantic obligation.
* It did not deliver a Todo, so its closeout marker can be false. Fresh
* admission requires a verified outstanding writeback for this binding.
* Explicit pending-closeout discovery can also find admitted replans before
* writeback, whether their settlement identity is a Todo or an obligation.
*
* Every prior Turn is identity-validated, so a conflicting receipt fails the
* read instead of being hidden behind a newer Turn. Results are newest first
Expand All @@ -74,8 +88,8 @@ export function selectCloseoutCandidates(
goalId: string,
agentId: string,
excludeTurnInstanceId: string | null,
{ includeAdmittedReplans } = { includeAdmittedReplans: false },
): { candidates: PriorHostTurnCloseoutCandidate[]; turnsValidated: number } {
{ includeUnwrittenReplans } = { includeUnwrittenReplans: false },
): { candidates: CloseoutCandidateCheck[]; turnsValidated: number } {
const perTurn = new Map<string, { fact: HeartbeatReceiptFact; value: JsonObject }[]>();
const firstRecorded = new Map<string, number>();
for (const event of receipts) {
Expand All @@ -100,7 +114,7 @@ export function selectCloseoutCandidates(
}
}

const candidates: PriorHostTurnCloseoutCandidate[] = [];
const candidates: CloseoutCandidateCheck[] = [];
const currentPosition = excludeTurnInstanceId === null
? undefined : firstRecorded.get(excludeTurnInstanceId);
let validated = 0;
Expand All @@ -112,10 +126,14 @@ export function selectCloseoutCandidates(
// must not conceal a conflicting persisted settlement binding.
if (currentPosition !== undefined && firstRecorded.get(turnInstanceId)! >= currentPosition) continue;
const fact = heartbeatReceiptFactFromEvent(effective);
if (!fact.closeout_required && !(includeAdmittedReplans && admitsAutonomousReplan(effective, fact))) continue;
const binding = heartbeatReceiptBinding(goalId, agentId, fact);
if (binding === null) continue;
candidates.push(candidate(binding, turnInstanceId, fact));
const admittedReplan = admitsAutonomousReplan(effective, fact);
const admittedTodoReplan = admittedReplan && binding.binding_kind === "todo";
if (!fact.closeout_required && !admittedTodoReplan &&
!(includeUnwrittenReplans && admittedReplan)) continue;
candidates.push({candidate: candidate(binding, turnInstanceId, fact, admittedTodoReplan),
requireWritebackForRecovery: !fact.closeout_required && !includeUnwrittenReplans});
}
return {candidates, turnsValidated: validated};
}
Expand Down Expand Up @@ -147,18 +165,27 @@ function bundleFailed(readback: JsonObject, step: string): boolean {

/**
* The closeout receipts a candidate's settlement readback still lacks, or
* null when that readback already validates the Turn's settlement.
* null when the Turn's settlement validates, or undefined when a candidate
* without a closeout marker has no verified writeback.
*/
export function missingCloseoutReceipts(readback: JsonObject): string[] | null {
export function missingCloseoutReceipts(
readback: JsonObject, requireWriteback: boolean,
): string[] | null | undefined {
if (readback.found === true && !bundleFailed(readback, "settlement")) {
return null;
}
const missingReceipts: string[] = [];
if (readback.found !== true || bundleFailed(readback, "writeback")) {
if (requireWriteback) return undefined;
missingReceipts.push(WRITEBACK_RECEIPT);
}
if (readback.found !== true || bundleFailed(readback, "spend")) {
missingReceipts.push(SPEND_RECEIPT);
}
return missingReceipts;
}

/** Only an exact writeback can leave the original debit as the sole debt. */
export function awaitingQuotaSpend(missing: readonly string[]): boolean {
return missing.length === 1 && missing[0] === SPEND_RECEIPT;
}
7 changes: 7 additions & 0 deletions loopx/control_plane/quota/settlement.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ class QuotaSpendPrecheck:
settlement: SettlementResult[dict[str, Any]]
progress: dict[str, Any] | None
spend_run_present: bool
spend_request_rejection: SettlementResult[dict[str, Any]] | None


@dataclass(frozen=True, slots=True)
Expand Down Expand Up @@ -424,6 +425,7 @@ def read_quota_spend_precheck(
allow_unbound_binding: bool = False,
registry_path: Path | None = None,
goal_ref: Mapping[str, Any] | None = None,
spend_request: Mapping[str, Any] | None = None,
) -> QuotaSpendPrecheck | None:
"""Read the identity and replay controls before building a fresh preview."""

Expand All @@ -438,6 +440,7 @@ def read_quota_spend_precheck(
"infer_turn_instance_id": infer_turn_instance_id,
"allow_unbound_binding": allow_unbound_binding,
"spend_precheck": True,
**({"spend_request": dict(spend_request)} if spend_request is not None else {}),
},
registry_path=registry_path,
goal_ref=goal_ref,
Expand All @@ -452,6 +455,10 @@ def read_quota_spend_precheck(
settlement=_readback_result(payload.get("settlement")),
progress=_optional_readback_record(payload.get("progress")),
spend_run_present=spend_run_present,
spend_request_rejection=(
_readback_result(payload["spend_request_rejection"])
if "spend_request_rejection" in payload else None
),
)


Expand Down
Loading
Loading