Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 41 additions & 12 deletions loopx/cli_commands/quota_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
cached_goal_run_index_is_current,
load_status_projection_cache,
resolve_status_projection_cache_runtime_root,
status_projection_cache_metadata,
write_status_projection_cache,
)
from ..control_plane.scheduler.execution_context import (
Expand Down Expand Up @@ -362,21 +363,49 @@ def prepare_quota_command_context(
available_capabilities=args.available_capabilities,
agent_lane_id=args.agent_id,
include_presentation_projections=False,
# A scheduler ACK/failure return writes cadence, not settlement
# facts. One damaged audit row must not fail the whole status read
# before the tolerant native fence can answer it.
tolerate_malformed_settlement_history=(
command in QUOTA_SCHEDULER_FOLLOWUP_COMMANDS
),
)
preview = command == "should-run" and getattr(args, "dry_run", False)
if not preview and bool(getattr(args, "write_projection_cache", False)):
cache_metadata = write_status_projection_cache(
registry_path=registry_path,
runtime_root=runtime_root,
limit=status_limit,
include_task_graph=False,
goal_id=status_goal_id,
payload=status_payload,
max_age_seconds=projection_cache_ttl_seconds,
available_capabilities=args.available_capabilities,
agent_lane_id=args.agent_id,
include_presentation_projections=False,
)
if command in QUOTA_SCHEDULER_FOLLOWUP_COMMANDS:
# The follow-up collector tolerates malformed history so a
# valid scheduler return can still be recorded. That partial
# view must never become the cache consumed by strict status
# readers under the same key.
cache_metadata = status_projection_cache_metadata(
registry_path=registry_path,
runtime_root=runtime_root,
limit=status_limit,
include_task_graph=False,
goal_id=status_goal_id,
max_age_seconds=projection_cache_ttl_seconds,
available_capabilities=args.available_capabilities,
agent_lane_id=args.agent_id,
include_presentation_projections=False,
)
cache_metadata.update({
"hit": False,
"written": False,
"skipped_reason": "tolerant_scheduler_followup_view",
})
else:
cache_metadata = write_status_projection_cache(
registry_path=registry_path,
runtime_root=runtime_root,
limit=status_limit,
include_task_graph=False,
goal_id=status_goal_id,
payload=status_payload,
max_age_seconds=projection_cache_ttl_seconds,
available_capabilities=args.available_capabilities,
agent_lane_id=args.agent_id,
include_presentation_projections=False,
)
elif isinstance(status_payload.get("projection_cache"), dict):
cache_metadata = dict(status_payload["projection_cache"])

Expand Down
38 changes: 9 additions & 29 deletions loopx/cli_commands/quota_scheduler_followup.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@ def _build_scheduler_followup_decision(
repository_delivery_interaction_hook(repo_path=Path.cwd()),
),
goal_ref=goal_ref,
# The compatibility ACK/failure route writes scheduler cadence, not
# settlement facts: one damaged audit row must not turn a valid return
# into a failure before the tolerant native fence runs. Exact
# settlement branches keep the strict readback.
tolerate_malformed_settlement_history=True,
)


Expand Down Expand Up @@ -105,35 +110,6 @@ def build_scheduler_followup_payload(
if turn_instance_id
else None
)
if turn_instance_id and heartbeat_receipt is None:
return {
"ok": False,
"schema_version": "quota_scheduler_followup_receipt_failure_v0",
"mode": args.quota_command,
"goal_id": args.goal_id,
"agent_id": args.agent_id,
"turn_instance_id": turn_instance_id,
"decision": "skip",
"should_run": False,
"status": "heartbeat_receipt_missing",
"state": "blocked_receipt",
"error_code": "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_MISSING",
"reason": (
"scheduler follow-up requires the originating heartbeat receipt; "
"refusing to rebuild authority from the current live frontier"
),
"retry_guidance": (
"retry with the exact Turn id from a committed quota should-run "
"heartbeat receipt"
),
"write_performed": False,
"appended": False,
"registry_mutated": False,
"scheduler_state_mutated": False,
"quota_spend_performed": False,
"delivery_outcome": "surface_only",
}

observed_rrule = str(
getattr(args, "app_automation_current_rrule", None)
or getattr(args, "codex_app_current_rrule", None)
Expand Down Expand Up @@ -229,6 +205,8 @@ def build_scheduler_followup_payload(
observed_host_rrule=observed_rrule,
failure_kind=args.failure_kind,
turn_instance_id=turn_instance_id,
goal_ref=goal_ref,
registry_path=registry_path,
)
return record_quota_scheduler_ack(
status_payload,
Expand All @@ -250,4 +228,6 @@ def build_scheduler_followup_payload(
operator_inbox_urgency_projector=operator_inbox_urgency_projector,
before_decision=before_decision,
turn_instance_id=turn_instance_id,
Comment thread
loopx-agent marked this conversation as resolved.
Resolved
goal_ref=goal_ref,
registry_path=registry_path,
)
2 changes: 2 additions & 0 deletions loopx/control_plane/quota/live_decision.py
Original file line number Diff line number Diff line change
Expand Up @@ -701,6 +701,7 @@ def build_live_quota_should_run_decision(
workspace_path: Path | None = None,
goal_ref: Mapping[str, object] | None = None,
inline_goal_source: bool = False,
tolerate_malformed_settlement_history: bool = False,
) -> dict[str, Any]:
"""Build one live CLI decision while keeping host observation injectable."""
resolved_context = resolve_scheduler_execution_context(scheduler_execution_context)
Expand Down Expand Up @@ -780,6 +781,7 @@ def build_live_quota_should_run_decision(
),
registry_path=registry_path,
goal_ref=goal_ref,
tolerate_malformed_history=tolerate_malformed_settlement_history,
)
receipt_bound_monitor_phase = (
settlement_readback.monitor_phase if settlement_readback else None
Expand Down
13 changes: 13 additions & 0 deletions loopx/control_plane/quota/scheduler_ack.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

from collections.abc import Mapping
from typing import Any, cast

import json
Expand Down Expand Up @@ -185,6 +186,8 @@ def _run_native_followup(
use_current_hint: bool,
reason_summary: str | None,
turn_instance_id: str | None = None,
goal_ref: Mapping[str, object] | None = None,
registry_path: Path | None = None,
) -> dict[str, Any]:
"""Decision-free compatibility transport for in-process Python callers.

Expand All @@ -201,6 +204,8 @@ def _run_native_followup(
request = {
"schema_version": _FOLLOWUP_REQUEST_SCHEMA,
"runtime_root": str(runtime_root),
"registry_path": str(registry_path) if registry_path is not None else None,
"goal_ref": dict(goal_ref) if goal_ref is not None else None,
"turn_instance_id": receipt_turn_instance_id,
"require_heartbeat_receipt": receipt_turn_instance_id is not None,
"before": before,
Expand Down Expand Up @@ -310,6 +315,8 @@ def record_quota_scheduler_ack_for_decision(
use_current_hint: bool = False,
host_match_observed: bool = False,
turn_instance_id: str | None = None,
goal_ref: Mapping[str, object] | None = None,
registry_path: Path | None = None,
) -> dict[str, Any]:
state_key = _followup_state_key(before, surface=surface, state_key=state_key)
safe_agent_id = normalize_todo_claimed_by(agent_id)
Expand Down Expand Up @@ -380,6 +387,8 @@ def record_quota_scheduler_ack_for_decision(
use_current_hint=use_current_hint,
reason_summary=reason_summary,
turn_instance_id=turn_instance_id,
goal_ref=goal_ref,
registry_path=registry_path,
)
except ValueError as exc:
return _failure_payload(
Expand Down Expand Up @@ -410,6 +419,8 @@ def record_quota_scheduler_failure_for_decision(
failure_kind: str = "host_tool_failure",
generated_at: str | None = None,
turn_instance_id: str | None = None,
goal_ref: Mapping[str, object] | None = None,
registry_path: Path | None = None,
) -> dict[str, Any]:
state_key = _followup_state_key(before, surface=surface, state_key=state_key)
safe_agent_id = normalize_todo_claimed_by(agent_id)
Expand Down Expand Up @@ -458,6 +469,8 @@ def record_quota_scheduler_failure_for_decision(
use_current_hint=False,
reason_summary=None,
turn_instance_id=turn_instance_id,
goal_ref=goal_ref,
registry_path=registry_path,
)
except ValueError as exc:
return _failure_payload(
Expand Down
6 changes: 6 additions & 0 deletions loopx/control_plane/quota/settlement.py
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,7 @@ def read_heartbeat_settlement(
borrow_source_admission: bool = False,
heartbeat_reentry_guard: dict[str, Any] | None = None,
include_record_bodies: bool = True,
tolerate_malformed_history: bool = False,
) -> QuotaSettlementReadback | None:
"""Read one complete heartbeat settlement through the TS domain owner."""

Expand All @@ -510,6 +511,11 @@ def read_heartbeat_settlement(
if include_record_bodies is not True
else {}
),
**(
{"tolerate_malformed_history": True}
if tolerate_malformed_history
else {}
),
**({"resolve_original_binding": True} if resolve_original_binding else {}),
**({"pending_closeout": True} if pending_closeout else {}),
**({"refresh_retry": refresh_retry} if refresh_retry is not None else {}),
Expand Down
37 changes: 32 additions & 5 deletions loopx/control_plane/quota/settlement_readback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { isAbsolute, join } from "node:path";

import { readReceiptLogSnapshot } from "../runtime/receipt_log_snapshot.ts";
import {
committedGoalRolloutEvents,
heartbeatReceiptsFromEvents,
readGoalRolloutEventSnapshot,
strictGoalRolloutEvents,
Expand Down Expand Up @@ -122,6 +123,7 @@ interface ReadbackRequest {
pending_closeout: boolean;
borrow_source_admission: boolean;
include_record_bodies: boolean;
tolerate_malformed_history: boolean;
refresh_retry: RefreshRetryRequest | null;
heartbeat_reentry_guard: JsonObject | null;
owner: QuotaAccountingOwner;
Expand Down Expand Up @@ -249,6 +251,14 @@ function decodeRequest(
if (request.include_record_bodies !== undefined && typeof request.include_record_bodies !== "boolean") {
throw new EffectRuntimeRequestError("include_record_bodies must be a boolean");
}
if (
request.tolerate_malformed_history !== undefined &&
typeof request.tolerate_malformed_history !== "boolean"
) {
throw new EffectRuntimeRequestError(
"tolerate_malformed_history must be a boolean",
);
}
if (request.resolve_original_binding === true && request.infer_turn_instance_id) {
throw new EffectRuntimeRequestError("original binding requires an explicit Turn identity");
}
Expand Down Expand Up @@ -316,16 +326,20 @@ function decodeRequest(
pending_closeout: request.pending_closeout === true,
borrow_source_admission: request.borrow_source_admission === true,
include_record_bodies: request.include_record_bodies !== false,
tolerate_malformed_history: request.tolerate_malformed_history === true,
refresh_retry: decodeRefreshRetry(request.refresh_retry),
heartbeat_reentry_guard: request.heartbeat_reentry_guard == null ? null
: requireJsonObject(request.heartbeat_reentry_guard, "heartbeat_reentry_guard"),
owner,
};
}

async function readRunReceipts(path: string): Promise<readonly JsonObject[]> {
async function readRunReceipts(
path: string,
tolerateMalformedHistory: boolean = false,
): Promise<readonly JsonObject[]> {
const snapshot = await readReceiptLogSnapshot(path);
if (snapshot?.firstErrorLine != null) {
if (!tolerateMalformedHistory && snapshot?.firstErrorLine != null) {
throw new EffectRuntimeRequestError(
`settlement readback line ${snapshot.firstErrorLine} is malformed`,
"malformed_settlement_state",
Expand Down Expand Up @@ -406,6 +420,7 @@ export async function readQuotaSettlementSnapshot(
runtimeRoot: string,
goalId: string,
rolloutSnapshot?: GoalRolloutEventSnapshot | null,
tolerateMalformedHistory: boolean = false,
): Promise<QuotaSettlementReadbackSnapshot> {
settlementScope(runtimeRoot, goalId);
if (
Expand All @@ -426,12 +441,19 @@ export async function readQuotaSettlementSnapshot(
rolloutSnapshot === undefined
? readGoalRolloutEventSnapshot(runtimeRoot, goalId)
: Promise.resolve(rolloutSnapshot),
readRunReceipts(join(goalRoot, "runs", "index.jsonl")),
readRunReceipts(join(goalRoot, "runs", "index.jsonl"), tolerateMalformedHistory),
]);
// Settlement-authority callers keep failing closed on the first damaged row.
// The scheduler follow-up route only needs the receipts it can actually read,
// so it opts into the same tolerant view the heartbeat reader already uses.
// Both views still ignore an unterminated row: the append owner will drop it
// before writing the next record.
return indexSettlementSnapshot(
runtimeRoot,
goalId,
strictGoalRolloutEvents(snapshot),
tolerateMalformedHistory
? committedGoalRolloutEvents(snapshot)
: strictGoalRolloutEvents(snapshot),
runs,
);
}
Expand Down Expand Up @@ -1644,7 +1666,12 @@ export async function readQuotaSettlement(value: unknown): Promise<JsonObject> {
async () => {
const matchedSpend: MatchedSpendRecords = {run: null, receipt: null};
const result = readQuotaSettlementFromRequest(request,
await readQuotaSettlementSnapshot(request.runtime_root, request.goal_id), true, matchedSpend);
await readQuotaSettlementSnapshot(
request.runtime_root,
request.goal_id,
undefined,
request.tolerate_malformed_history,
), true, matchedSpend);
const identity = jsonObject(jsonObject(jsonObject(result.identity)?.result)?.value);
const receipt = spendReceiptTerms(matchedSpend.receipt);
// Only exact readback opens its matched debit artifact. Check its
Expand Down
4 changes: 2 additions & 2 deletions loopx/control_plane/quota/unsettled_host_turn_recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import {
requireNonEmptyString,
} from "../runtime_decode.ts";
import {
goalHeartbeatReceiptsFromSnapshot,
committedGoalHeartbeatReceiptsFromSnapshot,
readGoalRolloutEventSnapshot,
} from "../rollout_receipt_log.ts";
import { isCommittedMonitorPollEffect } from "./settlement_phase.ts";
Expand Down Expand Up @@ -230,7 +230,7 @@ async function preflightPriorHostTurnCloseoutForOwner(
request.runtime_root,
request.goal_id,
);
const receipts = goalHeartbeatReceiptsFromSnapshot(
const receipts = committedGoalHeartbeatReceiptsFromSnapshot(
rolloutSnapshot,
request.goal_id,
request.agent_id,
Expand Down
Loading
Loading