From 6f33ca06066ae07dfc4333517c0690f4bf8e5a9e Mon Sep 17 00:00:00 2001 From: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:18:33 +0800 Subject: [PATCH 1/8] fix(scheduler): keep damaged history behind one receipt fence The Python compatibility route and the native owner now agree on a single freshness rule. A damaged row can no longer retire a valid ACK or failure return, a committed successor still blocks an older return when only its summary rotted, and the originating Turn plus Goal instance scope decide both before the write and again under the state lock. Re-applied on top of the merged #8708 minimal passthrough. This commit keeps only the extra hardening: tolerant scheduler reads with strict settlement and status reads, committed-successor scope proof, instance-range checks, and the lock-time re-read before replay. Verified on the current base with the six TypeScript suites (248 tests), the control-plane type check, and the Python receipt-fence (30), follow-up hint transport, single-request and rollout-log suites (53). The merged #8708 regression file passes 17 tests on the same tree. Windows, the installed host UI, PostgreSQL and hosted CI were not exercised. Signed-off-by: gcl-coder --- loopx/cli_commands/quota_context.py | 53 +- .../cli_commands/quota_scheduler_followup.py | 38 +- loopx/control_plane/quota/live_decision.py | 2 + loopx/control_plane/quota/scheduler_ack.py | 13 + loopx/control_plane/quota/settlement.py | 6 + .../quota/settlement_readback.ts | 34 +- loopx/control_plane/rollout_receipt_log.ts | 181 ++++- .../runtime/json_line_members.ts | 318 +++++++++ .../runtime/receipt_log_snapshot.ts | 93 ++- .../scheduler/heartbeat_commit.ts | 304 ++++++++- .../scheduler/heartbeat_followup.ts | 32 + .../scheduler/heartbeat_followup_cli.ts | 9 +- .../control_plane/scheduler/scheduler_hint.py | 46 +- loopx/control_plane/status/collection.py | 6 + .../work_items/attention_queue.py | 10 +- .../work_items/replan_history_codec.py | 6 + .../work_items/replan_history_settlement.ts | 19 +- loopx/quota.py | 4 + loopx/rollout_event_log.py | 41 +- ...roject_registry_io_classifications_v1.json | 1 + loopx/status.py | 19 +- .../test_scheduler_heartbeat_receipt_fence.py | 633 ++++++++++++++++++ .../test_scheduler_hint_single_request.py | 43 ++ ..._scheduler_host_followup_hint_transport.py | 40 ++ .../quota_settlement_readback.test.ts | 27 +- .../receipt_log_snapshot.test.ts | 93 +++ .../scheduler_heartbeat_commit.test.ts | 234 +++++++ .../scheduler_heartbeat_followup.test.ts | 231 ++++++- .../scheduler_heartbeat_followup_cli.test.ts | 66 +- .../scheduler_heartbeat_receipt_fence.test.ts | 293 ++++++++ tests/test_rollout_event_log_loading.py | 47 +- 31 files changed, 2830 insertions(+), 112 deletions(-) create mode 100644 loopx/control_plane/runtime/json_line_members.ts create mode 100644 tests/control_plane/test_scheduler_heartbeat_receipt_fence.py diff --git a/loopx/cli_commands/quota_context.py b/loopx/cli_commands/quota_context.py index c69859b269..7c2fde92cd 100644 --- a/loopx/cli_commands/quota_context.py +++ b/loopx/cli_commands/quota_context.py @@ -13,6 +13,7 @@ cached_goal_run_index_is_current, load_status_projection_cache, resolve_status_projection_cache_runtime_root, + status_projection_cache_metadata, write_status_projection_cache, ) from ..control_plane.scheduler.execution_context import ( @@ -362,21 +363,49 @@ def prepare_quota_command_context( available_capabilities=args.available_capabilities, agent_lane_id=args.agent_id, include_presentation_projections=False, + # A scheduler ACK/failure return writes cadence, not settlement + # facts. One damaged audit row must not fail the whole status read + # before the tolerant native fence can answer it. + tolerate_malformed_settlement_history=( + command in QUOTA_SCHEDULER_FOLLOWUP_COMMANDS + ), ) preview = command == "should-run" and getattr(args, "dry_run", False) if not preview and bool(getattr(args, "write_projection_cache", False)): - cache_metadata = write_status_projection_cache( - registry_path=registry_path, - runtime_root=runtime_root, - limit=status_limit, - include_task_graph=False, - goal_id=status_goal_id, - payload=status_payload, - max_age_seconds=projection_cache_ttl_seconds, - available_capabilities=args.available_capabilities, - agent_lane_id=args.agent_id, - include_presentation_projections=False, - ) + if command in QUOTA_SCHEDULER_FOLLOWUP_COMMANDS: + # The follow-up collector tolerates malformed history so a + # valid scheduler return can still be recorded. That partial + # view must never become the cache consumed by strict status + # readers under the same key. + cache_metadata = status_projection_cache_metadata( + registry_path=registry_path, + runtime_root=runtime_root, + limit=status_limit, + include_task_graph=False, + goal_id=status_goal_id, + max_age_seconds=projection_cache_ttl_seconds, + available_capabilities=args.available_capabilities, + agent_lane_id=args.agent_id, + include_presentation_projections=False, + ) + cache_metadata.update({ + "hit": False, + "written": False, + "skipped_reason": "tolerant_scheduler_followup_view", + }) + else: + cache_metadata = write_status_projection_cache( + registry_path=registry_path, + runtime_root=runtime_root, + limit=status_limit, + include_task_graph=False, + goal_id=status_goal_id, + payload=status_payload, + max_age_seconds=projection_cache_ttl_seconds, + available_capabilities=args.available_capabilities, + agent_lane_id=args.agent_id, + include_presentation_projections=False, + ) elif isinstance(status_payload.get("projection_cache"), dict): cache_metadata = dict(status_payload["projection_cache"]) diff --git a/loopx/cli_commands/quota_scheduler_followup.py b/loopx/cli_commands/quota_scheduler_followup.py index 8bbe506bba..8772c91299 100644 --- a/loopx/cli_commands/quota_scheduler_followup.py +++ b/loopx/cli_commands/quota_scheduler_followup.py @@ -76,6 +76,11 @@ def _build_scheduler_followup_decision( repository_delivery_interaction_hook(repo_path=Path.cwd()), ), goal_ref=goal_ref, + # The compatibility ACK/failure route writes scheduler cadence, not + # settlement facts: one damaged audit row must not turn a valid return + # into a failure before the tolerant native fence runs. Exact + # settlement branches keep the strict readback. + tolerate_malformed_settlement_history=True, ) @@ -105,35 +110,6 @@ def build_scheduler_followup_payload( if turn_instance_id else None ) - if turn_instance_id and heartbeat_receipt is None: - return { - "ok": False, - "schema_version": "quota_scheduler_followup_receipt_failure_v0", - "mode": args.quota_command, - "goal_id": args.goal_id, - "agent_id": args.agent_id, - "turn_instance_id": turn_instance_id, - "decision": "skip", - "should_run": False, - "status": "heartbeat_receipt_missing", - "state": "blocked_receipt", - "error_code": "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_MISSING", - "reason": ( - "scheduler follow-up requires the originating heartbeat receipt; " - "refusing to rebuild authority from the current live frontier" - ), - "retry_guidance": ( - "retry with the exact Turn id from a committed quota should-run " - "heartbeat receipt" - ), - "write_performed": False, - "appended": False, - "registry_mutated": False, - "scheduler_state_mutated": False, - "quota_spend_performed": False, - "delivery_outcome": "surface_only", - } - observed_rrule = str( getattr(args, "app_automation_current_rrule", None) or getattr(args, "codex_app_current_rrule", None) @@ -229,6 +205,8 @@ def build_scheduler_followup_payload( observed_host_rrule=observed_rrule, failure_kind=args.failure_kind, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, + registry_path=registry_path, ) return record_quota_scheduler_ack( status_payload, @@ -250,4 +228,6 @@ def build_scheduler_followup_payload( operator_inbox_urgency_projector=operator_inbox_urgency_projector, before_decision=before_decision, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, + registry_path=registry_path, ) diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index fde3e9c491..9212199184 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -697,6 +697,7 @@ def build_live_quota_should_run_decision( workspace_path: Path | None = None, goal_ref: Mapping[str, object] | None = None, inline_goal_source: bool = False, + tolerate_malformed_settlement_history: bool = False, ) -> dict[str, Any]: """Build one live CLI decision while keeping host observation injectable.""" resolved_context = resolve_scheduler_execution_context(scheduler_execution_context) @@ -776,6 +777,7 @@ def build_live_quota_should_run_decision( ), registry_path=registry_path, goal_ref=goal_ref, + tolerate_malformed_history=tolerate_malformed_settlement_history, ) receipt_bound_monitor_phase = ( settlement_readback.monitor_phase if settlement_readback else None diff --git a/loopx/control_plane/quota/scheduler_ack.py b/loopx/control_plane/quota/scheduler_ack.py index e973714e4d..bdcb76b134 100644 --- a/loopx/control_plane/quota/scheduler_ack.py +++ b/loopx/control_plane/quota/scheduler_ack.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Mapping from typing import Any, cast import json @@ -185,6 +186,8 @@ def _run_native_followup( use_current_hint: bool, reason_summary: str | None, turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, + registry_path: Path | None = None, ) -> dict[str, Any]: """Decision-free compatibility transport for in-process Python callers. @@ -201,6 +204,8 @@ def _run_native_followup( request = { "schema_version": _FOLLOWUP_REQUEST_SCHEMA, "runtime_root": str(runtime_root), + "registry_path": str(registry_path) if registry_path is not None else None, + "goal_ref": dict(goal_ref) if goal_ref is not None else None, "turn_instance_id": receipt_turn_instance_id, "require_heartbeat_receipt": receipt_turn_instance_id is not None, "before": before, @@ -310,6 +315,8 @@ def record_quota_scheduler_ack_for_decision( use_current_hint: bool = False, host_match_observed: bool = False, turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, + registry_path: Path | None = None, ) -> dict[str, Any]: state_key = _followup_state_key(before, surface=surface, state_key=state_key) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -380,6 +387,8 @@ def record_quota_scheduler_ack_for_decision( use_current_hint=use_current_hint, reason_summary=reason_summary, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, + registry_path=registry_path, ) except ValueError as exc: return _failure_payload( @@ -410,6 +419,8 @@ def record_quota_scheduler_failure_for_decision( failure_kind: str = "host_tool_failure", generated_at: str | None = None, turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, + registry_path: Path | None = None, ) -> dict[str, Any]: state_key = _followup_state_key(before, surface=surface, state_key=state_key) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -458,6 +469,8 @@ def record_quota_scheduler_failure_for_decision( use_current_hint=False, reason_summary=None, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, + registry_path=registry_path, ) except ValueError as exc: return _failure_payload( diff --git a/loopx/control_plane/quota/settlement.py b/loopx/control_plane/quota/settlement.py index c5d282fd63..10b61810aa 100644 --- a/loopx/control_plane/quota/settlement.py +++ b/loopx/control_plane/quota/settlement.py @@ -482,6 +482,7 @@ def read_heartbeat_settlement( borrow_source_admission: bool = False, heartbeat_reentry_guard: dict[str, Any] | None = None, include_record_bodies: bool = True, + tolerate_malformed_history: bool = False, ) -> QuotaSettlementReadback | None: """Read one complete heartbeat settlement through the TS domain owner.""" @@ -500,6 +501,11 @@ def read_heartbeat_settlement( if include_record_bodies is not True else {} ), + **( + {"tolerate_malformed_history": True} + if tolerate_malformed_history + else {} + ), **({"resolve_original_binding": True} if resolve_original_binding else {}), **({"pending_closeout": True} if pending_closeout else {}), **({"refresh_retry": refresh_retry} if refresh_retry is not None else {}), diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index f3064dbf36..92cd835645 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -121,6 +121,7 @@ interface ReadbackRequest { pending_closeout: boolean; borrow_source_admission: boolean; include_record_bodies: boolean; + tolerate_malformed_history: boolean; refresh_retry: RefreshRetryRequest | null; heartbeat_reentry_guard: JsonObject | null; owner: QuotaAccountingOwner; @@ -248,6 +249,14 @@ function decodeRequest( if (request.include_record_bodies !== undefined && typeof request.include_record_bodies !== "boolean") { throw new EffectRuntimeRequestError("include_record_bodies must be a boolean"); } + if ( + request.tolerate_malformed_history !== undefined && + typeof request.tolerate_malformed_history !== "boolean" + ) { + throw new EffectRuntimeRequestError( + "tolerate_malformed_history must be a boolean", + ); + } if (request.resolve_original_binding === true && request.infer_turn_instance_id) { throw new EffectRuntimeRequestError("original binding requires an explicit Turn identity"); } @@ -315,6 +324,7 @@ function decodeRequest( pending_closeout: request.pending_closeout === true, borrow_source_admission: request.borrow_source_admission === true, include_record_bodies: request.include_record_bodies !== false, + tolerate_malformed_history: request.tolerate_malformed_history === true, refresh_retry: decodeRefreshRetry(request.refresh_retry), heartbeat_reentry_guard: request.heartbeat_reentry_guard == null ? null : requireJsonObject(request.heartbeat_reentry_guard, "heartbeat_reentry_guard"), @@ -322,9 +332,12 @@ function decodeRequest( }; } -async function readRunReceipts(path: string): Promise { +async function readRunReceipts( + path: string, + tolerateMalformedHistory: boolean = false, +): Promise { const snapshot = await readReceiptLogSnapshot(path); - if (snapshot?.firstErrorLine != null) { + if (!tolerateMalformedHistory && snapshot?.firstErrorLine != null) { throw new EffectRuntimeRequestError( `settlement readback line ${snapshot.firstErrorLine} is malformed`, "malformed_settlement_state", @@ -405,6 +418,7 @@ export async function readQuotaSettlementSnapshot( runtimeRoot: string, goalId: string, rolloutSnapshot?: GoalRolloutEventSnapshot | null, + tolerateMalformedHistory: boolean = false, ): Promise { settlementScope(runtimeRoot, goalId); if ( @@ -425,12 +439,17 @@ export async function readQuotaSettlementSnapshot( rolloutSnapshot === undefined ? readGoalRolloutEventSnapshot(runtimeRoot, goalId) : Promise.resolve(rolloutSnapshot), - readRunReceipts(join(goalRoot, "runs", "index.jsonl")), + readRunReceipts(join(goalRoot, "runs", "index.jsonl"), tolerateMalformedHistory), ]); + // Settlement-authority callers keep failing closed on the first damaged row. + // The scheduler follow-up route only needs the receipts it can actually read, + // so it opts into the same tolerant view the heartbeat reader already uses. return indexSettlementSnapshot( runtimeRoot, goalId, - strictGoalRolloutEvents(snapshot), + tolerateMalformedHistory + ? snapshot?.events ?? [] + : strictGoalRolloutEvents(snapshot), runs, ); } @@ -1620,7 +1639,12 @@ export async function readQuotaSettlement(value: unknown): Promise { async () => { const matchedSpend: MatchedSpendRecords = {run: null, receipt: null}; const result = readQuotaSettlementFromRequest(request, - await readQuotaSettlementSnapshot(request.runtime_root, request.goal_id), true, matchedSpend); + await readQuotaSettlementSnapshot( + request.runtime_root, + request.goal_id, + undefined, + request.tolerate_malformed_history, + ), true, matchedSpend); const identity = jsonObject(jsonObject(jsonObject(result.identity)?.result)?.value); const receipt = spendReceiptTerms(matchedSpend.receipt); // Only exact readback opens its matched debit artifact. Check its diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index a02c8464cc..150d08413c 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -11,7 +11,7 @@ import { relative, resolve, sep } from "node:path"; import type { JsonObject } from "./effect_program.ts"; import { EffectRuntimeRequestError } from "./effect_runtime_errors.ts"; -import { requireNonEmptyString } from "./runtime_decode.ts"; +import { jsonObject, requireNonEmptyString } from "./runtime_decode.ts"; import { readReceiptLogSnapshot } from "./runtime/receipt_log_snapshot.ts"; export const ROLLOUT_EVENT_SCHEMA_VERSION = "loopx_rollout_event_v0"; @@ -32,6 +32,15 @@ export interface GoalRolloutEventSnapshot { readonly runtimeRoot: string; readonly goalId: string; readonly events: readonly JsonObject[]; + /** Physical line of each event, in `events` order. */ + readonly eventLines: readonly number[]; + /** Intact members of damaged lines, with their line numbers. */ + readonly malformedLines: readonly Readonly<{ + line: number; + members: readonly (readonly [string, unknown])[]; + /** Identity members the damaged line contained but could not read. */ + unreadable: readonly string[]; + }>[]; readonly firstStrictErrorLine: number | null; } @@ -81,10 +90,14 @@ export async function readGoalRolloutEventSnapshot( goalId: string, ): Promise { const snapshot = await readReceiptLogSnapshot( - goalRolloutEventLogPath(runtimeRoot, goalId), ROLLOUT_EVENT_SCHEMA_VERSION, + goalRolloutEventLogPath(runtimeRoot, goalId), + ROLLOUT_EVENT_SCHEMA_VERSION, + ADMISSION_IDENTITY_MEMBERS, ); return snapshot === null ? null : { runtimeRoot, goalId, events: snapshot.records, + eventLines: snapshot.recordLines, + malformedLines: snapshot.malformedLines, firstStrictErrorLine: snapshot.firstErrorLine, }; } @@ -154,44 +167,158 @@ export async function readGoalHeartbeatReceipts( ); } +/** A receipt without an exact Goal instance belongs to the legacy goal scope. */ +function receiptMatchesGoalRef(event: JsonObject, goalRef: JsonObject | null): boolean { + if (goalRef === null) return !Object.hasOwn(event, "goal_ref"); + const eventGoalRef = jsonObject(event.goal_ref); + return eventGoalRef !== null && + eventGoalRef.goal_id === goalRef.goal_id && + eventGoalRef.goal_instance_id === goalRef.goal_instance_id; +} + +/** + * One qualified receipt view for both originating and superseding reads. + * + * The originating Turn and every candidate successor must come from the same + * set: a heartbeat receipt with a real Turn identity in this Goal/Agent and + * this exact Goal instance scope. Diagnostic audit rows and rows from another + * Goal lifetime therefore cannot renew or retire this transaction. + */ +function scopedHeartbeatReceipts( + snapshot: GoalRolloutEventSnapshot, + goalId: string, + agentId: string, + goalRef: JsonObject | null, +): JsonObject[] { + if (snapshot.goalId !== goalId) { + throw new EffectRuntimeRequestError( + "rollout event snapshot does not belong to the requested goal", + "rollout_event_snapshot_scope_mismatch", + ); + } + return heartbeatReceiptsFromEvents(snapshot.events, goalId, agentId).filter( + (event) => receiptMatchesGoalRef(event, goalRef), + ); +} + +const ADMISSION_IDENTITY_MEMBERS = [ + "schema_version", + "event_kind", + "goal_id", + "agent_id", + "run_id", + "goal_ref", +] as const; + +/** + * What a finished row still proves about the Turn that superseded this receipt. + * + * The row's commit fact is its own writer's newline: the log owner withdraws + * every member of a row a *later* append had to terminate, so a torn prefix + * cannot manufacture an admission. What remains is whether the row still + * carries one identity. A duplicated identity member proves nothing, so a + * stray write cannot invent a successor. A row whose display field was + * corrupted still proves the Turn its writer committed, and a row that is + * demonstrably a receipt of this scope but whose Turn bytes are unreadable + * proves that *some* Turn was admitted: it is not evidence for the old receipt. + */ +type CommittedSuccessor = + | { readonly kind: "superseded"; readonly runId: string } + | { readonly kind: "unattributed" }; + +function committedSuccessor( + damaged: Readonly<{ + members: readonly (readonly [string, unknown])[]; + unreadable: readonly string[]; + }>, + goalId: string, + agentId: string, + goalRef: JsonObject | null, +): CommittedSuccessor | null { + const fields = new Map(); + // Whether the row carries more than one Turn identity. Its scope is checked + // first: an ambiguous Turn in somebody else's row must not retire this + // transaction, and neither must a diagnostic row that is not a receipt at all. + let ambiguousTurn = false; + for (const [key, value] of damaged.members) { + if ( + fields.has(key) && + (ADMISSION_IDENTITY_MEMBERS as readonly string[]).includes(key) + ) { + if (key !== "run_id") return null; + if (fields.get(key) !== value) ambiguousTurn = true; + } + fields.set(key, value); + } + for (const key of damaged.unreadable) { + if (!(ADMISSION_IDENTITY_MEMBERS as readonly string[]).includes(key)) continue; + // Only the Turn id may be unknown: without the receipt kind or its scope the + // row cannot be attributed to this transaction at all. + if (key !== "run_id") return null; + if (fields.has("run_id")) ambiguousTurn = true; + } + if (fields.get("schema_version") !== ROLLOUT_EVENT_SCHEMA_VERSION) return null; + if (fields.get("event_kind") !== HEARTBEAT_RECEIPT_EVENT_KIND) return null; + if (fields.get("goal_id") !== goalId) return null; + if (fields.get("agent_id") !== agentId) return null; + if (goalRef === null) { + if (fields.has("goal_ref")) return null; + } else { + const eventGoalRef = jsonObject(fields.get("goal_ref")); + if ( + eventGoalRef === null || + eventGoalRef.goal_id !== goalRef.goal_id || + eventGoalRef.goal_instance_id !== goalRef.goal_instance_id + ) return null; + } + // The row now proves it is one of this scope's heartbeat receipts. Only then + // does an unknown Turn mean "somebody else's turn superseded mine". + if (ambiguousTurn) return { kind: "unattributed" }; + const runId = fields.get("run_id"); + if (typeof runId === "string" && runId.trim().length > 0) { + return { kind: "superseded", runId }; + } + return damaged.unreadable.includes("run_id") ? { kind: "unattributed" } : null; +} + export async function heartbeatReceiptStatus(params: { runtimeRoot: string; goalId: string; agentId: string; turnInstanceId: string; + goalRef?: JsonObject | null; }): Promise { + const goalRef = params.goalRef ?? null; const snapshot = await readGoalRolloutEventSnapshot( params.runtimeRoot, params.goalId, ); - const receipts = goalHeartbeatReceiptsFromSnapshot( + if (snapshot === null) return "missing"; + const receipts = scopedHeartbeatReceipts( snapshot, params.goalId, params.agentId, + goalRef, ); - if (snapshot === null || receipts === null) return "missing"; - const firstReceipt = receipts.find( - (event) => event.run_id === params.turnInstanceId, + const receiptLines = snapshot.events + .map((event, index) => ({ event, line: snapshot.eventLines[index] })) + .filter(({ event }) => receipts.includes(event)); + const firstIndex = receiptLines.findIndex( + ({ event }) => event.run_id === params.turnInstanceId, ); - const firstMatch = firstReceipt === undefined - ? -1 - : snapshot.events.indexOf(firstReceipt); - if (firstMatch < 0) return "missing"; - return snapshot.events.slice(firstMatch + 1).some((event) => { - if ( - event.event_kind !== HEARTBEAT_RECEIPT_EVENT_KIND || - event.goal_id !== params.goalId || - event.agent_id !== params.agentId - ) return false; - if (typeof event.run_id === "string" && event.run_id.trim().length > 0) { - return event.run_id !== params.turnInstanceId; - } - const details = event.details; - return event.status === "normal_run" && - typeof details === "object" && details !== null && !Array.isArray(details) && - (details as JsonObject).ok === true && - (details as JsonObject).should_run === true; - }) - ? "stale" - : "fresh"; + if (firstIndex < 0) return "missing"; + const originatingLine = receiptLines[firstIndex].line; + const laterReceipt = receiptLines.slice(firstIndex + 1).some( + ({ event }) => event.run_id !== params.turnInstanceId, + ); + const laterAdmission = snapshot.malformedLines.some((damaged) => { + if (damaged.line <= originatingLine) return false; + const admitted = committedSuccessor( + damaged, params.goalId, params.agentId, goalRef, + ); + if (admitted === null) return false; + return admitted.kind === "unattributed" || + admitted.runId !== params.turnInstanceId; + }); + return laterReceipt || laterAdmission ? "stale" : "fresh"; } diff --git a/loopx/control_plane/runtime/json_line_members.ts b/loopx/control_plane/runtime/json_line_members.ts new file mode 100644 index 0000000000..2c6d0eb40e --- /dev/null +++ b/loopx/control_plane/runtime/json_line_members.ts @@ -0,0 +1,318 @@ +/** + * Recovery of the intact top-level members of a persisted JSONL row. + * + * The writer of a row owns its commit fact; nothing here decides whether a row + * was finished. This module answers one narrower question for a row whose + * bytes are damaged *after* its writer finished it: which of the members a + * caller names can still be read without guessing. A member whose key or value + * meets an undecodable byte is dropped, and a damaged region is resynchronized + * so a display field early in the serialized row cannot hide the identity + * members that follow it. + * + * Callers name the members they need because those names are schema-level + * top-level keys. Recovery never walks into a nested object to invent a + * top-level member, and it never reports a member twice: a duplicated identity + * is ambiguity, which the caller must reject rather than resolve. + */ + +const WHITESPACE = new Set([" ", "\t", "\n", "\r"]); +const MAX_DEPTH = 64; +const MAX_MEMBERS = 256; +const NUMBER_AT = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/; + +/** One row decoded byte by byte, retaining where decoding had to guess. */ +interface DecodedRow { + readonly text: string; + /** Indexes of characters produced from bytes that are not valid UTF-8. */ + readonly damaged: ReadonlySet; +} + +/** + * Length of the well-formed UTF-8 sequence at `index`, or 0 when the byte does + * not start one. Overlong, surrogate and out-of-range encodings are invalid, + * which keeps a replacement character from ever looking like a legal code point. + */ +function utf8SequenceLength(bytes: Uint8Array, index: number): number { + const first = bytes[index]; + if (first < 0x80) return 1; + const continuation = (at: number, low = 0x80, high = 0xbf): boolean => + at < bytes.length && bytes[at] >= low && bytes[at] <= high; + if (first >= 0xc2 && first <= 0xdf) { + return continuation(index + 1) ? 2 : 0; + } + if (first >= 0xe0 && first <= 0xef) { + const low = first === 0xe0 ? 0xa0 : 0x80; + const high = first === 0xed ? 0x9f : 0xbf; + return continuation(index + 1, low, high) && continuation(index + 2) ? 3 : 0; + } + if (first >= 0xf0 && first <= 0xf4) { + const low = first === 0xf0 ? 0x90 : 0x80; + const high = first === 0xf4 ? 0x8f : 0xbf; + return continuation(index + 1, low, high) && + continuation(index + 2) && continuation(index + 3) ? 4 : 0; + } + return 0; +} + +const VALID_UTF8 = new TextDecoder("utf-8", { ignoreBOM: true }); + +function decodeRow(bytes: Uint8Array): DecodedRow { + let text = ""; + const damaged = new Set(); + let index = 0; + while (index < bytes.length) { + const length = utf8SequenceLength(bytes, index); + if (length === 0) { + damaged.add(text.length); + text += "\uFFFD"; + index += 1; + continue; + } + text += VALID_UTF8.decode(bytes.subarray(index, index + length)); + index += length; + } + return { text, damaged }; +} + +function skipWhitespace(row: DecodedRow, index: number): number { + let next = index; + while (next < row.text.length && WHITESPACE.has(row.text[next])) next += 1; + return next; +} + +interface CompleteValue { + readonly ok: true; + readonly value: unknown; + readonly next: number; +} + +type ParsedValue = CompleteValue | { readonly ok: false }; + +function parseStringAt(row: DecodedRow, index: number): ParsedValue { + let next = index + 1; + let value = ""; + while (next < row.text.length) { + if (row.damaged.has(next)) return { ok: false }; + const character = row.text[next]; + if (character === '"') return { ok: true, value, next: next + 1 }; + if (character !== "\\") { + if (character.charCodeAt(0) < 0x20) return { ok: false }; + value += character; + next += 1; + continue; + } + const escape = row.text[next + 1]; + if (escape === undefined || row.damaged.has(next + 1)) return { ok: false }; + const simple: Record = { + '"': '"', "\\": "\\", "/": "/", b: "\b", f: "\f", + n: "\n", r: "\r", t: "\t", + }; + if (Object.hasOwn(simple, escape)) { + value += simple[escape]; + next += 2; + continue; + } + if (escape !== "u") return { ok: false }; + const code = row.text.slice(next + 2, next + 6); + if (!/^[0-9a-fA-F]{4}$/.test(code)) return { ok: false }; + value += String.fromCharCode(Number.parseInt(code, 16)); + next += 6; + } + return { ok: false }; +} + +function parseValueAt(row: DecodedRow, index: number, depth: number): ParsedValue { + if (depth > MAX_DEPTH) return { ok: false }; + const at = skipWhitespace(row, index); + const character = row.text[at]; + if (character === undefined || row.damaged.has(at)) return { ok: false }; + if (character === '"') return parseStringAt(row, at); + if (character === "{") return parseObjectAt(row, at, depth + 1); + if (character === "[") return parseArrayAt(row, at, depth + 1); + if (row.text.startsWith("true", at)) return { ok: true, value: true, next: at + 4 }; + if (row.text.startsWith("false", at)) return { ok: true, value: false, next: at + 5 }; + if (row.text.startsWith("null", at)) return { ok: true, value: null, next: at + 4 }; + const number = NUMBER_AT.exec(row.text.slice(at)); + if (number === null) return { ok: false }; + return { ok: true, value: Number(number[0]), next: at + number[0].length }; +} + +function parseArrayAt(row: DecodedRow, index: number, depth: number): ParsedValue { + const items: unknown[] = []; + let next = skipWhitespace(row, index + 1); + if (row.text[next] === "]") return { ok: true, value: items, next: next + 1 }; + while (next < row.text.length) { + const item = parseValueAt(row, next, depth); + if (!item.ok) return item; + items.push(item.value); + next = skipWhitespace(row, item.next); + if (row.text[next] === ",") { + next = skipWhitespace(row, next + 1); + continue; + } + if (row.text[next] === "]") return { ok: true, value: items, next: next + 1 }; + return { ok: false }; + } + return { ok: false }; +} + +function parseObjectAt(row: DecodedRow, index: number, depth: number): ParsedValue { + const pairs: [string, unknown][] = []; + let next = skipWhitespace(row, index + 1); + if (row.text[next] === "}") return { ok: true, value: {}, next: next + 1 }; + while (next < row.text.length) { + if (row.text[next] !== '"' || row.damaged.has(next)) return { ok: false }; + const key = parseStringAt(row, next); + if (!key.ok) return key; + next = skipWhitespace(row, key.next); + if (row.text[next] !== ":") return { ok: false }; + next = skipWhitespace(row, next + 1); + const value = parseValueAt(row, next, depth); + if (!value.ok) return value; + pairs.push([key.value as string, value.value]); + next = skipWhitespace(row, value.next); + if (row.text[next] === ",") { + next = skipWhitespace(row, next + 1); + continue; + } + if (row.text[next] === "}") { + return { ok: true, value: Object.fromEntries(pairs), next: next + 1 }; + } + return { ok: false }; + } + return { ok: false }; +} + +/** + * The quote that closes a string whose bytes were damaged. + * + * A damaged byte leaves the string's own end unknowable in theory, but the row + * still shows which structural token follows it. Only a quote followed by the + * separator a finished key or value has can be that end, so scanning stops on + * the first one. Without such a quote the rest of the row is unreadable. + */ +function closingQuoteAfterDamage(row: DecodedRow, from: number): number { + for (let at = from; at < row.text.length; at += 1) { + if (row.damaged.has(at)) continue; + if (row.text[at] === "\\") { + at += 1; + continue; + } + if (row.text[at] !== '"') continue; + const after = skipWhitespace(row, at + 1); + const token = row.text[after]; + if (token === "," || token === "}" || token === "]" || token === ":") return at; + } + return -1; +} + +/** + * Byte offsets of the named keys that sit directly in the row's root object. + * + * Depth is tracked through nested objects and arrays, so a key inside a + * diagnostic `details` value can never be read as a top-level schema member. + * Damage inside a string is resynchronized on the quote its own row shows, which + * keeps the depth available for the members that follow it. + */ +function topLevelMemberOffsets(row: DecodedRow, wanted: ReadonlySet): number[] { + const offsets: number[] = []; + let index = skipWhitespace(row, 0); + if (row.text[index] !== "{" || row.damaged.has(index)) return offsets; + index += 1; + let depth = 1; + let inString = false; + let escaped = false; + while (index < row.text.length && depth >= 1 && offsets.length <= MAX_MEMBERS) { + const character = row.text[index]; + if (row.damaged.has(index)) { + if (!inString) { + index += 1; + continue; + } + const close = closingQuoteAfterDamage(row, index + 1); + if (close < 0) break; + inString = false; + escaped = false; + index = close + 1; + continue; + } + if (inString) { + if (escaped) { + escaped = false; + } else if (character === "\\") { + escaped = true; + } else if (character === '"') { + inString = false; + } + index += 1; + continue; + } + if (character === '"') { + const key = parseStringAt(row, index); + if (key.ok && typeof key.value === "string" && depth === 1) { + const colon = skipWhitespace(row, key.next); + if (row.text[colon] === ":" && !row.damaged.has(colon)) { + if (wanted.has(key.value)) offsets.push(index); + index = colon + 1; + continue; + } + } + inString = true; + escaped = false; + index += 1; + continue; + } + if (character === "{" || character === "[") { + depth += 1; + } else if (character === "}" || character === "]") { + depth -= 1; + } + index += 1; + } + return offsets; +} + +/** What one row still carries of the members a caller named. */ +export interface LineMembers { + /** Complete members: key and value both read with no guessing. */ + readonly members: readonly (readonly [string, unknown])[]; + /** + * Named keys that are present but whose value could not be read intact. + * + * A caller that requires such a member must treat it as unreadable rather than + * absent: the row did contain it, so "absent" would be a different fact. + */ + readonly unreadable: readonly string[]; +} + +/** + * The named top-level members that a row still carries intact. + * + * Only complete members count: a key or value that meets an undecodable byte is + * dropped instead of repaired, and its key is reported as unreadable. After a + * dropped member the scan resynchronizes on the next named key, so damage early + * in the row does not hide the identity members the writer finished later in + * the same row. + */ +export function jsonLineMembers( + bytes: Uint8Array, + keys: readonly string[], +): LineMembers { + const row = decodeRow(bytes); + const wanted = new Set(keys); + const members: [string, unknown][] = []; + const unreadable: string[] = []; + if (wanted.size === 0) return { members, unreadable }; + for (const offset of topLevelMemberOffsets(row, wanted)) { + const key = parseStringAt(row, offset); + if (!key.ok || typeof key.value !== "string") continue; + const colon = skipWhitespace(row, key.next); + const value = parseValueAt(row, skipWhitespace(row, colon + 1), 1); + if (!value.ok) { + unreadable.push(key.value); + continue; + } + members.push([key.value, value.value]); + } + return { members, unreadable: [...new Set(unreadable)] }; +} diff --git a/loopx/control_plane/runtime/receipt_log_snapshot.ts b/loopx/control_plane/runtime/receipt_log_snapshot.ts index da0495f76e..4857c30aed 100644 --- a/loopx/control_plane/runtime/receipt_log_snapshot.ts +++ b/loopx/control_plane/runtime/receipt_log_snapshot.ts @@ -6,9 +6,28 @@ import { setImmediate } from "node:timers/promises"; import type { JsonObject } from "../effect_program.ts"; import { jsonObject } from "../runtime_decode.ts"; +import { jsonLineMembers } from "./json_line_members.ts"; + +/** What one line that failed to parse still carries of the named members. */ +export interface MalformedLineMembers { + readonly line: number; + readonly members: readonly (readonly [string, unknown])[]; + /** Named members the line contained but could not read intact. */ + readonly unreadable: readonly string[]; +} export interface ReceiptLogSnapshot { readonly records: readonly JsonObject[]; + /** Physical 1-based line number of each record, in `records` order. */ + readonly recordLines: readonly number[]; + /** + * Intact members of lines that were not records, restricted to the members a + * caller named. A damaged display field must not erase the identity members + * the row's writer finished elsewhere in the same row; a row whose own writer + * never terminated it supplies no members at all. Callers still decide which + * identities they trust. + */ + readonly malformedLines: readonly MalformedLineMembers[]; readonly firstErrorLine: number | null; } @@ -47,6 +66,19 @@ function remember(key: string, prefix: Prefix): void { } /** `readonly` is erased; freeze nested values so a caller cannot poison reuse. */ +function freezeMalformed(line: MalformedLineMembers): MalformedLineMembers { + for (const [, value] of line.members) { + if (value !== null && typeof value === "object") freezeRecord(value as JsonObject); + } + return Object.freeze({ + line: line.line, + members: Object.freeze(line.members.map(([key, value]) => ( + Object.freeze([key, value] as const) + ))), + unreadable: Object.freeze([...line.unreadable]), + }); +} + function freezeRecord(record: JsonObject): JsonObject { const pending: object[] = [record]; while (pending.length > 0) { @@ -59,6 +91,18 @@ function freezeRecord(record: JsonObject): JsonObject { return record; } +// One strict decoder for the whole read: replacement characters must never +// become a different, valid-looking receipt identity. `ignoreBOM` matches +// Buffer#toString, so a leading BOM still fails JSON parsing as before. +const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +function strictUtf8(bytes: Uint8Array): string | null { + try { + return STRICT_UTF8.decode(bytes); + } catch { + return null; + } +} + function decodeLine(line: string, schemaVersion?: string): JsonObject { const record = jsonObject(JSON.parse(line)); if (record === null || (schemaVersion !== undefined && @@ -81,8 +125,9 @@ function decodeLine(line: string, schemaVersion?: string): JsonObject { export async function readReceiptLogSnapshot( path: string, schemaVersion?: string, + recoveryKeys: readonly string[] = [], ): Promise { - const key = JSON.stringify([resolve(path), schemaVersion ?? null]); + const key = JSON.stringify([resolve(path), schemaVersion ?? null, recoveryKeys]); let bytes: Buffer; try { bytes = await readFile(path); @@ -100,21 +145,42 @@ export async function readReceiptLogSnapshot( prefix = reusable; } else { const records = reusable ? [...reusable.records] : []; + const recordLines = reusable ? [...reusable.recordLines] : []; + const malformedLines = reusable ? [...reusable.malformedLines] : []; let firstErrorLine = reusable?.firstErrorLine ?? null; let lines = reusable?.lines ?? 0; let offset = reusable?.bytes ?? 0; let batchStart = offset; + // The append owner drops uncommitted torn bytes before it writes its own + // row, so this prefix can only end on a finished row. Any rewrite (including + // that drop) still fails the digest check and lands here as a cold read. while (offset < completeBytes) { const end = bytes.indexOf(10, offset); - const line = bytes.subarray(offset, end).toString("utf8"); + const raw = bytes.subarray(offset, end); + const line = strictUtf8(raw); lines += 1; - if (line.trim()) { + let record: JsonObject | null = null; + if (line === null) { + firstErrorLine ??= lines; + const recovered = jsonLineMembers(raw, recoveryKeys); + if (recovered.members.length > 0 || recovered.unreadable.length > 0) { + malformedLines.push(freezeMalformed({ line: lines, ...recovered })); + } + } else if (line.trim()) { try { - records.push(decodeLine(line, schemaVersion)); + record = decodeLine(line, schemaVersion); } catch { firstErrorLine ??= lines; + const recovered = jsonLineMembers(raw, recoveryKeys); + if (recovered.members.length > 0 || recovered.unreadable.length > 0) { + malformedLines.push(freezeMalformed({ line: lines, ...recovered })); + } } } + if (record !== null) { + records.push(record); + recordLines.push(lines); + } offset = end + 1; // Cold/rebuilt histories must let the shared server service other sockets. if (offset - batchStart >= 1024 * 1024) { @@ -123,21 +189,36 @@ export async function readReceiptLogSnapshot( } } prefix = Object.freeze({ - records: Object.freeze(records), firstErrorLine, lines, + records: Object.freeze(records), + recordLines: Object.freeze(recordLines), + malformedLines: Object.freeze(malformedLines), + firstErrorLine, lines, bytes: completeBytes, digest: digest(bytes.subarray(0, completeBytes)), }); } remember(key, prefix); - const tail = bytes.subarray(completeBytes).toString("utf8"); + const tail = strictUtf8(bytes.subarray(completeBytes)); + if (tail === null) { + return Object.freeze({ + records: prefix.records, + recordLines: prefix.recordLines, + malformedLines: prefix.malformedLines, + firstErrorLine: prefix.firstErrorLine ?? prefix.lines + 1, + }); + } if (!tail.trim()) return prefix; try { return Object.freeze({ records: Object.freeze([...prefix.records, decodeLine(tail, schemaVersion)]), + recordLines: Object.freeze([...prefix.recordLines, prefix.lines + 1]), + malformedLines: prefix.malformedLines, firstErrorLine: prefix.firstErrorLine, }); } catch { return Object.freeze({ records: prefix.records, + recordLines: prefix.recordLines, + malformedLines: prefix.malformedLines, firstErrorLine: prefix.firstErrorLine ?? prefix.lines + 1, }); } diff --git a/loopx/control_plane/scheduler/heartbeat_commit.ts b/loopx/control_plane/scheduler/heartbeat_commit.ts index 2017c07143..e58551c6e5 100644 --- a/loopx/control_plane/scheduler/heartbeat_commit.ts +++ b/loopx/control_plane/scheduler/heartbeat_commit.ts @@ -3,6 +3,7 @@ import { readFile } from "node:fs/promises"; import type { JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; +import { parseExactGoalRef } from "../goals/goal_instance_identity.ts"; import { atomicWriteJson, withFileMutationLock, @@ -142,6 +143,7 @@ interface CommitMetadata { effect_id: string; request_digest: string; operation: SchedulerHeartbeatCommitOperation; + goal_instance_id: string | null; } export interface SchedulerHeartbeatCommitResult extends JsonObject { @@ -171,6 +173,9 @@ export interface SchedulerHeartbeatCommitResult extends JsonObject { export interface SchedulerHeartbeatCommitOptions { readonly heartbeatReceiptTurnInstanceId?: string | null; + readonly heartbeatReceiptGoalRef?: JsonObject | null; + /** Project registry; supplies the current authoritative Goal lifetime. */ + readonly registryPath?: string | null; } function stableValue(value: unknown): unknown { @@ -472,6 +477,83 @@ function hostFactsRequest(value: unknown): { return { request, operation_id: operationId }; } +interface CurrentGoalInstance { + readonly kind: "legacy" | "absent" | "instance" | "ambiguous" | "unverifiable"; + readonly goalInstanceId?: string; + readonly active?: boolean; + readonly issue?: string; +} + +/** + * The registered Goal lifetime that owns scheduler writes right now. + * + * Only the registry declares a current instance; a legacy registry keeps the + * established goal-scoped path. A registry that cannot be read is reported as + * legacy so an unreadable local file does not invent an instance. + */ +async function readCurrentGoalInstance( + registryPath: string, + goalId: string, +): Promise { + let text: string; + try { + text = await readFile(registryPath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { kind: "absent" }; + } + return { + kind: "unverifiable", + issue: "the project registry could not be read at commit time", + }; + } + let root: unknown; + try { + root = JSON.parse(text); + } catch { + return { + kind: "unverifiable", + issue: "the project registry is not valid JSON at commit time", + }; + } + const envelope = Array.isArray(root) && root.length === 2 ? root : null; + const header = envelope === null ? null : envelope[0]; + const payload = envelope === null ? root : envelope[1]; + if (header !== null && ( + typeof header !== "object" || header === null || + typeof (header as JsonObject).schema_version !== "string" || + !String((header as JsonObject).schema_version) + .startsWith("loopx_project_registry_envelope_") + )) return { kind: "legacy" }; + if (typeof payload !== "object" || payload === null) return { kind: "legacy" }; + const goals = (payload as JsonObject).goals; + if (!Array.isArray(goals)) return { kind: "legacy" }; + const matching = goals.filter((item) => ( + typeof item === "object" && item !== null && + (item as JsonObject).id === goalId + )); + if (matching.length === 0) return { kind: "absent" }; + // One registry entry names one current lifetime. Duplicate entries are not + // resolved by position: an arbitrary pick could authorize a superseded + // lifetime or silently switch the owner mid-commit. + if (matching.length > 1) { + return { + kind: "ambiguous", + issue: "the project registry lists the Goal more than once", + }; + } + const goal = matching[0] as JsonObject; + const instanceId = goal.goal_instance_id; + if (typeof instanceId !== "string" || !/^ginst_[0-9a-f]{32}$/.test(instanceId)) { + return { kind: "legacy" }; + } + return { + kind: "instance", + goalInstanceId: instanceId, + active: (goal as JsonObject).status === "active", + }; +} + function commitMetadata(state: JsonObject | null): CommitMetadata | null { if (!state) return null; const value = state.heartbeat_commit; @@ -490,6 +572,9 @@ function commitMetadata(state: JsonObject | null): CommitMetadata | null { effect_id: metadata.effect_id, request_digest: metadata.request_digest, operation: metadata.operation, + goal_instance_id: typeof metadata.goal_instance_id === "string" + ? metadata.goal_instance_id + : null, }; } @@ -497,6 +582,7 @@ function withCommitMetadata( state: JsonObject, request: SchedulerHeartbeatCommitRequest, digest: string, + goalInstanceId: string | null, ): JsonObject { return { ...state, @@ -505,6 +591,7 @@ function withCommitMetadata( effect_id: request.effect_id, request_digest: digest, operation: request.operation, + ...(goalInstanceId === null ? {} : { goal_instance_id: goalInstanceId }), }, }; } @@ -635,6 +722,7 @@ function buildAckState( existing: JsonObject | null, path: string, fingerprint: string, + goalInstanceId: string | null, ): { state: JsonObject; extra: Partial } | SchedulerHeartbeatCommitResult { if (request.ack_needed === false && request.apply_needed === false) { @@ -725,7 +813,7 @@ function buildAckState( source: request.source, host_update_failures: priorFailures.length ? priorFailures : null, }); - state = withCommitMetadata(state, request, fingerprint); + state = withCommitMetadata(state, request, fingerprint, goalInstanceId); return { state, extra: { @@ -742,6 +830,7 @@ function buildHostFailureState( existing: JsonObject | null, path: string, fingerprint: string, + goalInstanceId: string | null, ): { state: JsonObject; extra: Partial } | SchedulerHeartbeatCommitResult { if (request.apply_needed !== true) { @@ -808,7 +897,7 @@ function buildHostFailureState( host_update_failure: failure, host_update_failures: failures, }); - state = withCommitMetadata(state, request, fingerprint); + state = withCommitMetadata(state, request, fingerprint, goalInstanceId); return { state, extra: { @@ -819,6 +908,133 @@ function buildHostFailureState( }; } +interface GoalLifetimeRejection { + readonly kind: "reject"; + readonly message: string; + readonly reasonCode: string; +} + +type GoalLifetimeDecision = + | { readonly kind: "accept"; readonly goalInstanceId: string | null } + | GoalLifetimeRejection; + +/** + * Which Goal lifetime the project registry lets this commit write. + * + * A Goal that declares a lifetime cannot be written by a receipt that names + * none: an older host's report would otherwise land on whichever instance is + * current now. A registry without instance identity keeps the established + * goal-scoped path. + */ +function decideGoalLifetime( + current: CurrentGoalInstance, + receiptGoalRef: JsonObject | null, +): GoalLifetimeDecision { + if (receiptGoalRef === null) { + return current.kind === "instance" + ? { + kind: "reject", + message: + "scheduler writeback from this Goal lifetime must name the current Goal reference", + reasonCode: "heartbeat_receipt_goal_ref_missing", + } + : { kind: "accept", goalInstanceId: null }; + } + switch (current.kind) { + case "absent": + return { + kind: "reject", + message: + "scheduler writeback requires the registered Goal; it is not in the project registry", + reasonCode: "goal_instance_absent", + }; + case "unverifiable": + return { + kind: "reject", + message: `the current Goal lifetime is not verifiable from the project registry: ${ + current.issue ?? "unknown registry state" + }`, + reasonCode: "goal_instance_unverifiable", + }; + case "ambiguous": + return { + kind: "reject", + message: `the current Goal lifetime is not verifiable from the project registry: ${ + current.issue ?? "duplicate entries" + }`, + reasonCode: "goal_instance_ambiguous", + }; + case "instance": + if (current.active !== true) { + return { + kind: "reject", + message: + "the Goal is not active in the project registry; refusing scheduler writeback from its previous lifetime", + reasonCode: "goal_instance_inactive", + }; + } + if (receiptGoalRef.goal_instance_id !== current.goalInstanceId) { + return { + kind: "reject", + message: + "heartbeat receipt belongs to a superseded Goal instance; refusing scheduler writeback", + reasonCode: "heartbeat_receipt_goal_instance_superseded", + }; + } + // An `instance` row without an id cannot be matched against the receipt's + // Goal reference; the null keeps the state field absent rather than + // writing a lifetime the registry did not name. + return { kind: "accept", goalInstanceId: current.goalInstanceId ?? null }; + case "legacy": + return { kind: "accept", goalInstanceId: null }; + } +} + +/** Registry transactions hold this marker across their whole read-modify-write. */ +const GOAL_REGISTRY_FENCE_TIMEOUT_MS = 60_000; + +/** + * Rename scheduler state only while the registry still names its lifetime. + * + * The read above is a fast rejection; this fence is the authoritative one. + * Python registry transactions take the same cross-runtime mutation marker + * (`.ts-effect.lock`), so a Goal recreation either publishes its new + * lifetime before this read or waits for the state rename to finish. Without + * the marker a recreation could land between the two and let a superseded + * lifetime overwrite the state its replacement already wrote. + */ +async function writeStateUnderRegistryFence(params: { + registryPath: string | null; + goalId: string; + receiptGoalRef: JsonObject | null; + goalInstanceId: string | null; + write: () => Promise; +}): Promise { + const fenced = async (): Promise => { + if (params.registryPath !== null) { + const current = await readCurrentGoalInstance( + params.registryPath, params.goalId, + ); + const lifetime = decideGoalLifetime(current, params.receiptGoalRef); + if (lifetime.kind === "reject") return lifetime; + if (lifetime.goalInstanceId !== params.goalInstanceId) { + return { + kind: "reject", + message: + "the Goal lifetime changed before the scheduler commit was written; refusing scheduler writeback", + reasonCode: "heartbeat_receipt_goal_instance_superseded", + }; + } + } + await params.write(); + return null; + }; + if (params.registryPath === null) return await fenced(); + return await withFileMutationLock( + params.registryPath, fenced, GOAL_REGISTRY_FENCE_TIMEOUT_MS, + ); +} + export async function evaluateSchedulerHeartbeatCommit( value: unknown, options: SchedulerHeartbeatCommitOptions = {}, @@ -831,6 +1047,30 @@ export async function evaluateSchedulerHeartbeatCommit( options.heartbeatReceiptTurnInstanceId, "heartbeatReceiptTurnInstanceId", ).trim(); + let receiptGoalRef: JsonObject | null = null; + if (options.heartbeatReceiptGoalRef !== undefined && + options.heartbeatReceiptGoalRef !== null) { + const parsed = parseExactGoalRef(options.heartbeatReceiptGoalRef); + if (parsed.kind === "invalid") { + throw new EffectRuntimeRequestError( + `heartbeat receipt goal_ref is invalid: ${parsed.issue}`, + "heartbeat_receipt_goal_ref_invalid", + ); + } + receiptGoalRef = { + goal_id: parsed.value.goalId.value, + goal_instance_id: parsed.value.goalInstanceId.value, + }; + if (receiptGoalRef.goal_id !== request.scope.goalId) { + throw new EffectRuntimeRequestError( + "heartbeat receipt goal_ref does not match the scheduler scope", + "heartbeat_receipt_goal_ref_mismatch", + ); + } + } + const registryPath = options.registryPath === null || options.registryPath === undefined + ? null + : requiredString(options.registryPath, "registryPath").trim(); const path = schedulerStatePath(request.runtime_root, request.scope); const fingerprint = requestDigest(request); // Legacy state stays read-only until the commit holds every applicable @@ -847,12 +1087,38 @@ export async function evaluateSchedulerHeartbeatCommit( const commit = async (): Promise => { const canonical = await readStateUnderLock(path, request.scope); let existing = canonical ?? loaded.state; + // One owner decides which Goal lifetime may still write scheduler state. + // The registry is read under the state lock, before any receipt or replay + // read, so an replaced or cancelled lifetime cannot overwrite or replay a + // newer instance's commit with the same Goal/Agent/surface/state key. + let goalInstanceId: string | null = null; + if (registryPath !== null) { + // One owner decides which Goal lifetime may still write scheduler state. + // The registry is read under the state lock, before any receipt or + // replay read, so an replaced or cancelled lifetime cannot overwrite or + // replay a newer instance's commit with the same Goal/Agent/surface/state + // key. The rename repeats the same decision under the registry marker + // lock, because a Goal recreation can publish a new lifetime while this + // commit is still preparing. + const current = await readCurrentGoalInstance( + registryPath, request.scope.goalId, + ); + const lifetime = decideGoalLifetime(current, receiptGoalRef); + if (lifetime.kind === "reject") { + return result( + request, path, "conflict", existing, lifetime.message, + { reason_code: lifetime.reasonCode }, + ); + } + goalInstanceId = lifetime.goalInstanceId; + } if (receiptTurnInstanceId !== null) { const status = await heartbeatReceiptStatus({ runtimeRoot: request.runtime_root, goalId: request.scope.goalId, agentId: request.scope.agentId, turnInstanceId: receiptTurnInstanceId, + goalRef: receiptGoalRef, }); if (status !== "fresh") { return result( @@ -890,6 +1156,16 @@ export async function evaluateSchedulerHeartbeatCommit( const existingDigest = schedulerHeartbeatCommitStateDigest(existing); const metadata = commitMetadata(existing); if (metadata?.effect_id === request.effect_id) { + const committedInstance = typeof metadata.goal_instance_id === "string" + ? metadata.goal_instance_id + : null; + if (committedInstance !== goalInstanceId) { + return result( + request, path, "conflict", existing, + "effect_id is already bound to a scheduler commit from another Goal instance", + { reason_code: "goal_instance_replay_conflict" }, + ); + } if ( metadata.operation === request.operation && metadata.request_digest === fingerprint @@ -990,8 +1266,8 @@ export async function evaluateSchedulerHeartbeatCommit( ); } const built = request.operation === "ack" - ? buildAckState(request, existing, path, fingerprint) - : buildHostFailureState(request, existing, path, fingerprint); + ? buildAckState(request, existing, path, fingerprint, goalInstanceId) + : buildHostFailureState(request, existing, path, fingerprint, goalInstanceId); if ("status" in built) return built; if (!request.execute) { return result( @@ -1003,7 +1279,25 @@ export async function evaluateSchedulerHeartbeatCommit( built.extra, ); } - await atomicWriteJson(path, stableValue(built.state) as JsonObject); + const fence = await writeStateUnderRegistryFence({ + registryPath, + goalId: request.scope.goalId, + receiptGoalRef, + goalInstanceId, + write: async () => { + await atomicWriteJson(path, stableValue(built.state) as JsonObject); + }, + }); + if (fence !== null) { + return result( + request, + path, + "conflict", + existing, + fence.message, + { reason_code: fence.reasonCode }, + ); + } return result( request, path, diff --git a/loopx/control_plane/scheduler/heartbeat_followup.ts b/loopx/control_plane/scheduler/heartbeat_followup.ts index d1047f0454..c4767a1fd1 100644 --- a/loopx/control_plane/scheduler/heartbeat_followup.ts +++ b/loopx/control_plane/scheduler/heartbeat_followup.ts @@ -1,5 +1,6 @@ import type { JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; +import { parseExactGoalRef } from "../goals/goal_instance_identity.ts"; import { goalPathSegment, heartbeatReceiptStatus, @@ -32,6 +33,8 @@ const FAILURE_CLASSIFICATION = "quota_scheduler_host_update_failure"; export interface SchedulerHeartbeatFollowupRequest extends JsonObject { schema_version: typeof SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA; runtime_root: string; + registry_path: string | null; + goal_ref: JsonObject | null; turn_instance_id: string | null; require_heartbeat_receipt: boolean; before: JsonObject; @@ -81,6 +84,28 @@ function requestObject(value: unknown): SchedulerHeartbeatFollowupRequest { goalPathSegment(hostFacts.goal_id); requireNonEmptyString(hostFacts.agent_id, "agent_id"); followupOperation(hostFacts); + const rawGoalRef = input.goal_ref; + let goalRef: JsonObject | null = null; + if (rawGoalRef !== undefined && rawGoalRef !== null) { + const parsed = parseExactGoalRef(rawGoalRef); + if (parsed.kind === "invalid") { + throw new EffectRuntimeRequestError( + `scheduler follow-up goal_ref is invalid: ${parsed.issue}`, + "scheduler_followup_goal_ref_invalid", + ); + } + goalRef = { + goal_id: parsed.value.goalId.value, + goal_instance_id: parsed.value.goalInstanceId.value, + }; + if (goalRef.goal_id !== String(hostFacts.goal_id)) { + throw new EffectRuntimeRequestError( + "scheduler follow-up goal_ref does not match goal_id", + "scheduler_followup_goal_ref_mismatch", + ); + } + } + const registryPath = optionalText(input.registry_path); const turnInstanceId = optionalText(input.turn_instance_id); const requireReceipt = input.require_heartbeat_receipt === undefined ? turnInstanceId !== null @@ -94,6 +119,8 @@ function requestObject(value: unknown): SchedulerHeartbeatFollowupRequest { return { schema_version: SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA, runtime_root: runtimeRoot, + registry_path: registryPath, + goal_ref: goalRef, turn_instance_id: turnInstanceId, require_heartbeat_receipt: requireReceipt, before: requireJsonObject(input.before ?? {}, "before"), @@ -341,6 +368,7 @@ export async function evaluateSchedulerHeartbeatFollowup( goalId: String(facts.goal_id), agentId: String(facts.agent_id), turnInstanceId: String(request.turn_instance_id), + goalRef: request.goal_ref, }); if (status !== "fresh") return receiptFailure(request, status); } @@ -351,6 +379,10 @@ export async function evaluateSchedulerHeartbeatFollowup( heartbeatReceiptTurnInstanceId: request.require_heartbeat_receipt ? request.turn_instance_id : null, + // A manual ACK carries no Turn fence, but a captured Goal lifetime still + // owns the state and replay scope it may write. + heartbeatReceiptGoalRef: request.goal_ref, + registryPath: request.registry_path, }); const receiptStatus = commitReceiptFailureStatus(commit); if (receiptStatus !== null) return receiptFailure(request, receiptStatus); diff --git a/loopx/control_plane/scheduler/heartbeat_followup_cli.ts b/loopx/control_plane/scheduler/heartbeat_followup_cli.ts index d7c448debf..7a0a055899 100644 --- a/loopx/control_plane/scheduler/heartbeat_followup_cli.ts +++ b/loopx/control_plane/scheduler/heartbeat_followup_cli.ts @@ -29,6 +29,8 @@ interface ParsedArgs { chunks: string[]; values: Record; booleans: Set; + /** Invocation routing only: the project registry that owns Goal lifetimes. */ + registry_path: string | null; } function errorEnvelope(error: unknown) { @@ -55,6 +57,7 @@ function parseArgs(argv: string[]): ParsedArgs { const booleans = new Set(); const chunks: string[] = []; const positionals: string[] = []; + let registryPath: string | null = null; const valueOptions = new Set([ "--format", "--registry", @@ -103,7 +106,8 @@ function parseArgs(argv: string[]): ParsedArgs { ); } if (option === FACTS_FLAG) chunks.push(value); - else if (option !== "--available-capability" && option !== "--registry") { + else if (option === "--registry") registryPath = value; + else if (option !== "--available-capability") { values[option] = value; } continue; @@ -150,6 +154,7 @@ function parseArgs(argv: string[]): ParsedArgs { chunks, values, booleans, + registry_path: registryPath, }; } @@ -289,6 +294,8 @@ function requestFromArgs(argv: string[]): { request: Record; fo request: { schema_version: SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA, runtime_root: parsed.runtime_root, + registry_path: parsed.registry_path, + goal_ref: hint.goal_ref ?? null, turn_instance_id: parsed.turn_instance_id, require_heartbeat_receipt: true, before: hint.before ?? {}, diff --git a/loopx/control_plane/scheduler/scheduler_hint.py b/loopx/control_plane/scheduler/scheduler_hint.py index 356ea03b99..653e43715e 100644 --- a/loopx/control_plane/scheduler/scheduler_hint.py +++ b/loopx/control_plane/scheduler/scheduler_hint.py @@ -49,11 +49,26 @@ ) +def _scheduler_host_followup_goal_ref( + goal_ref: Mapping[str, Any] | None, +) -> dict[str, str] | None: + """Carry one exact Goal instance as transport data, never as an authority claim.""" + + if not isinstance(goal_ref, Mapping): + return None + goal_id = str(goal_ref.get("goal_id") or "").strip() + goal_instance_id = str(goal_ref.get("goal_instance_id") or "").strip() + if not goal_id or not goal_instance_id: + return None + return {"goal_id": goal_id, "goal_instance_id": goal_instance_id} + + def _scheduler_host_followup_transport_args( scheduler_host_facts: Mapping[str, Any] | None, *, before: Mapping[str, Any] | None, use_current_hint: bool, + goal_ref: Mapping[str, Any] | None = None, ) -> list[str]: """Encode a bounded, transport-only native scheduler follow-up hint.""" @@ -63,6 +78,7 @@ def _scheduler_host_followup_transport_args( "schema_version": "loopx_scheduler_host_followup_hint_v0", "before": compact_quota_decision(dict(before or {})), "use_current_hint": use_current_hint, + "goal_ref": _scheduler_host_followup_goal_ref(goal_ref), "host_facts": dict(scheduler_host_facts), } raw = json.dumps( @@ -100,7 +116,9 @@ def _bounded_scheduler_followup_cli_args( def _encode_followup_args( - packet: dict[str, Any], before: Mapping[str, Any] | None + packet: dict[str, Any], + before: Mapping[str, Any] | None, + goal_ref: Mapping[str, Any] | None = None, ) -> dict[str, Any]: if "cli_args" not in packet: return packet @@ -121,6 +139,7 @@ def _encode_followup_args( argument["host_facts"], before=before, use_current_hint=argument["use_current_hint"], + goal_ref=goal_ref, ) args.extend(encoded) native_args.extend(encoded) @@ -137,9 +156,11 @@ def _encode_followup_args( def _finish_transport( - packet: dict[str, Any], before: Mapping[str, Any] | None + packet: dict[str, Any], + before: Mapping[str, Any] | None, + goal_ref: Mapping[str, Any] | None = None, ) -> dict[str, Any]: - completed = _encode_followup_args(packet, before) + completed = _encode_followup_args(packet, before, goal_ref) # Source metadata can also contain cli_args. Only these owned wire slots are executable hints. for view in ("app_automation", "codex_app"): app = completed.get(view) @@ -149,13 +170,19 @@ def _finish_transport( for followup in ("ack_hint", "failure_hint"): command = app.get(followup) if isinstance(command, dict): - encoded_app[followup] = _encode_followup_args(command, before) + encoded_app[followup] = _encode_followup_args( + command, before, goal_ref + ) completed[view] = encoded_app return completed def _project_hint( - operation: str, params: dict[str, Any], before: Mapping[str, Any] | None = None + operation: str, + params: dict[str, Any], + before: Mapping[str, Any] | None = None, + *, + goal_ref: Mapping[str, Any] | None = None, ) -> dict[str, Any]: request = {"operation": operation, **params} large_snapshot = ( @@ -173,7 +200,7 @@ def _project_hint( or not isinstance(projected.get("hint"), dict) ): raise RuntimeError("TypeScript scheduler hint projection shape mismatch") - return _finish_transport(projected["hint"], before) + return _finish_transport(projected["hint"], before, goal_ref) def _identity_json(payload: dict[str, Any]) -> dict[str, str]: @@ -344,11 +371,12 @@ def build_scheduler_hint( current_time, ), } - return _project_hint("hint", params, payload) + return _project_hint("hint", params, payload, goal_ref=payload.get("goal_ref")) def _followup(operation: str, kwargs: dict[str, Any]) -> dict[str, Any]: before = kwargs.pop("scheduler_before", None) + goal_ref = kwargs.pop("goal_ref", None) if "available_capabilities" in kwargs: kwargs["available_capabilities"] = _capability_values( kwargs["available_capabilities"] @@ -368,7 +396,7 @@ def _followup(operation: str, kwargs: dict[str, Any]) -> dict[str, Any]: kwargs[field] = str(kwargs[field] or "").strip() if "host_match_observed" in kwargs: kwargs["host_match_observed"] = bool(kwargs["host_match_observed"]) - return _project_hint(operation, kwargs, before) + return _project_hint(operation, kwargs, before, goal_ref=goal_ref) def build_app_automation_scheduler_ack_hint( @@ -385,6 +413,7 @@ def build_app_automation_scheduler_ack_hint( state_key: str = APP_AUTOMATION_STATEFUL_BACKOFF_STATE_KEY, scheduler_host_facts: Mapping[str, Any] | None = None, scheduler_before: Mapping[str, Any] | None = None, + goal_ref: Mapping[str, Any] | None = None, ) -> dict[str, Any]: return _followup("ack", dict(locals())) @@ -400,6 +429,7 @@ def build_app_automation_scheduler_failure_hint( scheduler_before: Mapping[str, Any] | None = None, surface: str = CODEX_APP_SURFACE, state_key: str = APP_AUTOMATION_STATEFUL_BACKOFF_STATE_KEY, + goal_ref: Mapping[str, Any] | None = None, ) -> dict[str, Any]: return _followup("failure", dict(locals())) diff --git a/loopx/control_plane/status/collection.py b/loopx/control_plane/status/collection.py index 8d65923da3..46abee332a 100644 --- a/loopx/control_plane/status/collection.py +++ b/loopx/control_plane/status/collection.py @@ -71,6 +71,7 @@ class StatusCollectionContext: build_goal_channel_notification_projection: StatusCallback status_control_plane_context_limit: int max_todo_index_items: int + tolerate_malformed_settlement_history: bool = False def collect_status( @@ -89,6 +90,7 @@ def collect_status( agent_lane_id: str | None = None, allow_missing_registry: bool = False, include_presentation_projections: bool = True, + tolerate_malformed_settlement_history: bool = False, ) -> dict[str, Any]: """Collect the status read model. @@ -187,6 +189,10 @@ def canonical_runtime_for_goal(current_goal_id: str) -> Path: current_registry=registry, registry_path=registry_path, include_goal_channel_projection=include_presentation_projections, + tolerate_malformed_settlement_history=( + tolerate_malformed_settlement_history + or context.tolerate_malformed_settlement_history + ), agent_lane_id=agent_lane_id, ) # No later projection consumes the shared status snapshot. diff --git a/loopx/control_plane/work_items/attention_queue.py b/loopx/control_plane/work_items/attention_queue.py index a9c21a41e7..5f6ae9f88e 100644 --- a/loopx/control_plane/work_items/attention_queue.py +++ b/loopx/control_plane/work_items/attention_queue.py @@ -50,6 +50,7 @@ class AttentionQueueContext: monitor_signal_waiting_on: str registry_path: Path | None = None external_progress_review_context: Optional[Callable[..., dict[str, Any] | None]] = None + tolerate_malformed_settlement_history: bool = False def merge_global_registry_findings( @@ -336,7 +337,14 @@ def build_attention_queue( next_action_projection_warning=context.next_action_projection_warning, autonomous_replan_obligation_from_runs=context.autonomous_replan_obligation_from_runs, external_progress_review=external_progress_review, - effective_turn_cadence=effective_turn_cadence_context(goal, runtime_root, registry_path=context.registry_path), + effective_turn_cadence=effective_turn_cadence_context( + goal, + runtime_root, + registry_path=context.registry_path, + tolerate_malformed_history=( + context.tolerate_malformed_settlement_history + ), + ), agent_lane_id=agent_lane_id, ) item["quota"] = context.quota_status( diff --git a/loopx/control_plane/work_items/replan_history_codec.py b/loopx/control_plane/work_items/replan_history_codec.py index 6b0d63f1c2..0b41d5ba2b 100644 --- a/loopx/control_plane/work_items/replan_history_codec.py +++ b/loopx/control_plane/work_items/replan_history_codec.py @@ -38,6 +38,7 @@ def effective_turn_cadence_context( registry_path: Path | None = None, goal_ref: dict[str, str] | None = None, source_admission: dict[str, Any] | None = None, + tolerate_malformed_history: bool = False, ) -> dict[str, Any] | None: """Encode a Goal whose live cadence was resolved by the caller.""" profile = goal.get("execution_profile") or {} @@ -58,6 +59,11 @@ def effective_turn_cadence_context( "settlement_source": { "runtime_root": str(runtime_root.resolve()), "goal_id": goal["id"], + **( + {"tolerate_malformed_history": True} + if tolerate_malformed_history + else {} + ), }, } diff --git a/loopx/control_plane/work_items/replan_history_settlement.ts b/loopx/control_plane/work_items/replan_history_settlement.ts index be0578ff4b..858afe483b 100644 --- a/loopx/control_plane/work_items/replan_history_settlement.ts +++ b/loopx/control_plane/work_items/replan_history_settlement.ts @@ -23,6 +23,11 @@ export async function projectSettledReplanHistory(value: unknown): Promise quotaOwnerOwnsProjection(owner, jsonObject(raw)?.goal_ref)); const rolloutSnapshot = await readGoalRolloutEventSnapshot(runtimeRoot, goalId); - const events = strictGoalRolloutEvents(rolloutSnapshot); + // Scheduler writeback returns only need the receipts they can read; a + // damaged row is never a receipt. Settlement-authority callers keep the + // strict view. + const events = tolerateMalformedHistory === true + ? rolloutSnapshot?.events ?? [] + : strictGoalRolloutEvents(rolloutSnapshot); const receiptKeys = new Set(events.filter(event => event.event_kind === "quota_should_run" && event.goal_id === goalId && typeof event.agent_id === "string" && typeof event.run_id === "string" && @@ -52,7 +62,12 @@ export async function projectSettledReplanHistory(value: unknown): Promise(); const seen = new Set(); for (const raw of settlementRuns) { diff --git a/loopx/quota.py b/loopx/quota.py index 0c8bbfbfb6..6c58264055 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1249,6 +1249,8 @@ def record_quota_scheduler_ack( operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None = None, before_decision: Mapping[str, Any] | None = None, turn_instance_id: str | None = None, + goal_ref: Mapping[str, Any] | None = None, + registry_path: Path | None = None, ) -> dict[str, Any]: safe_goal_id = _validate_goal_id_path_segment(str(goal_id or "")) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -1284,6 +1286,8 @@ def record_quota_scheduler_ack( use_current_hint=use_current_hint, host_match_observed=host_match_observed, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, + registry_path=registry_path, ) diff --git a/loopx/rollout_event_log.py b/loopx/rollout_event_log.py index c0b0b6dc9d..e5d83352ca 100644 --- a/loopx/rollout_event_log.py +++ b/loopx/rollout_event_log.py @@ -233,21 +233,50 @@ def _append_rollout_event_line( log_path: Path, payload: Mapping[str, Any], ) -> None: - """Append one readable JSONL row after any torn final record.""" + """Append one readable JSONL row after any torn final record. + + A row is committed because the writer that appended it also wrote its + terminating newline. Bytes a writer left behind before that newline were + never committed and are not readable as a row, so this append drops them + instead of finishing them: giving them a newline would claim a commit their + writer never made, and marking them would introduce a format every older + reader has to learn. Callers hold the cross-runtime append lock for this + log, so an unterminated tail here is abandoned bytes, never a write still + in progress. + """ encoded = ( json.dumps(dict(payload), sort_keys=True, ensure_ascii=False).encode("utf-8") + b"\n" ) with log_path.open("a+b") as handle: - handle.seek(0, 2) - if handle.tell() > 0: - handle.seek(-1, 2) - if handle.read(1) != b"\n": - handle.write(b"\n") + _drop_unterminated_tail(handle) handle.write(encoded) +def _drop_unterminated_tail(handle: BinaryIO) -> None: + """Discard bytes after the last newline; they were never committed.""" + + end = handle.seek(0, 2) + if end == 0: + return + handle.seek(-1, 2) + if handle.read(1) == b"\n": + return + cut = 0 + position = end + while position > 0: + start = max(0, position - 65536) + handle.seek(start) + chunk = handle.read(position - start) + index = chunk.rfind(b"\n") + if index >= 0: + cut = start + index + 1 + break + position = start + handle.truncate(cut) + + def rollout_event_log_path(runtime_root: Path, goal_id: str) -> Path: # Goal ids are used as directory names throughout the control plane. Keep # this shared path helper fail-closed so every reader and writer inherits diff --git a/loopx/semantics/project_registry_io_classifications_v1.json b/loopx/semantics/project_registry_io_classifications_v1.json index 7d92e680dc..3ab5f90fc5 100644 --- a/loopx/semantics/project_registry_io_classifications_v1.json +++ b/loopx/semantics/project_registry_io_classifications_v1.json @@ -5,6 +5,7 @@ "loopx/control_plane/goals/deletion_service.py::._restore_locked_registry::direct_json_write:atomic_write_json#1": "global_registry_io", "loopx/control_plane/goals/deletion_service.py::._write_deletion::direct_json_read:read_json#1": "global_registry_io", "loopx/control_plane/goals/deletion_service.py::._write_locked_registry::direct_json_write:atomic_write_json#1": "global_registry_io", + "loopx/control_plane/scheduler/heartbeat_commit.ts::readCurrentGoalInstance::direct_json_read:readFile#1": "codec_internal", "loopx/state_migration.py::.legacy_registry_goal_ids::direct_json_read:read_json_object#1": "legacy_registry_source", "loopx/state_migration.py::.migrate_legacy_state::direct_json_read:read_json_object#1": "legacy_registry_source" } diff --git a/loopx/status.py b/loopx/status.py index cbb5528f2f..91c42e7ae8 100644 --- a/loopx/status.py +++ b/loopx/status.py @@ -1091,6 +1091,7 @@ def build_attention_queue( current_registry: dict[str, Any] | None = None, registry_path: Path | None = None, include_goal_channel_projection: bool = True, + tolerate_malformed_settlement_history: bool = False, agent_lane_id: str | None = None, ) -> dict[str, Any]: def request_active_state_todo_fields( @@ -1154,6 +1155,9 @@ def request_active_state_todo_fields( monitor_signal_waiting_on=MONITOR_SIGNAL_WAITING_ON, external_progress_review_context=external_progress_review_context, registry_path=registry_path, + tolerate_malformed_settlement_history=( + tolerate_malformed_settlement_history + ), ), runtime_root=runtime_root, include_task_graph=include_task_graph, @@ -1325,7 +1329,10 @@ def build_status_runtime_summaries( ) -def build_status_collection_context() -> StatusCollectionContext: +def build_status_collection_context( + *, + tolerate_malformed_settlement_history: bool = False, +) -> StatusCollectionContext: return StatusCollectionContext( load_registry=load_registry, resolve_runtime_root=resolve_runtime_root, @@ -1341,6 +1348,9 @@ def build_status_collection_context() -> StatusCollectionContext: build_goal_channel_notification_projection=build_goal_channel_notification_projection, status_control_plane_context_limit=STATUS_CONTROL_PLANE_CONTEXT_LIMIT, max_todo_index_items=MAX_TODO_INDEX_ITEMS, + tolerate_malformed_settlement_history=( + tolerate_malformed_settlement_history + ), ) @@ -1359,6 +1369,7 @@ def collect_status( agent_lane_id: str | None = None, allow_missing_registry: bool = False, include_presentation_projections: bool = True, + tolerate_malformed_settlement_history: bool = False, ) -> dict[str, Any]: with effect_runtime_request_scope(): return _collect_status_read_model( @@ -1377,5 +1388,9 @@ def collect_status( activation_state_filter=activation_state_filter, agent_lane_id=agent_lane_id, include_presentation_projections=include_presentation_projections, - context=build_status_collection_context(), + context=build_status_collection_context( + tolerate_malformed_settlement_history=( + tolerate_malformed_settlement_history + ), + ), ) diff --git a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py new file mode 100644 index 0000000000..c7b83a00a1 --- /dev/null +++ b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py @@ -0,0 +1,633 @@ +from __future__ import annotations + +import json +from argparse import Namespace +from pathlib import Path + +import pytest + +from examples.control_plane.quota_plan_fixtures import ( + SCOPED_AGENT_ID, + write_cli_fixture, +) +from loopx.cli_commands import quota_scheduler_followup +from loopx.control_plane.quota.scheduler_ack import _run_native_followup +from loopx.control_plane.testing.canary_harness import run_json_cli_result +from loopx.rollout_event_log import build_rollout_event, rollout_event_log_path + + +GOAL_ID = "receipt-fence" +AGENT_ID = "agent-a" + + +def _append(runtime_root: Path, **fields: object) -> None: + path = rollout_event_log_path(runtime_root, GOAL_ID) + path.parent.mkdir(parents=True, exist_ok=True) + event = build_rollout_event(goal_id=GOAL_ID, agent_id=AGENT_ID, **fields) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, ensure_ascii=False) + "\n") + + +def test_native_followup_request_carries_the_turn_fence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + captured: dict[str, object] = {} + + class _Completed: + returncode = 0 + stdout = '{"ok": true}' + stderr = "" + + class _Probe: + returncode = 0 + stdout = "v24.19.0" + stderr = "" + + def _run(command, **kwargs): + if "input" not in kwargs: + return _Probe() + captured["request"] = json.loads(kwargs["input"]) + return _Completed() + + monkeypatch.setattr( + "loopx.control_plane.quota.scheduler_ack.subprocess.run", _run + ) + goal_ref = { + "goal_id": "receipt-fence", + "goal_instance_id": "ginst_" + "a" * 32, + } + _run_native_followup( + runtime_root=tmp_path, + before={}, + host_facts={}, + use_current_hint=False, + reason_summary=None, + turn_instance_id="turn-1", + goal_ref=goal_ref, + ) + request = captured["request"] + assert request["turn_instance_id"] == "turn-1" + assert request["goal_ref"] == goal_ref + assert request["require_heartbeat_receipt"] is True + + _run_native_followup( + runtime_root=tmp_path, + before={}, + host_facts={}, + use_current_hint=False, + reason_summary=None, + ) + request = captured["request"] + assert request["turn_instance_id"] is None + assert request["goal_ref"] is None + assert request["require_heartbeat_receipt"] is False + + +def test_ack_route_forwards_the_turn_fence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from loopx import quota + + captured: dict[str, object] = {} + + def _capture(before, **kwargs): + captured.update(kwargs) + return {"ok": True} + + monkeypatch.setattr( + quota, "record_quota_scheduler_ack_for_decision", _capture + ) + goal_ref = { + "goal_id": "goals", + "goal_instance_id": "ginst_" + "b" * 32, + } + quota.record_quota_scheduler_ack( + {"runtime_root": str(tmp_path)}, + goal_id="goals", + agent_id="agent-a", + before_decision={"decision": {}}, + turn_instance_id="turn-1", + goal_ref=goal_ref, + ) + assert captured["turn_instance_id"] == "turn-1" + assert captured["goal_ref"] == goal_ref + + +def _followup_args(**overrides: object) -> Namespace: + values: dict[str, object] = { + "quota_command": "scheduler-ack-current", + "goal_id": GOAL_ID, + "agent_id": AGENT_ID, + "codex_app_current_rrule": None, + "applied_rrule": "FREQ=MINUTELY;INTERVAL=3", + "host_match_observed": False, + "execute": True, + "surface": "trae_app", + "state_key": None, + "available_capabilities": [], + "reset_token": None, + "identity_signature": None, + "reason_summary": None, + "use_current_hint": False, + } + values.update(overrides) + return Namespace(**values) + + +def _capture_ack(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]: + calls: list[dict[str, object]] = [] + + def _ack(_status, **kwargs): + calls.append(kwargs) + return {"ok": True} + + monkeypatch.setattr( + quota_scheduler_followup, "record_quota_scheduler_ack", _ack + ) + monkeypatch.setattr( + quota_scheduler_followup, + "_build_scheduler_followup_decision", + lambda *_args, **_kwargs: {"effective_action": "execute_todo"}, + ) + return calls + + +def test_superseded_receipt_is_decided_by_the_native_fence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runtime_root = tmp_path / "runtime" + _append(runtime_root, event_kind="quota_should_run", run_id="turn-1") + _append(runtime_root, event_kind="quota_should_run", run_id="turn-2") + calls = _capture_ack(monkeypatch) + + payload = quota_scheduler_followup.build_scheduler_followup_payload( + {}, + _followup_args(), + registry_path=tmp_path / "registry.json", + runtime_root=runtime_root, + turn_instance_id="turn-1", + scheduler_context=None, + operator_inbox_urgency_projector=lambda **_kwargs: {}, + ) + + # The Python route keeps no second freshness opinion: it binds the + # originating Turn and lets the native fence answer stale/missing. + assert payload == {"ok": True} + assert calls and calls[0]["turn_instance_id"] == "turn-1" + + +def test_scheduler_followup_builds_from_the_tolerant_settlement_view( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + captured: dict[str, object] = {} + + def _decision(*_args, **kwargs): + captured.update(kwargs) + return {"effective_action": "execute_todo"} + + monkeypatch.setattr( + quota_scheduler_followup, + "build_live_quota_should_run_decision", + _decision, + ) + quota_scheduler_followup._build_scheduler_followup_decision( + {}, + _followup_args(), + registry_path=tmp_path / "registry.json", + runtime_root=tmp_path / "runtime", + heartbeat_receipt=None, + turn_instance_id=None, + codex_app_current_rrule=None, + scheduler_context=None, + operator_inbox_urgency_projector=lambda **_kwargs: {}, + ) + assert captured["tolerate_malformed_settlement_history"] is True + + +def test_malformed_log_line_does_not_block_the_ack_route( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runtime_root = tmp_path / "runtime" + _append(runtime_root, event_kind="quota_should_run", run_id="turn-1") + path = rollout_event_log_path(runtime_root, GOAL_ID) + with path.open("ab") as handle: + handle.write( + b'{"schema_version":"loopx_rollout_event_v0",' + b'"event_kind":"quota_should_run","goal_id":"receipt-fence",' + b'"agent_id":"agent-a","run_id":"turn-bad","summary":"' + b"\xff" + b'"}\n' + ) + calls = _capture_ack(monkeypatch) + + payload = quota_scheduler_followup.build_scheduler_followup_payload( + {}, + _followup_args(), + registry_path=tmp_path / "registry.json", + runtime_root=runtime_root, + turn_instance_id="turn-1", + scheduler_context=None, + operator_inbox_urgency_projector=lambda **_kwargs: {}, + ) + + assert payload == {"ok": True} + assert calls and calls[0]["turn_instance_id"] == "turn-1" + + +def test_fresh_receipt_still_reaches_the_ack_with_its_turn_fence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runtime_root = tmp_path / "runtime" + _append(runtime_root, event_kind="quota_should_run", run_id="turn-1") + calls: list[dict[str, object]] = [] + + def _ack(_status, **kwargs): + calls.append(kwargs) + return {"ok": True} + + monkeypatch.setattr( + quota_scheduler_followup, "record_quota_scheduler_ack", _ack + ) + monkeypatch.setattr( + quota_scheduler_followup, + "_build_scheduler_followup_decision", + lambda *_args, **_kwargs: {"effective_action": "execute_todo"}, + ) + args = Namespace( + quota_command="scheduler-ack-current", + goal_id=GOAL_ID, + agent_id=AGENT_ID, + codex_app_current_rrule=None, + applied_rrule="FREQ=MINUTELY;INTERVAL=3", + host_match_observed=False, + execute=True, + surface="trae_app", + state_key=None, + available_capabilities=[], + reset_token=None, + identity_signature=None, + reason_summary=None, + use_current_hint=False, + ) + payload = quota_scheduler_followup.build_scheduler_followup_payload( + {}, + args, + registry_path=tmp_path / "registry.json", + runtime_root=runtime_root, + turn_instance_id="turn-1", + scheduler_context=None, + operator_inbox_urgency_projector=lambda **_kwargs: {}, + ) + + assert payload == {"ok": True} + assert calls and calls[0]["turn_instance_id"] == "turn-1" + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + ("command", "include_turn"), + [ + ("scheduler-ack-current", True), + ("scheduler-fail-current", True), + ("scheduler-ack", True), + ("scheduler-ack", False), + ], +) +def test_real_compatibility_cli_survives_a_damaged_audit_row( + tmp_path: Path, provider: str, command: str, include_turn: bool +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / provider, scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + turn_id = "turn-damaged-history" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + turn_id, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + with log_path.open("ab") as handle: + handle.write( + b'{"schema_version":"loopx_rollout_event_v0",' + b'"event_kind":"quota_should_run","goal_id":"needs-operator",' + b'"agent_id":"' + SCOPED_AGENT_ID.encode() + b'",' + b'"summary":"' + b"\xff" + b'"}\n' + ) + + # The ACK derives its applied RRULE from the current hint; the failure + # return records the hint target. Both are ordinary scheduler writebacks. + operation_args = ("--use-current-hint",) if command == "scheduler-ack" else () + turn_args = ("--turn-instance-id", turn_id) if include_turn else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + *turn_args, + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, result + assert result["ok"] is True, result + assert result["scheduler_state_mutated"] is True + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_scheduler_followup_cache_preview_cannot_poison_strict_status( + tmp_path: Path, provider: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / provider, scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + turn_id = "turn-cache-preview" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + turn_id, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + with log_path.open("ab") as handle: + handle.write( + b'{"schema_version":"loopx_rollout_event_v0",' + b'"event_kind":"quota_should_run","goal_id":"needs-operator",' + b'"agent_id":"' + SCOPED_AGENT_ID.encode() + b'",' + b'"summary":"' + b"\xff" + b'"}\n' + ) + + before_code, before_status = run_json_cli_result( + "quota", + "status", + "--goal-id", + goal_id, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + code, preview = run_json_cli_result( + "quota", + "scheduler-ack-current", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + turn_id, + "--execute", + "--write-projection-cache", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, preview + cache = preview["status_projection_cache"] + assert cache["written"] is False + assert cache["skipped_reason"] == "tolerant_scheduler_followup_view" + assert not Path(cache["path"]).exists() + + after_code, after_status = run_json_cli_result( + "quota", + "status", + "--goal-id", + goal_id, + "--use-projection-cache", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert after_code == before_code + assert after_status.get("status_projection_cache", {}).get("hit") is not True + + +def _damage_committed_summary(path: Path, turn_id: str) -> None: + """Break one byte of a committed receipt's display summary only.""" + lines = path.read_bytes().split(b"\n") + damaged = 0 + for index, line in enumerate(lines): + if turn_id.encode() not in line: + continue + if b"stall=not_applicable" not in line: + raise AssertionError("fixture receipt has no summary to damage") + lines[index] = line.replace(b"stall=not_applicable", b"stall=\xffnot_applicable", 1) + damaged += 1 + assert damaged == 1 + path.write_bytes(b"\n".join(lines)) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +def test_damaged_committed_successor_still_supersedes_the_older_turn( + tmp_path: Path, provider: str, command: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / provider, scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + for turn_id in ("turn-1", "turn-2"): + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + turn_id, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + _damage_committed_summary(log_path, "turn-2") + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # The damaged row still proves the Turn its owner committed, so the older + # return must not regain write authority. + assert code == 1, result + assert result["ok"] is False + assert result["status"] == "heartbeat_receipt_stale" + assert result["scheduler_state_mutated"] is False + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("command", ["scheduler-ack-current", "scheduler-fail-current"]) +@pytest.mark.parametrize("row_kind", ["other_agent", "validation"]) +def test_a_duplicated_turn_outside_this_scope_does_not_retire_the_turn( + tmp_path: Path, provider: str, command: str, row_kind: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-{row_kind}", + scoped_agents=True, + provider=provider, + ) + goal_id = "needs-operator" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + "turn-1", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + agent_id = "other-agent" if row_kind == "other_agent" else SCOPED_AGENT_ID + event_kind = "quota_should_run" if row_kind == "other_agent" else "validation" + log_path = rollout_event_log_path(runtime_root, goal_id) + with log_path.open("ab") as handle: + handle.write( + ( + '{"agent_id":"' + agent_id + '","event_kind":"' + event_kind + '",' + '"goal_id":"' + goal_id + '","run_id":"turn-2",' + '"schema_version":"loopx_rollout_event_v0","run_id":"turn-3"}\n' + ).encode() + ) + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # The row is not a receipt of this Goal/Agent/instance, so its duplicated + # Turn cannot stand in for a same-scope successor. + assert code == 0, result + assert result["ok"] is True, result + assert result["scheduler_state_mutated"] is True + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_torn_tail_damage_does_not_manufacture_a_successor( + tmp_path: Path, provider: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / provider, scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + "turn-1", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + with log_path.open("ab") as handle: + # An unterminated write may be torn or still in flight; it proves no + # committed admission, so it cannot retire the current receipt. + handle.write( + b'{"schema_version":"loopx_rollout_event_v0",' + b'"event_kind":"quota_should_run","goal_id":"needs-operator",' + b'"agent_id":"' + SCOPED_AGENT_ID.encode() + b'",' + b'"run_id":"turn-torn","summary":"\xff' + ) + + code, result = run_json_cli_result( + "quota", + "scheduler-ack-current", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + "--use-current-hint", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, result + assert result["ok"] is True, result + assert result["scheduler_state_mutated"] is True diff --git a/tests/control_plane/test_scheduler_hint_single_request.py b/tests/control_plane/test_scheduler_hint_single_request.py index 5ee1543d39..1941fa9b3b 100644 --- a/tests/control_plane/test_scheduler_hint_single_request.py +++ b/tests/control_plane/test_scheduler_hint_single_request.py @@ -1,3 +1,6 @@ +import base64 +import json +import zlib from copy import deepcopy import pytest @@ -71,6 +74,46 @@ def observed(method, params, **kwargs): ) +def _native_followup_payload(cli_args: list[str]) -> dict[str, object]: + encoded = "".join( + cli_args[index + 1] + for index, value in enumerate(cli_args) + if value == "--scheduler-host-facts-chunk" + ) + padding = "=" * (-len(encoded) % 4) + return json.loads( + zlib.decompress(base64.urlsafe_b64decode(encoded + padding)) + ) + + +def test_complete_hint_carries_the_exact_goal_instance_scope() -> None: + goal_ref = { + "goal_id": "hint-fixture", + "goal_instance_id": "ginst_" + "e" * 32, + } + payload = { + "goal_id": "hint-fixture", + "goal_ref": goal_ref, + "agent_identity": {"agent_id": "agent-fixture"}, + "heartbeat_recommendation": {"recommended_mode": "normal_run"}, + "recommended_action": "Perform the bounded work.", + "effective_action": "normal_run", + "interaction_contract": {"mode": "normal_run"}, + } + hint = scheduler_hint.build_scheduler_hint( + payload, + arbitration=SchedulerArbitration( + SchedulerDisposition("active_work"), "active_work", "normal_run" + ), + scheduler_execution_context=scheduler_execution_context_for_runtime_profile( + "codex_app_heartbeat" + ), + ) + + ack = hint["app_automation"]["ack_hint"] + assert _native_followup_payload(ack["cli_args"])["goal_ref"] == goal_ref + + def test_equal_monitor_windows_use_unicode_codepoint_identity_order(monkeypatch): from datetime import UTC, datetime diff --git a/tests/control_plane/test_scheduler_host_followup_hint_transport.py b/tests/control_plane/test_scheduler_host_followup_hint_transport.py index e9fdeaf019..4129c82776 100644 --- a/tests/control_plane/test_scheduler_host_followup_hint_transport.py +++ b/tests/control_plane/test_scheduler_host_followup_hint_transport.py @@ -142,6 +142,46 @@ def test_failure_hint_carries_the_same_versioned_native_boundary() -> None: assert payload["use_current_hint"] is False +def test_native_hints_carry_the_exact_goal_instance_scope() -> None: + goal_ref = { + "goal_id": "goal-native-followup", + "goal_instance_id": "ginst_" + "a" * 32, + } + ack = build_app_automation_scheduler_ack_hint( + goal_id="goal-native-followup", + agent_id="agent-native-followup", + applied_rrule="FREQ=MINUTELY;INTERVAL=15", + reset_token="reset-native-followup", + identity_signature="identity-native-followup", + scheduler_host_facts=_host_facts("ack"), + scheduler_before=_before(), + goal_ref=goal_ref, + ) + assert _decode(ack["cli_args"])["goal_ref"] == goal_ref + + failure = build_app_automation_scheduler_failure_hint( + goal_id="goal-native-followup", + agent_id="agent-native-followup", + failed_rrule="FREQ=MINUTELY;INTERVAL=15", + observed_host_rrule="FREQ=MINUTELY;INTERVAL=3", + scheduler_host_facts=_host_facts("host_failure"), + scheduler_before=_before(), + goal_ref=goal_ref, + ) + assert _decode(failure["cli_args"])["goal_ref"] == goal_ref + + legacy = build_app_automation_scheduler_ack_hint( + goal_id="goal-native-followup", + agent_id="agent-native-followup", + applied_rrule="FREQ=MINUTELY;INTERVAL=15", + reset_token="reset-native-followup", + identity_signature="identity-native-followup", + scheduler_host_facts=_host_facts("ack"), + scheduler_before=_before(), + ) + assert _decode(legacy["cli_args"])["goal_ref"] is None + + def test_native_facts_are_not_dropped_when_cli_args_exceed_legacy_budget() -> None: capabilities = [ f"capability-{index}-" + (chr(97 + index) * 140) for index in range(12) diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index 9a50f9f0dc..eb107c41f0 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -27,8 +27,8 @@ import { projectSemanticReplanGuard, QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, readQuotaSettlement, - readQuotaSettlementSnapshot, readQuotaSettlementFromSnapshot, + readQuotaSettlementSnapshot, } from "../../loopx/control_plane/quota/settlement_readback.ts"; import { PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, @@ -740,6 +740,31 @@ test("scoped supersede settles only its exact Turn and never terminal acceptance }); }); +test("only the tolerant readback route survives a damaged audit row", async () => { + const root = await fixture({writeback: true, spend: true}); + try { + const clean = await readQuotaSettlement(request(root)); + const path = join(root, "goals", goalId, "rollout-event-log.jsonl"); + await appendFile(path, Buffer.concat([ + Buffer.from('{"schema_version":"loopx_rollout_event_v0","event_kind":"quota_should_run","goal_id":"'), + Buffer.from(goalId), + Buffer.from('","agent_id":"'), + Buffer.from(agentId), + Buffer.from('","run_id":"later-turn","summary":"'), + Buffer.from([0xff]), + Buffer.from('"}\n'), + ])); + await assert.rejects(readQuotaSettlement(request(root)), { + code: "malformed_settlement_state", + }); + const tolerant = await readQuotaSettlement(request(root, { + tolerate_malformed_history: true, + })); + assert.deepEqual(tolerant, clean, + "a damaged row must not change the settlement facts a valid return can read"); + } finally { await rm(root, {recursive: true, force: true}); } +}); + test("settlement progress requires both effects and exact receipts", async t => { const cases = [ { guard: false, state: "identity_required", next: "validation" }, diff --git a/tests/control_plane_ts/receipt_log_snapshot.test.ts b/tests/control_plane_ts/receipt_log_snapshot.test.ts index a5119950d2..fdeecf32ba 100644 --- a/tests/control_plane_ts/receipt_log_snapshot.test.ts +++ b/tests/control_plane_ts/receipt_log_snapshot.test.ts @@ -95,6 +95,99 @@ test("schema, malformed line numbers and tolerant valid rows survive reuse", asy }); }); +test("invalid UTF-8 rows are malformed, never receipts", async () => { + await fixture(async (path) => { + const valid = Buffer.from('{"run_id":"turn-1"}\n'); + const bad = Buffer.concat([ + Buffer.from('{"run_id":"turn-2","summary":"'), + Buffer.from([0xff]), + Buffer.from('"}\n'), + ]); + await writeFile(path, Buffer.concat([valid, bad])); + const snapshot = (await read(path))!; + assert.deepEqual(snapshot.records.map((r) => r.run_id), ["turn-1"]); + assert.equal(snapshot.firstErrorLine, 2); + // Cached prefix reuse must not resurrect or forget the bad row. + const reused = (await read(path))!; + assert.equal(reused.firstErrorLine, 2); + assert.deepEqual(reused.records.map((r) => r.run_id), ["turn-1"]); + // An unterminated tail with invalid bytes is malformed, never a record. + await writeFile(path, Buffer.concat([ + valid, + Buffer.from('{"run_id":"turn-3","summary":"'), + Buffer.from([0xff]), + ])); + const tail = (await read(path))!; + assert.deepEqual(tail.records.map((r) => r.run_id), ["turn-1"]); + assert.equal(tail.firstErrorLine, 2); + // Terminating the damaged tail keeps it malformed but never hides a later row. + await appendFile(path, Buffer.from('\n{"run_id":"turn-4"}\n')); + const completed = (await read(path))!; + assert.deepEqual( + completed.records.map((r) => r.run_id), + ["turn-1", "turn-4"], + ); + assert.equal(completed.firstErrorLine, 2); + }); +}); + +test("a damaged row still yields the named members its writer finished", async () => { + await fixture(async (path) => { + await writeFile(path, Buffer.concat([ + Buffer.from('{"agent_id":"a","details":{"todo_id":"'), + Buffer.from([0xff]), + Buffer.from('"},"event_id":"e","event_kind":"quota_should_run",' + + '"goal_id":"g","run_id":"turn-2","schema_version":"v0"}\n'), + ])); + const snapshot = (await read(path, "v0", ["event_kind", "run_id", "goal_id"]))!; + assert.deepEqual(snapshot.records, []); + assert.equal(snapshot.firstErrorLine, 1); + assert.deepEqual( + Object.fromEntries(snapshot.malformedLines[0].members), + { event_kind: "quota_should_run", goal_id: "g", run_id: "turn-2" }, + ); + assert.deepEqual(snapshot.malformedLines[0].unreadable, []); + // A damaged value is unreadable, never absent and never guessed. + await writeFile(path, Buffer.concat([ + Buffer.from('{"run_id":"turn-'), + Buffer.from([0xff]), + Buffer.from('","schema_version":"v0"}\n'), + ])); + const damaged = (await read(path, "v0", ["run_id"]))!; + assert.deepEqual(damaged.malformedLines[0].members, []); + assert.deepEqual(damaged.malformedLines[0].unreadable, ["run_id"]); + // Members nobody asked for are not recovered at all. + assert.deepEqual((await read(path, "v0"))!.malformedLines, []); + }); +}); + +test("uncommitted bytes the writer drops leave no trace in history", async () => { + await fixture(async (path) => { + // A torn write can hold a complete JSON object and still be missing its + // committing newline. The append owner drops those bytes before writing + // its own row, so no reader ever sees the half-write. + await writeFile(path, '{"run_id":"turn-lost","schema_version":"v0"}'); + await writeFile(path, '{"run_id":"kept","schema_version":"v0"}\n'); + const snapshot = (await read(path, "v0", ["run_id"]))!; + assert.equal(snapshot.firstErrorLine, null); + assert.deepEqual(snapshot.records.map((record) => record.run_id), ["kept"]); + await appendFile(path, '{"run_id":"turn-2","schema_version":"v0"}\n'); + const grown = (await read(path, "v0", ["run_id"]))!; + assert.deepEqual(grown.records.map((record) => record.run_id), ["kept", "turn-2"]); + }); +}); + +test("a readable row stays a record beside a blank history line", async () => { + await fixture(async (path) => { + await writeFile(path, '{"run_id":"kept","schema_version":"v0"}\n\n'); + const snapshot = (await read(path, "v0", ["run_id"]))!; + assert.deepEqual(snapshot.records.map((record) => record.run_id), ["kept"]); + await appendFile(path, '{"run_id":"later","schema_version":"v0"}\n'); + const grown = (await read(path, "v0", ["run_id"]))!; + assert.deepEqual(grown.records.map((record) => record.run_id), ["kept", "later"]); + }); +}); + test("eviction and concurrent cold readers affect cost, never evidence", async () => { await fixture(async (path) => { const row = JSON.stringify({payload: "x".repeat(2048)}) + "\n"; diff --git a/tests/control_plane_ts/scheduler_heartbeat_commit.test.ts b/tests/control_plane_ts/scheduler_heartbeat_commit.test.ts index f36dbe21b9..910e66928d 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_commit.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_commit.test.ts @@ -16,6 +16,7 @@ import { normalizeSchedulerState, schedulerStatePath, } from "../../loopx/control_plane/scheduler/state_store.ts"; +import { withFileMutationLock } from "../../loopx/control_plane/effect_runtime_io.ts"; import type { JsonObject } from "../../loopx/control_plane/effect_program.ts"; const scope = { @@ -739,3 +740,236 @@ test("a tighter progression requires a fresh identity and preserves CAS and repl assert.equal(old.reason_code, "state_digest_conflict"); assert.equal(old.state_digest, reset.state_digest); }); + +async function writeGoalRegistry( + t: test.TestContext, + runtimeRoot: string, + goalInstanceId: string, + options: { readonly active?: boolean; readonly duplicates?: boolean } = {}, +): Promise { + const goal = { + id: scope.goal_id, + goal_instance_id: goalInstanceId, + status: options.active === false ? "archived" : "active", + }; + const goals = options.duplicates === true ? [goal, { ...goal }] : [goal]; + const path = join(runtimeRoot, "projects.json"); + await writeFile(path, JSON.stringify([ + { + schema_version: "loopx_project_registry_envelope_v1", + minimum_writer_protocol: "goal_instance_v1", + payload_sha256: "sha256:fixture", + }, + { goals, projects: [] }, + ]), "utf8"); + t.after(() => rm(path, { force: true })); + return path; +} + +test("a Goal recreation that publishes during the commit cannot be overwritten", async (t) => { + const runtimeRoot = await tempRuntime(t); + const instanceA = `ginst_${"a".repeat(32)}`; + const instanceB = `ginst_${"b".repeat(32)}`; + const goalRefA = { goal_id: scope.goal_id, goal_instance_id: instanceA }; + const registryPath = await writeGoalRegistry(t, runtimeRoot, instanceA); + const options = { registryPath, heartbeatReceiptGoalRef: goalRefA }; + const first = await evaluateSchedulerHeartbeatCommit(request(runtimeRoot), options); + assert.equal(first.status, "written"); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const committedBytes = await readFile(statePath, "utf8"); + + // Hold the marker a recreation takes while it publishes the new lifetime, so + // the commit reads instance A and only then sees instance B. + let late: Promise>> | null = null; + await withFileMutationLock(registryPath, async () => { + late = evaluateSchedulerHeartbeatCommit(request(runtimeRoot, { + effect_id: "heartbeat-effect-late-a", + applied_rrule: "FREQ=MINUTELY;INTERVAL=15", + expected_state_digest: first.state_digest, + }), options); + await new Promise((resolve) => { setTimeout(resolve, 100); }); + await writeGoalRegistry(t, runtimeRoot, instanceB); + }); + const lateResult = await late!; + assert.equal(lateResult.status, "conflict"); + assert.equal(lateResult.reason_code, "heartbeat_receipt_goal_instance_superseded"); + assert.equal(await readFile(statePath, "utf8"), committedBytes); +}); + +test("a superseded Goal instance cannot overwrite or replay a newer lifetime", async (t) => { + const runtimeRoot = await tempRuntime(t); + const instanceA = `ginst_${"a".repeat(32)}`; + const instanceB = `ginst_${"b".repeat(32)}`; + const goalRefA = { goal_id: scope.goal_id, goal_instance_id: instanceA }; + const goalRefB = { goal_id: scope.goal_id, goal_instance_id: instanceB }; + const registryPath = await writeGoalRegistry(t, runtimeRoot, instanceA); + const options = { registryPath, heartbeatReceiptGoalRef: goalRefA }; + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + + const firstA = await evaluateSchedulerHeartbeatCommit( + request(runtimeRoot), options, + ); + assert.equal(firstA.status, "written"); + assert.equal( + (firstA.state?.heartbeat_commit as Record | undefined) + ?.goal_instance_id, + instanceA, + ); + + // The project registry starts a new Goal lifetime with the same name. + await writeGoalRegistry(t, runtimeRoot, instanceB); + const newInstanceCommit = await evaluateSchedulerHeartbeatCommit( + request(runtimeRoot, { + effect_id: "heartbeat-effect-new-instance", + progression_index: 1, + expected_rrule: "FREQ=MINUTELY;INTERVAL=30", + applied_rrule: "FREQ=MINUTELY;INTERVAL=30", + expected_state_digest: firstA.state_digest, + }), + { registryPath, heartbeatReceiptGoalRef: goalRefB }, + ); + assert.equal(newInstanceCommit.status, "written"); + assert.equal(newInstanceCommit.state?.last_applied_rrule, "FREQ=MINUTELY;INTERVAL=30"); + assert.equal( + (newInstanceCommit.state?.heartbeat_commit as Record | undefined) + ?.goal_instance_id, + instanceB, + ); + const committedBytes = await readFile(statePath, "utf8"); + + const staleAck = await evaluateSchedulerHeartbeatCommit( + request(runtimeRoot, { + effect_id: "heartbeat-effect-late-a", + applied_rrule: "FREQ=MINUTELY;INTERVAL=15", + }), + options, + ); + assert.equal(staleAck.status, "conflict"); + assert.equal(staleAck.reason_code, "heartbeat_receipt_goal_instance_superseded"); + const staleFailure = await evaluateSchedulerHeartbeatCommit( + request(runtimeRoot, { + operation: "host_failure", + effect_id: "heartbeat-effect-late-a-failure", + observed_host_rrule: "FREQ=MINUTELY;INTERVAL=3", + apply_needed: true, + failure_kind: "timeout", + }), + options, + ); + assert.equal(staleFailure.status, "conflict"); + assert.equal(staleFailure.reason_code, "heartbeat_receipt_goal_instance_superseded"); + // The old response-loss retry must not read the new lifetime's receipt. + // A superseded lifetime is refused before any replay read happens. + const staleReplay = await evaluateSchedulerHeartbeatCommit( + request(runtimeRoot, { effect_id: "heartbeat-effect-new-instance" }), + options, + ); + assert.equal(staleReplay.status, "conflict"); + assert.equal(staleReplay.reason_code, "heartbeat_receipt_goal_instance_superseded"); + assert.notEqual(staleReplay.replayed, true); + // A caller that cannot prove which lifetime it belongs to must not read the + // replay of another lifetime's commit either. + const unverifiableReplay = await evaluateSchedulerHeartbeatCommit( + request(runtimeRoot, { effect_id: "heartbeat-effect-new-instance" }), + ); + assert.equal(unverifiableReplay.status, "conflict"); + assert.equal(unverifiableReplay.reason_code, "goal_instance_replay_conflict"); + assert.notEqual(unverifiableReplay.replayed, true); + assert.equal(await readFile(statePath, "utf8"), committedBytes); + + // Returning to the exact receipt's lifetime re-enables its own commands. + await writeGoalRegistry(t, runtimeRoot, instanceA); + const currentA = await evaluateSchedulerHeartbeatCommit( + request(runtimeRoot, { + effect_id: "heartbeat-effect-late-a", + progression_index: 2, + expected_rrule: "FREQ=MINUTELY;INTERVAL=60", + applied_rrule: "FREQ=MINUTELY;INTERVAL=60", + expected_state_digest: newInstanceCommit.state_digest, + }), options, + ); + assert.equal(currentA.status, "written"); + assert.equal( + (currentA.state?.heartbeat_commit as Record | undefined) + ?.goal_instance_id, + instanceA, + ); +}); + +test("a cancelled Goal lifetime and duplicate registry entries refuse writeback", async (t) => { + const runtimeRoot = await tempRuntime(t); + const instanceA = `ginst_${"c".repeat(32)}`; + const goalRef = { goal_id: scope.goal_id, goal_instance_id: instanceA }; + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const registryPath = await writeGoalRegistry(t, runtimeRoot, instanceA, { active: false }); + const inactive = await evaluateSchedulerHeartbeatCommit(request(runtimeRoot), { + registryPath, heartbeatReceiptGoalRef: goalRef, + }); + assert.equal(inactive.status, "conflict"); + assert.equal(inactive.reason_code, "goal_instance_inactive"); + await assert.rejects(readFile(statePath, "utf8"), { code: "ENOENT" }); + + const duplicatePath = await writeGoalRegistry(t, runtimeRoot, instanceA, { + duplicates: true, + }); + const ambiguous = await evaluateSchedulerHeartbeatCommit(request(runtimeRoot), { + registryPath: duplicatePath, heartbeatReceiptGoalRef: goalRef, + }); + assert.equal(ambiguous.status, "conflict"); + assert.equal(ambiguous.reason_code, "goal_instance_ambiguous"); + await assert.rejects(readFile(statePath, "utf8"), { code: "ENOENT" }); + + // A legacy registry without instance identity keeps the goal-scoped path. + await writeFile(duplicatePath, JSON.stringify([ + { schema_version: "loopx_project_registry_envelope_v1" }, + { goals: [{ id: scope.goal_id, status: "active" }] }, + ]), "utf8"); + const legacy = await evaluateSchedulerHeartbeatCommit(request(runtimeRoot), { + registryPath: duplicatePath, heartbeatReceiptGoalRef: goalRef, + }); + assert.equal(legacy.status, "written"); +}); + +test("a lifetime-scoped Goal refuses a receipt that names no Goal reference", async (t) => { + const runtimeRoot = await tempRuntime(t); + const instanceA = `ginst_${"d".repeat(32)}`; + const registryPath = await writeGoalRegistry(t, runtimeRoot, instanceA); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + + const unnamed = await evaluateSchedulerHeartbeatCommit(request(runtimeRoot), { + registryPath, + }); + assert.equal(unnamed.status, "conflict"); + assert.equal(unnamed.reason_code, "heartbeat_receipt_goal_ref_missing"); + await assert.rejects(readFile(statePath, "utf8"), { code: "ENOENT" }); + + // An old-style Goal keeps its goal-scoped path. + await writeFile(registryPath, JSON.stringify([ + { schema_version: "loopx_project_registry_envelope_v1" }, + { goals: [{ id: scope.goal_id, status: "active" }] }, + ]), "utf8"); + const legacy = await evaluateSchedulerHeartbeatCommit(request(runtimeRoot), { + registryPath, + }); + assert.equal(legacy.status, "written"); +}); diff --git a/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts b/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts index a636ce3df3..bb4c91b125 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { + appendFile, mkdir, mkdtemp, readFile, @@ -68,7 +69,11 @@ function receiptPath(runtimeRoot: string): string { return join(runtimeRoot, "goals", scope.goal_id, "rollout-event-log.jsonl"); } -async function appendReceipt(runtimeRoot: string, turnId: string): Promise { +async function appendReceipt( + runtimeRoot: string, + turnId: string, + goalRef?: Record, +): Promise { const path = receiptPath(runtimeRoot); await mkdir(dirname(path), { recursive: true }); await writeFile(path, `${JSON.stringify({ @@ -77,6 +82,7 @@ async function appendReceipt(runtimeRoot: string, turnId: string): Promise goal_id: scope.goal_id, agent_id: scope.agent_id, run_id: turnId, + ...(goalRef === undefined ? {} : { goal_ref: goalRef }), })}\n`, { encoding: "utf8", flag: "a" }); } @@ -226,6 +232,135 @@ test("missing or superseded heartbeat receipt rejects before scheduler mutation" assert.equal(JSON.parse(await readFile(legacyPath, "utf8")).goal_id, scope.goal_id); }); +async function appendMalformedReceipt( + runtimeRoot: string, + turnId: string | null, +): Promise { + const path = receiptPath(runtimeRoot); + await mkdir(dirname(path), { recursive: true }); + // Structurally a receipt-shaped row whose summary carries one raw 0xff byte. + // A row with a Turn identity is a committed admission whose display field + // was later damaged; an identity-less row is only a diagnostic. + const turn = turnId === null ? "" : `"run_id":"${turnId}",`; + const head = Buffer.from( + `{"schema_version":"loopx_rollout_event_v0","event_kind":"quota_should_run",` + + `"goal_id":"${scope.goal_id}","agent_id":"${scope.agent_id}",` + + `${turn}"summary":"`, + ); + await appendFile(path, Buffer.concat([head, Buffer.from([0xff]), Buffer.from('"}\n')])); +} + +test("a damaged diagnostic row cannot block a still-valid ACK or failure", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const base = request(runtimeRoot); + + await appendMalformedReceipt(runtimeRoot, null); + const snapshot = await readGoalRolloutEventSnapshot(runtimeRoot, scope.goal_id); + assert.equal(snapshot!.firstStrictErrorLine, 2); + assert.deepEqual( + goalHeartbeatReceiptsFromSnapshot(snapshot, scope.goal_id)!.map((r) => r.run_id), + ["turn-followup-1"], + ); + // Strict settlement readback still reports the damaged row. + assert.throws(() => strictGoalRolloutEvents(snapshot), { + code: "malformed_settlement_state", + }); + + const ack = await evaluateSchedulerHeartbeatFollowup(base); + assert.equal(ack.ok, true); + assert.equal((ack.scheduler_commit as Record).status, "written"); + + const failure = await evaluateSchedulerHeartbeatFollowup({ + ...base, + host_facts: { + ...(base.host_facts as Record), + operation: "host_failure", + applied_rrule: "FREQ=MINUTELY;INTERVAL=3", + observed_host_rrule: "FREQ=MINUTELY;INTERVAL=3", + failure_kind: "timeout", + source: "quota_scheduler_host_update_failure", + }, + }); + assert.equal(failure.ok, true); + assert.equal(failure.mode, "scheduler-fail-current"); +}); + +test("a committed successor keeps fencing after its summary is damaged", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + // A genuine successor admission whose display summary was corrupted after it + // committed: its finished Turn identity still owns the newer heartbeat. + await appendMalformedReceipt(runtimeRoot, "turn-damaged-successor"); + const before = await readFile(statePath, "utf8").catch(() => null); + + const ack = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); + assert.equal(ack.ok, false); + assert.equal(ack.status, "heartbeat_receipt_stale"); + const failure = await evaluateSchedulerHeartbeatFollowup({ + ...request(runtimeRoot), + host_facts: { + ...(request(runtimeRoot).host_facts as Record), + operation: "host_failure", + applied_rrule: "FREQ=MINUTELY;INTERVAL=3", + observed_host_rrule: "FREQ=MINUTELY;INTERVAL=3", + failure_kind: "timeout", + source: "quota_scheduler_host_update_failure", + }, + }); + assert.equal(failure.ok, false); + assert.equal(failure.status, "heartbeat_receipt_stale"); + assert.equal(await readFile(statePath, "utf8").catch(() => null), before); + + // A torn tail without a closing newline is not a committed admission. + const path = receiptPath(runtimeRoot); + await appendFile(path, Buffer.from( + `{"event_kind":"quota_should_run","goal_id":"${scope.goal_id}",` + + `"agent_id":"${scope.agent_id}","run_id":"turn-torn-tail","summary":"`, + )); + const torn = await evaluateSchedulerHeartbeatFollowup({ + ...request(runtimeRoot), + host_facts: { + ...(request(runtimeRoot).host_facts as Record), + generated_at: "2026-08-27T06:40:00Z", + }, + }); + assert.equal(torn.ok, false); + assert.equal(torn.status, "heartbeat_receipt_stale"); +}); + +test("a genuinely newer receipt still fences both writebacks", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const base = request(runtimeRoot); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + // A genuinely newer receipt still fences both writebacks. + await appendReceipt(runtimeRoot, "turn-followup-2"); + const staleAck = await evaluateSchedulerHeartbeatFollowup({ + ...base, + host_facts: { + ...(base.host_facts as Record), + generated_at: "2026-08-27T06:35:00Z", + }, + }); + assert.equal(staleAck.ok, false); + assert.equal(staleAck.status, "heartbeat_receipt_stale"); + assert.equal(staleAck.scheduler_state_mutated, false); + // The stale writeback never created scheduler state. + await assert.rejects(readFile(statePath, "utf8"), { code: "ENOENT" }); +}); + test("receipt-bound ACK rechecks freshness after waiting for the scheduler lock", async (t) => { const runtimeRoot = await tempRuntime(t); await appendReceipt(runtimeRoot, "turn-followup-1"); @@ -253,6 +388,87 @@ test("receipt-bound ACK rechecks freshness after waiting for the scheduler lock" await assert.rejects(readFile(path, "utf8"), { code: "ENOENT" }); }); +test("another Goal instance cannot authorize or supersede an exact-instance receipt", async (t) => { + const runtimeRoot = await tempRuntime(t); + const goalRefA = { + goal_id: scope.goal_id, + goal_instance_id: `ginst_${"a".repeat(32)}`, + }; + const goalRefB = { + goal_id: scope.goal_id, + goal_instance_id: `ginst_${"b".repeat(32)}`, + }; + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + await appendReceipt(runtimeRoot, "turn-followup-1", goalRefB); + + const foreign = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: goalRefA, + })); + assert.equal(foreign.ok, false); + assert.equal(foreign.status, "heartbeat_receipt_missing"); + assert.equal(foreign.scheduler_state_mutated, false); + await assert.rejects(readFile(statePath, "utf8"), { code: "ENOENT" }); + + await appendReceipt(runtimeRoot, "turn-followup-1", goalRefA); + const fresh = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: goalRefA, + })); + assert.equal(fresh.ok, true); + assert.equal((fresh.scheduler_commit as Record).status, "written"); + + // A newer receipt for another Goal lifetime is outside this transaction's + // freshness scope and cannot turn the exact instance's receipt stale. + await appendReceipt(runtimeRoot, "turn-followup-2", goalRefB); + const stillFresh = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: goalRefA, + host_facts: { + ...((request(runtimeRoot).host_facts) as Record), + generated_at: "2026-08-27T06:35:00Z", + }, + })); + assert.equal(stillFresh.ok, true); +}); + +test("the scheduler lock rechecks freshness inside the exact Goal instance", async (t) => { + const runtimeRoot = await tempRuntime(t); + const goalRefA = { + goal_id: scope.goal_id, + goal_instance_id: `ginst_${"c".repeat(32)}`, + }; + const goalRefB = { + goal_id: scope.goal_id, + goal_instance_id: `ginst_${"d".repeat(32)}`, + }; + const exactRequest = request(runtimeRoot, { goal_ref: goalRefA }); + await appendReceipt(runtimeRoot, "turn-followup-1", goalRefA); + const path = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const held = await acquireFileMutationLock(path); + let released = false; + t.after(async () => { + if (!released) await releaseFileMutationLock(held.targetPath, held.token); + }); + + const inFlight = evaluateSchedulerHeartbeatFollowup(exactRequest); + await delay(250); + await appendReceipt(runtimeRoot, "turn-followup-2", goalRefB); + released = await releaseFileMutationLock(held.targetPath, held.token); + + const written = await inFlight; + assert.equal(written.ok, true); + assert.equal((written.scheduler_commit as Record).status, "written"); + assert.equal(JSON.parse(await readFile(path, "utf8")).goal_id, scope.goal_id); +}); + test("decision-free compatibility requests can omit a heartbeat receipt", async (t) => { const runtimeRoot = await tempRuntime(t); const result = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { @@ -325,9 +541,12 @@ test("another Agent receipt cannot supersede ACK or host-failure writeback", asy assert.equal(failed.failure_count, 1); }); -test("a successful identity-less quota guard supersedes an older host receipt", async (t) => { +test("a successful identity-less quota guard cannot supersede a host receipt", async (t) => { const runtimeRoot = await tempRuntime(t); await appendReceipt(runtimeRoot, "turn-followup-1"); + // A historical-shaped audit row carries no Turn identity. It cannot retire + // the originating receipt, because the same bytes are indistinguishable + // from an audit row copied out of older history. await writeFile(receiptPath(runtimeRoot), `${JSON.stringify({ schema_version: "loopx_rollout_event_v0", event_kind: "quota_should_run", @@ -337,6 +556,14 @@ test("a successful identity-less quota guard supersedes an older host receipt", details: { ok: true, should_run: true }, })}\n`, { flag: "a" }); + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); + assert.equal(acknowledged.ok, true); + const commit = acknowledged.scheduler_commit as Record; + assert.equal(commit.status, "written"); + assert.equal(commit.written, true); + + // A real newer Turn in the same Goal instance remains authoritative. + await appendReceipt(runtimeRoot, "turn-followup-2"); const stale = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); assert.equal(stale.error_code, "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE"); assert.equal(stale.scheduler_state_mutated, false); diff --git a/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts b/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts index baa30efd6c..77aebb866f 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts @@ -36,7 +36,10 @@ async function tempRuntime(t: CleanupContext): Promise { return runtimeRoot; } -async function writeReceipt(runtimeRoot: string): Promise { +async function writeReceipt( + runtimeRoot: string, + goalRef?: Record, +): Promise { const path = join(runtimeRoot, "goals", goalId, "rollout-event-log.jsonl"); await mkdir(dirname(path), { recursive: true }); await writeFile(path, `${JSON.stringify({ @@ -45,12 +48,25 @@ async function writeReceipt(runtimeRoot: string): Promise { goal_id: goalId, agent_id: agentId, run_id: turnId, + ...(goalRef === undefined ? {} : { goal_ref: goalRef }), })}\n`, "utf8"); } -function hintPayload(): Record { +const goalRefA = { + goal_id: goalId, + goal_instance_id: `ginst_${"a".repeat(32)}`, +}; +const goalRefB = { + goal_id: goalId, + goal_instance_id: `ginst_${"b".repeat(32)}`, +}; + +function hintPayload( + goalRef?: Record, +): Record { return { schema_version: "loopx_scheduler_host_followup_hint_v0", + ...(goalRef === undefined ? {} : { goal_ref: goalRef }), before: { should_run: true, normal_delivery_allowed: true, @@ -115,8 +131,12 @@ function argv(runtimeRoot: string, format = "json", hint = hintPayload()): strin return result; } -function failureArgv(runtimeRoot: string, format = "json"): string[] { - const payload = hintPayload(); +function failureArgv( + runtimeRoot: string, + format = "json", + goalRef?: Record, +): string[] { + const payload = hintPayload(goalRef); const facts = payload.host_facts as Record; payload.use_current_hint = false; payload.host_facts = { @@ -314,6 +334,44 @@ test("native host-failure CLI preserves the legacy Markdown surface", async (t) assert.match(child.stdout, /- failure_count: `1`/); }); +test("native scheduler CLI carries the hint Goal instance into ACK and failure writeback", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceipt(runtimeRoot, goalRefA); + + const acknowledged = runCli(argv(runtimeRoot, "json", hintPayload(goalRefA))); + assert.equal(acknowledged.status, 0, acknowledged.stderr); + const ack = JSON.parse(acknowledged.stdout) as Record; + assert.equal((ack.scheduler_commit as Record).status, "written"); + + const failed = runCli(failureArgv(runtimeRoot, "json", goalRefA)); + assert.equal(failed.status, 0, failed.stderr); + const failure = JSON.parse(failed.stdout) as Record; + assert.equal(failure.ok, true); + assert.equal(failure.failure_count, 1); +}); + +test("native scheduler CLI rejects a receipt from another Goal instance without writing", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceipt(runtimeRoot, goalRefA); + + const child = runCli(argv(runtimeRoot, "json", hintPayload(goalRefB))); + + assert.notEqual(child.status, 0); + const result = JSON.parse(child.stdout) as Record; + assert.equal(result.status, "heartbeat_receipt_missing"); + assert.equal( + result.error_code, + "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_MISSING", + ); + assert.equal(result.scheduler_state_mutated, false); + assert.equal(result.write_performed, false); + assert.equal(result.appended, false); + assert.deepEqual( + await readdir(join(runtimeRoot, "goals", goalId)), + ["rollout-event-log.jsonl"], + ); +}); + test("native host-followup CLI returns the legacy receipt failure without writing", async (t) => { const runtimeRoot = await tempRuntime(t); diff --git a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts index 37d9f834f5..f4b62e8e9b 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { once } from "node:events"; import fs, { + appendFile, mkdir, mkdtemp, readFile, @@ -190,6 +191,298 @@ function startPythonReceiptWriter({ return { started, appended, done }; } +const PYTHON_UNRELATED_APPEND = [ + "import sys", + "from pathlib import Path", + "from loopx.rollout_event_log import append_rollout_event, build_rollout_event, rollout_event_log_path", + "runtime_root, goal_id = sys.argv[1:]", + "event = build_rollout_event(goal_id=goal_id, event_kind='todo_complete')", + "append_rollout_event(rollout_event_log_path(Path(runtime_root), goal_id), event)", + "print('appended', flush=True)", +].join("\n"); + +async function runPython( + t: CleanupContext, + script: string, + args: readonly string[], +): Promise { + const child = spawn(PYTHON, ["-c", script, ...args], { + cwd: REPOSITORY_ROOT, + env: { ...process.env, PYTHONPATH: REPOSITORY_ROOT }, + stdio: ["ignore", "pipe", "pipe"], + }); + t.after(() => { + if (child.exitCode === null) child.kill(); + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8").on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.setEncoding("utf8").on("data", (chunk: string) => { + stderr += chunk; + }); + await once(child, "close"); + assert.equal(child.exitCode, 0, `Python writer failed: ${stderr}`); + return stdout; +} + +/** One receipt row whose identity is intact and whose payload may be damaged. */ +function receiptRow(turnId: string, summary = `receipt ${turnId}`): string { + return JSON.stringify({ + schema_version: "loopx_rollout_event_v0", + event_kind: "quota_should_run", + goal_id: scope.goal_id, + agent_id: scope.agent_id, + run_id: turnId, + summary, + }); +} + +function receiptLogPath(runtimeRoot: string): string { + return join(runtimeRoot, "goals", scope.goal_id, "rollout-event-log.jsonl"); +} + +test("a committed successor stays superseding after its early field is damaged", async (t) => { + for (const operation of OPERATIONS) { + await t.test(operation, { timeout: 30_000 }, async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const path = receiptLogPath(runtimeRoot); + // The successor's `details` value meets an undecodable byte, so its + // identity members come later in the serialized row than the damage. + await appendFile(path, Buffer.concat([ + Buffer.from( + `{"agent_id":"${scope.agent_id}","details":{"command":"quota",` + + `"quota_command":"should-run","todo_id":"`, + ), + Buffer.from([0xff]), + Buffer.from( + `"},"event_id":"damaged","event_kind":"quota_should_run",` + + `"goal_id":"${scope.goal_id}","recorded_at":"2026-08-27T06:30:00Z",` + + `"run_id":"turn-followup-2","schema_version":"loopx_rollout_event_v0",` + + `"status":"normal_run","summary":"damaged successor"}\n`, + ), + ])); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const before = await readFile(path); + const result = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + assert.equal( + result.error_code, + "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE", + ); + assert.equal(result.scheduler_state_mutated, false); + assert.deepEqual(await readFile(path), before); + await assert.rejects(readFile(statePath, "utf8")); + }); + } +}); + +test("only an ambiguous receipt of this scope fails closed", async (t) => { + const rows: readonly { + row: string; + damaged: boolean; + stale: boolean; + why: string; + }[] = [ + { + // Two different Turn ids cannot both name what the row admitted. + row: + `{"agent_id":"${scope.agent_id}","event_kind":"quota_should_run",` + + `"goal_id":"${scope.goal_id}","run_id":"turn-followup-2",` + + `"schema_version":"loopx_rollout_event_v0","run_id":"turn-followup-3"}`, + damaged: false, + stale: true, + why: "same-scope receipt with two Turn ids", + }, + { + // A readable Turn id beside an unreadable one is still ambiguous. + row: + `{"agent_id":"${scope.agent_id}","event_kind":"quota_should_run",` + + `"goal_id":"${scope.goal_id}","run_id":"turn-followup-2",` + + `"schema_version":"loopx_rollout_event_v0","run_id":"turn-`, + damaged: true, + stale: true, + why: "same-scope receipt with a damaged second Turn id", + }, + { + // A duplicated Turn in another Agent's row proves nothing here. + row: + `{"agent_id":"other-agent","event_kind":"quota_should_run",` + + `"goal_id":"${scope.goal_id}","run_id":"turn-followup-2",` + + `"schema_version":"loopx_rollout_event_v0","run_id":"turn-followup-3"}`, + damaged: false, + stale: false, + why: "another Agent's duplicated Turn", + }, + { + // A validation diagnostic is not a heartbeat receipt of this scope. + row: + `{"agent_id":"${scope.agent_id}","event_kind":"validation",` + + `"goal_id":"${scope.goal_id}","run_id":"turn-followup-2",` + + `"schema_version":"loopx_rollout_event_v0","run_id":"turn-followup-3"}`, + damaged: false, + stale: false, + why: "a validation row with a duplicated Turn", + }, + ]; + for (const operation of OPERATIONS) { + await t.test(operation, { timeout: 30_000 }, async (t) => { + for (const { row, damaged, stale, why } of rows) { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const path = receiptLogPath(runtimeRoot); + await appendFile(path, Buffer.concat([ + Buffer.from(row), + ...(damaged ? [Buffer.from([0xff])] : []), + Buffer.from('"}\n'), + ])); + const before = await readFile(path); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const result = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + if (stale) { + assert.equal( + result.error_code, + "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE", + why, + ); + assert.equal(result.scheduler_state_mutated, false, why); + assert.deepEqual(await readFile(path), before, why); + await assert.rejects(readFile(statePath, "utf8"), why); + } else { + assert.equal(result.ok, true, why); + assert.equal(result.scheduler_state_mutated, true, why); + assert.deepEqual(await readFile(path), before, why); + const persisted = JSON.parse(await readFile(statePath, "utf8")); + assert.equal(persisted.heartbeat_commit.operation, operation, why); + } + } + }); + } +}); + +test("a damaged audit row keeps its nested diagnostic Turn out of scope", async (t) => { + for (const operation of OPERATIONS) { + await t.test(operation, { timeout: 30_000 }, async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const path = receiptLogPath(runtimeRoot); + // The audit row is a quota receipt without a top-level Turn. A damaged + // display field before `details` must not let recovery read the nested + // diagnostic `run_id` as a committed successor. + await appendFile(path, Buffer.concat([ + Buffer.from( + `{"agent_id":"${scope.agent_id}","note":"`, + ), + Buffer.from([0xff]), + Buffer.from( + `","details":{"run_id":"turn-diagnostic-nested"},` + + `"event_kind":"quota_should_run","goal_id":"${scope.goal_id}",` + + `"status":"normal_run","schema_version":"loopx_rollout_event_v0"}\n`, + ), + ])); + const before = await readFile(path); + const result = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + assert.equal(result.ok, true); + assert.equal(result.scheduler_state_mutated, true); + assert.deepEqual(await readFile(path), before); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const persisted = JSON.parse(await readFile(statePath, "utf8")); + assert.equal(persisted.heartbeat_commit.operation, operation); + }); + } +}); + +test("a torn successor stays uncommitted after its append is repaired", async (t) => { + for (const operation of OPERATIONS) { + await t.test(operation, { timeout: 30_000 }, async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const path = receiptLogPath(runtimeRoot); + // A writer that never finished the row leaves a torn prefix; the next + // append drops it before writing its own row. + await appendFile(path, Buffer.from( + `{"agent_id":"${scope.agent_id}","details":{"turn_instance_id":` + + `"turn-followup-2"},"event_kind":"quota_should_run",` + + `"goal_id":"${scope.goal_id}","run_id":"turn-followup-2",` + + `"schema_version":"loopx_rollout_event_v0"`, + )); + await runPython(t, PYTHON_UNRELATED_APPEND, [runtimeRoot, scope.goal_id]); + const repaired = await readFile(path); + assert.ok(!repaired.includes("turn-followup-2"), inspected(repaired)); + + const result = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + assert.equal(result.ok, true); + assert.equal(result.scheduler_state_mutated, true); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const persisted = JSON.parse(await readFile(statePath, "utf8")); + assert.equal(persisted.heartbeat_commit.operation, operation); + }); + } +}); + +function inspected(bytes: Buffer): string { + return bytes.toString("hex"); +} + +test("a completed row missing only its newline stays uncommitted after repair", async (t) => { + for (const operation of OPERATIONS) { + await t.test(operation, { timeout: 30_000 }, async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const path = receiptLogPath(runtimeRoot); + // The row's own bytes are complete JSON; only its terminating newline is + // missing, so the append owner drops the uncommitted prefix. + await appendFile(path, Buffer.from(receiptRow("turn-followup-2"))); + await runPython(t, PYTHON_UNRELATED_APPEND, [runtimeRoot, scope.goal_id]); + const repaired = await readFile(path); + assert.ok(!repaired.includes("turn-followup-2"), inspected(repaired)); + + const result = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + assert.equal(result.ok, true); + assert.equal(result.scheduler_state_mutated, true); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const persisted = JSON.parse(await readFile(statePath, "utf8")); + assert.equal(persisted.heartbeat_commit.operation, operation); + }); + } +}); + test("receipt append cannot split freshness from ACK or host-failure persistence", async (t) => { for (const operation of OPERATIONS) { await t.test(operation, { timeout: 30_000 }, async (t) => { diff --git a/tests/test_rollout_event_log_loading.py b/tests/test_rollout_event_log_loading.py index b5ffff77c6..43239071de 100644 --- a/tests/test_rollout_event_log_loading.py +++ b/tests/test_rollout_event_log_loading.py @@ -266,10 +266,55 @@ def test_append_starts_a_new_row_after_a_torn_final_record( else: written = append_rollout_event(log_path, event) - assert torn_record + b"\n" in log_path.read_bytes() + assert torn_record not in log_path.read_bytes() assert load_rollout_events(log_path) == [existing, written] +def test_a_torn_record_is_dropped_before_the_next_row( + tmp_path: Path, +) -> None: + """Uncommitted torn bytes vanish; the log stays readable by every reader. + + A torn write can hold a complete JSON object and still be missing its + committing newline. The append owner drops those bytes instead of finishing + them, so no reader treats a row its writer never committed as an admission + and older strict readers still parse the history. + """ + + log_path = tmp_path / "rollout-event-log.jsonl" + committed = build_rollout_event( + goal_id="goal-a", + event_kind="quota_should_run", + agent_id="agent-a", + run_id="turn-before", + recorded_at="2026-09-22T00:00:00Z", + ) + torn = ( + b'{"agent_id":"agent-a","event_kind":"quota_should_run",' + b'"goal_id":"goal-a","run_id":"turn-lost",' + b'"schema_version":"loopx_rollout_event_v0"}' + ) + log_path.write_bytes( + json.dumps(committed, sort_keys=True).encode("utf-8") + b"\n" + torn + ) + appended = build_rollout_event( + goal_id="goal-a", + event_kind="todo_complete", + agent_id="agent-a", + recorded_at="2026-09-23T00:00:00Z", + ) + + append_rollout_event(log_path, appended) + + assert torn not in log_path.read_bytes() + assert log_path.read_bytes() == ( + json.dumps(committed, sort_keys=True).encode("utf-8") + b"\n" + + json.dumps(appended, sort_keys=True).encode("utf-8") + b"\n" + ) + assert load_rollout_events(log_path) == [committed, appended] + assert list(iter_rollout_events(log_path, strict=True)) == [committed, appended] + + @pytest.mark.parametrize("idempotent", [False, True]) def test_append_replays_an_event_before_a_mid_utf8_torn_record( tmp_path: Path, From 6490be785f50ed5b18686f252157ed94c4bca4da Mon Sep 17 00:00:00 2001 From: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> Date: Wed, 18 Nov 2026 07:29:48 +0800 Subject: [PATCH 2/8] fix(scheduler): read an unreadable successor identity as superseding A committed quota_should_run row can lose one byte of a nested display string (the opening or closing quote of a details value) while every identity member and the terminating newline stay intact. The tolerant scanner resynchronised on the next plausible quote, swallowed the members that followed the damaged nested value and reported the row as unrelated, so an older heartbeat return regained write authority and both compatibility commands wrote again. Receipt recovery now keeps the row's container state while it reads members. A damaged byte where a value must start is that value's opening quote; a damaged byte inside a string is the lost closing quote only when the separator its role requires follows it, and a lost key name is reported as such. Recovery also reports whether a separator or boundary was unknowable. A same-scope row whose identity cannot be attributed now answers "unknown", and the freshness check treats "cannot prove the successor is unrelated" as stale. Rows whose readable identity proves another Agent, another Goal lifetime or another event kind stay tolerated, and a diagnostic row with all key names intact still cannot supersede a Turn. Verified with real File and SQLite scheduler-ack-current and scheduler-fail-current commands: opening-quote, closing-quote and in-text damage of a committed turn-2 receipt all exit 1 with heartbeat_receipt_stale, scheduler_state_mutated=false and unchanged log bytes, while another Agent's damaged row still lets the current turn write. TypeScript receipt log, commit, followup and CLI suites pass; the Python receipt-fence suite passes on File and SQLite; control-plane typecheck passes. Windows, hosted UI and hosted CI were not exercised here. Signed-off-by: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> --- loopx/control_plane/rollout_receipt_log.ts | 56 +++- .../runtime/json_line_members.ts | 273 ++++++++++++++---- .../runtime/receipt_log_snapshot.ts | 12 +- .../test_scheduler_heartbeat_receipt_fence.py | 170 +++++++++++ .../scheduler_heartbeat_receipt_fence.test.ts | 109 +++++++ 5 files changed, 555 insertions(+), 65 deletions(-) diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index 150d08413c..84acde88ec 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -40,6 +40,10 @@ export interface GoalRolloutEventSnapshot { members: readonly (readonly [string, unknown])[]; /** Identity members the damaged line contained but could not read. */ unreadable: readonly string[]; + /** A key name was lost to damage, so recovered members may be incomplete. */ + lostKeyName: boolean; + /** A separator or container boundary was unknowable after damage. */ + ambiguous: boolean; }>[]; readonly firstStrictErrorLine: number | null; } @@ -224,12 +228,18 @@ const ADMISSION_IDENTITY_MEMBERS = [ */ type CommittedSuccessor = | { readonly kind: "superseded"; readonly runId: string } - | { readonly kind: "unattributed" }; + | { readonly kind: "unattributed" } + // The row belongs to this scope but its damage left an identity unknowable. + // Absence of proof is not proof of absence: a caller must treat this receipt + // as possibly superseding rather than as unrelated. + | { readonly kind: "unknown" }; function committedSuccessor( damaged: Readonly<{ members: readonly (readonly [string, unknown])[]; unreadable: readonly string[]; + lostKeyName: boolean; + ambiguous: boolean; }>, goalId: string, agentId: string, @@ -257,13 +267,18 @@ function committedSuccessor( if (key !== "run_id") return null; if (fields.has("run_id")) ambiguousTurn = true; } - if (fields.get("schema_version") !== ROLLOUT_EVENT_SCHEMA_VERSION) return null; - if (fields.get("event_kind") !== HEARTBEAT_RECEIPT_EVENT_KIND) return null; - if (fields.get("goal_id") !== goalId) return null; - if (fields.get("agent_id") !== agentId) return null; + // A member that is present with another value proves the row is outside this + // scope; only then may damage elsewhere in the row be tolerated. Missing + // members are handled below, because damage can hide a member entirely. + if (fields.has("schema_version") && + fields.get("schema_version") !== ROLLOUT_EVENT_SCHEMA_VERSION) return null; + if (fields.has("event_kind") && + fields.get("event_kind") !== HEARTBEAT_RECEIPT_EVENT_KIND) return null; + if (fields.has("goal_id") && fields.get("goal_id") !== goalId) return null; + if (fields.has("agent_id") && fields.get("agent_id") !== agentId) return null; if (goalRef === null) { if (fields.has("goal_ref")) return null; - } else { + } else if (fields.has("goal_ref")) { const eventGoalRef = jsonObject(fields.get("goal_ref")); if ( eventGoalRef === null || @@ -271,14 +286,34 @@ function committedSuccessor( eventGoalRef.goal_instance_id !== goalRef.goal_instance_id ) return null; } + // Scope members whose bytes could not be read keep the row inside this + // transaction without proving which Turn admitted it. + for (const key of damaged.unreadable) { + if (!(ADMISSION_IDENTITY_MEMBERS as readonly string[]).includes(key)) continue; + if (key !== "run_id") return { kind: "unknown" }; + } + const identityUncertain = damaged.lostKeyName || damaged.ambiguous; + const required = goalRef === null + ? ["schema_version", "event_kind", "goal_id", "agent_id"] + : ["schema_version", "event_kind", "goal_id", "agent_id", "goal_ref"]; + if (required.some((key) => !fields.has(key))) { + // Without damage such a row simply is not one of this scope's receipts. + // Damage can have hidden the member behind a key name or separator it + // could not read, so the row is unreadable instead of unrelated. + return identityUncertain ? { kind: "unknown" } : null; + } // The row now proves it is one of this scope's heartbeat receipts. Only then // does an unknown Turn mean "somebody else's turn superseded mine". + if (damaged.ambiguous) return { kind: "unknown" }; if (ambiguousTurn) return { kind: "unattributed" }; const runId = fields.get("run_id"); if (typeof runId === "string" && runId.trim().length > 0) { return { kind: "superseded", runId }; } - return damaged.unreadable.includes("run_id") ? { kind: "unattributed" } : null; + if (damaged.unreadable.includes("run_id")) return { kind: "unattributed" }; + // Nothing names a Turn here. A lost key name could have been that Turn, so + // only a row whose key names all survived may be read as a diagnostic. + return damaged.lostKeyName ? { kind: "unknown" } : null; } export async function heartbeatReceiptStatus(params: { @@ -317,8 +352,11 @@ export async function heartbeatReceiptStatus(params: { damaged, params.goalId, params.agentId, goalRef, ); if (admitted === null) return false; - return admitted.kind === "unattributed" || - admitted.runId !== params.turnInstanceId; + if (admitted.kind === "superseded") { + return admitted.runId !== params.turnInstanceId; + } + // An unreadable admission of this scope still retires the old Turn. + return true; }); return laterReceipt || laterAdmission ? "stale" : "fresh"; } diff --git a/loopx/control_plane/runtime/json_line_members.ts b/loopx/control_plane/runtime/json_line_members.ts index 2c6d0eb40e..0159ebb0ba 100644 --- a/loopx/control_plane/runtime/json_line_members.ts +++ b/loopx/control_plane/runtime/json_line_members.ts @@ -184,92 +184,233 @@ function parseObjectAt(row: DecodedRow, index: number, depth: number): ParsedVal } /** - * The quote that closes a string whose bytes were damaged. + * How much a row's structure still shows after its bytes were damaged. * - * A damaged byte leaves the string's own end unknowable in theory, but the row - * still shows which structural token follows it. Only a quote followed by the - * separator a finished key or value has can be that end, so scanning stops on - * the first one. Without such a quote the rest of the row is unreadable. + * Recovery answers one narrow question: which named top-level members can be + * read without guessing. When a damaged byte leaves a separator unknowable, + * the members after it may be missing for reasons the row itself cannot state; + * callers must then treat any missing identity as possibly hidden rather than + * absent. */ -function closingQuoteAfterDamage(row: DecodedRow, from: number): number { - for (let at = from; at < row.text.length; at += 1) { - if (row.damaged.has(at)) continue; - if (row.text[at] === "\\") { - at += 1; - continue; - } - if (row.text[at] !== '"') continue; - const after = skipWhitespace(row, at + 1); - const token = row.text[after]; - if (token === "," || token === "}" || token === "]" || token === ":") return at; - } - return -1; +export interface MemberScan { + readonly offsets: readonly number[]; + /** A key's name met an undecodable byte, so the recovered set is incomplete. */ + readonly lostKeyName: boolean; + /** A separator or container boundary was unknowable, so the scan is inexact. */ + readonly ambiguous: boolean; } /** * Byte offsets of the named keys that sit directly in the row's root object. * - * Depth is tracked through nested objects and arrays, so a key inside a + * A small JSON state machine tracks the container stack, so a key inside a * diagnostic `details` value can never be read as a top-level schema member. - * Damage inside a string is resynchronized on the quote its own row shows, which - * keeps the depth available for the members that follow it. + * Damage is handled by context rather than by scanning for a plausible quote: + * + * - A damaged byte where a value must start can only stand for that value's + * opening quote, so the scan resumes inside the string and stops at its own + * closing quote. + * - A damaged byte where a key must start likewise stands for that key's + * opening quote; the key's name is lost, which the caller is told about. + * - A damaged byte inside a string is the lost closing quote only when the + * separator its role requires follows it. Otherwise it is damage inside the + * text, and the string ends at the next unescaped quote as it normally would. + * + * Anything the state machine cannot place leaves `ambiguous` set. The scan + * never guesses a member from text that may be nested, and the caller must not + * read a missing identity as an absent one when either flag is set. */ -function topLevelMemberOffsets(row: DecodedRow, wanted: ReadonlySet): number[] { +function topLevelMemberOffsets(row: DecodedRow, wanted: ReadonlySet): MemberScan { const offsets: number[] = []; let index = skipWhitespace(row, 0); - if (row.text[index] !== "{" || row.damaged.has(index)) return offsets; - index += 1; - let depth = 1; - let inString = false; + if (row.text[index] !== "{" || row.damaged.has(index)) { + return { + offsets, + lostKeyName: false, + ambiguous: row.text[index] !== "{", + }; + } + const containers: ("object" | "array")[] = ["object"]; + // True when the row's next token must be a value (after `:` or in an array). + let expectValue = false; + // True when the row's next token must be a key of the current object. + let expectKey = true; + let inString: "key" | "value" | null = null; let escaped = false; - while (index < row.text.length && depth >= 1 && offsets.length <= MAX_MEMBERS) { + let lostKeyName = false; + let ambiguous = false; + index += 1; + while (index < row.text.length && containers.length > 0) { + if (offsets.length > MAX_MEMBERS) { + ambiguous = true; + break; + } const character = row.text[index]; if (row.damaged.has(index)) { - if (!inString) { + if (inString === null) { + if (expectValue) { + // A value was required here: the damaged byte is its opening quote. + inString = "value"; + escaped = false; + expectValue = false; + expectKey = false; + index += 1; + continue; + } + if (expectKey && containers[containers.length - 1] === "object") { + // A key was required here: its name is lost, its structure is not. + lostKeyName = true; + inString = "key"; + escaped = false; + expectKey = false; + index += 1; + continue; + } + ambiguous = true; index += 1; continue; } - const close = closingQuoteAfterDamage(row, index + 1); - if (close < 0) break; - inString = false; - escaped = false; - index = close + 1; + if (escaped) { + escaped = false; + index += 1; + continue; + } + const after = skipWhitespace(row, index + 1); + const follower = row.text[after]; + const closesKey = inString === "key" && follower === ":"; + const closesValue = inString === "value" && + (follower === "," || follower === "}" || follower === "]"); + if (closesKey || closesValue) { + // The damaged byte sits where this string's closing quote was. + inString = null; + expectKey = false; + expectValue = false; + index = after; + continue; + } + // Damage inside the text; the string still ends at its own closing quote. + index += 1; continue; } - if (inString) { + if (inString !== null) { if (escaped) { escaped = false; - } else if (character === "\\") { + index += 1; + continue; + } + if (character === "\\") { escaped = true; - } else if (character === '"') { - inString = false; + index += 1; + continue; } - index += 1; + if (character !== '"') { + index += 1; + continue; + } + const role = inString; + inString = null; + const after = skipWhitespace(row, index + 1); + const follower = row.text[after]; + if (role === "key") { + if (follower !== ":") ambiguous = true; + } else if (follower !== "," && follower !== "}" && follower !== "]") { + // A value string must be followed by the end of its container or the + // next member; anything else means the closing quote was misread. + ambiguous = true; + } + expectKey = false; + expectValue = false; + index = after; continue; } if (character === '"') { - const key = parseStringAt(row, index); - if (key.ok && typeof key.value === "string" && depth === 1) { - const colon = skipWhitespace(row, key.next); - if (row.text[colon] === ":" && !row.damaged.has(colon)) { - if (wanted.has(key.value)) offsets.push(index); - index = colon + 1; + const role: "key" | "value" = expectValue || + containers[containers.length - 1] === "array" ? "value" : "key"; + const parsed = parseStringAt(row, index); + if (role === "key") { + if (!parsed.ok || typeof parsed.value !== "string") { + // The name cannot be read; keep scanning inside the string so the + // structure behind it stays available. + lostKeyName = true; + inString = "key"; + escaped = false; + index += 1; + continue; + } + const colon = skipWhitespace(row, parsed.next); + if (row.text[colon] !== ":" || row.damaged.has(colon)) { + // This was a value string mistaken for a key; it still ended here. + expectKey = false; + expectValue = false; + index = parsed.next; continue; } + if (containers.length === 1 && wanted.has(parsed.value)) { + offsets.push(index); + } + expectValue = true; + expectKey = false; + index = colon + 1; + continue; + } + if (!parsed.ok) { + // The value string is damaged; keep scanning inside it. + inString = "value"; + escaped = false; + index += 1; + continue; + } + expectValue = false; + expectKey = false; + index = parsed.next; + continue; + } + if (character === "{") { + containers.push("object"); + expectKey = true; + expectValue = false; + index += 1; + continue; + } + if (character === "[") { + containers.push("array"); + expectKey = false; + expectValue = true; + index += 1; + continue; + } + if (character === "}" || character === "]") { + const expected = character === "}" ? "object" : "array"; + if (containers[containers.length - 1] !== expected) { + ambiguous = true; + } else { + containers.pop(); } - inString = true; - escaped = false; + expectKey = false; + expectValue = false; + index += 1; + continue; + } + if (character === ",") { + const top = containers[containers.length - 1]; + expectKey = top === "object"; + expectValue = top === "array"; index += 1; continue; } - if (character === "{" || character === "[") { - depth += 1; - } else if (character === "}" || character === "]") { - depth -= 1; + if (character === ":") { + expectKey = false; + expectValue = true; + index += 1; + continue; } + // Any other byte continues a scalar value (number or literal). + expectKey = false; + expectValue = false; index += 1; } - return offsets; + if (containers.length > 0) ambiguous = true; + return { offsets, lostKeyName, ambiguous }; } /** What one row still carries of the members a caller named. */ @@ -283,6 +424,22 @@ export interface LineMembers { * absent: the row did contain it, so "absent" would be a different fact. */ readonly unreadable: readonly string[]; + /** + * A key's name met an undecodable byte. + * + * The recovered member set is therefore incomplete: a member a caller needs + * may be the one whose name was lost, and a caller cannot read its absence as + * proof that the row never carried it. + */ + readonly lostKeyName: boolean; + /** + * A separator or container boundary was unknowable. + * + * The members before the damage are exact; the ones after it are best effort. + * A caller that must decide an identity from this row has to treat a missing + * identity as unreadable rather than absent. + */ + readonly ambiguous: boolean; } /** @@ -302,8 +459,11 @@ export function jsonLineMembers( const wanted = new Set(keys); const members: [string, unknown][] = []; const unreadable: string[] = []; - if (wanted.size === 0) return { members, unreadable }; - for (const offset of topLevelMemberOffsets(row, wanted)) { + if (wanted.size === 0) { + return { members, unreadable, lostKeyName: false, ambiguous: false }; + } + const scan = topLevelMemberOffsets(row, wanted); + for (const offset of scan.offsets) { const key = parseStringAt(row, offset); if (!key.ok || typeof key.value !== "string") continue; const colon = skipWhitespace(row, key.next); @@ -314,5 +474,10 @@ export function jsonLineMembers( } members.push([key.value, value.value]); } - return { members, unreadable: [...new Set(unreadable)] }; + return { + members, + unreadable: [...new Set(unreadable)], + lostKeyName: scan.lostKeyName, + ambiguous: scan.ambiguous, + }; } diff --git a/loopx/control_plane/runtime/receipt_log_snapshot.ts b/loopx/control_plane/runtime/receipt_log_snapshot.ts index 4857c30aed..d97720af50 100644 --- a/loopx/control_plane/runtime/receipt_log_snapshot.ts +++ b/loopx/control_plane/runtime/receipt_log_snapshot.ts @@ -14,6 +14,10 @@ export interface MalformedLineMembers { readonly members: readonly (readonly [string, unknown])[]; /** Named members the line contained but could not read intact. */ readonly unreadable: readonly string[]; + /** A key's name met an undecodable byte; the member set is incomplete. */ + readonly lostKeyName: boolean; + /** A separator or container boundary was unknowable; later members are guessed. */ + readonly ambiguous: boolean; } export interface ReceiptLogSnapshot { @@ -76,6 +80,8 @@ function freezeMalformed(line: MalformedLineMembers): MalformedLineMembers { Object.freeze([key, value] as const) ))), unreadable: Object.freeze([...line.unreadable]), + lostKeyName: line.lostKeyName, + ambiguous: line.ambiguous, }); } @@ -163,7 +169,8 @@ export async function readReceiptLogSnapshot( if (line === null) { firstErrorLine ??= lines; const recovered = jsonLineMembers(raw, recoveryKeys); - if (recovered.members.length > 0 || recovered.unreadable.length > 0) { + if (recovered.members.length > 0 || recovered.unreadable.length > 0 || + recovered.ambiguous || recovered.lostKeyName) { malformedLines.push(freezeMalformed({ line: lines, ...recovered })); } } else if (line.trim()) { @@ -172,7 +179,8 @@ export async function readReceiptLogSnapshot( } catch { firstErrorLine ??= lines; const recovered = jsonLineMembers(raw, recoveryKeys); - if (recovered.members.length > 0 || recovered.unreadable.length > 0) { + if (recovered.members.length > 0 || recovered.unreadable.length > 0 || + recovered.ambiguous || recovered.lostKeyName) { malformedLines.push(freezeMalformed({ line: lines, ...recovered })); } } diff --git a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py index c7b83a00a1..68547318a9 100644 --- a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py +++ b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py @@ -451,6 +451,46 @@ def _damage_committed_summary(path: Path, turn_id: str) -> None: path.write_bytes(b"\n".join(lines)) +def _damage_committed_detail_quote( + path: Path, turn_id: str, which: str +) -> None: + """Damage one byte of a nested display value in a committed receipt. + + ``which`` names the byte: the opening quote of the value, its closing + quote, or a byte inside the text. Identity members and the terminating + newline stay untouched, so the row still proves its writer finished it. + """ + lines = path.read_bytes().split(b"\n") + damaged = 0 + for index, line in enumerate(lines): + if turn_id.encode() not in line: + continue + marker = b'"quota_command":' + at = line.find(marker) + if at < 0: + raise AssertionError("fixture receipt has no nested detail value") + opener = at + len(marker) + while line[opener:opener + 1] in (b" ", b"\t"): + opener += 1 + if line[opener:opener + 1] != b'"': + raise AssertionError("fixture receipt detail value is not a string") + closer = line.find(b'"', opener + 1) + if closer < 0: + raise AssertionError("fixture receipt has no closing quote") + if which == "open": + position = opener + elif which == "close": + position = closer + elif which == "inside": + position = opener + 3 + else: # pragma: no cover - parametrization guard + raise AssertionError(f"unknown damage position {which!r}") + lines[index] = line[:position] + b"\xff" + line[position + 1:] + damaged += 1 + assert damaged == 1 + path.write_bytes(b"\n".join(lines)) + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize( "command", ["scheduler-ack-current", "scheduler-fail-current"], @@ -509,6 +549,136 @@ def test_damaged_committed_successor_still_supersedes_the_older_turn( assert result["scheduler_state_mutated"] is False +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +@pytest.mark.parametrize("damage", ["open", "close", "inside"]) +def test_damaged_detail_quote_still_retires_the_older_turn( + tmp_path: Path, provider: str, command: str, damage: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-{damage}", + scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + for turn_id in ("turn-1", "turn-2"): + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + turn_id, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + _damage_committed_detail_quote(log_path, "turn-2", damage) + before = log_path.read_bytes() + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # A damaged display value never hides the Turn its writer committed. Even + # when the scan cannot say what the row contained, "unknown" is not + # "absent": the older return must not regain write authority. + assert code == 1, result + assert result["ok"] is False + assert result["status"] == "heartbeat_receipt_stale" + assert result["scheduler_state_mutated"] is False + assert log_path.read_bytes() == before + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +def test_damaged_detail_quote_in_another_agent_row_is_tolerated( + tmp_path: Path, provider: str, command: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}", scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + "turn-1", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + foreign = build_rollout_event( + goal_id=goal_id, + event_kind="quota_should_run", + agent_id="another-agent", + run_id="turn-other", + details={"quota_command": "should-run", "turn_instance_id": "turn-other"}, + ) + with log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(foreign, sort_keys=True, ensure_ascii=False) + "\n") + _damage_committed_detail_quote(log_path, "turn-other", "open") + before = log_path.read_bytes() + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # The damaged row's readable agent identity proves it cannot supersede this + # Agent's Turn, so an otherwise valid return must still complete. + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + assert log_path.read_bytes() == before + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize("command", ["scheduler-ack-current", "scheduler-fail-current"]) @pytest.mark.parametrize("row_kind", ["other_agent", "validation"]) diff --git a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts index f4b62e8e9b..873dec7f8f 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts @@ -285,6 +285,115 @@ test("a committed successor stays superseding after its early field is damaged", } }); +/** A committed successor row shaped like the writer's sorted-key output. */ +function receiptRowWithDetails(turnId: string, agentId = scope.agent_id): string { + return JSON.stringify({ + agent_id: agentId, + boundary: { absolute_paths_recorded: false, credential_values_recorded: false }, + details: { + command: "quota", + quota_command: "should-run", + todo_id: "", + turn_instance_id: turnId, + }, + event_id: `event-${turnId}`, + event_kind: "quota_should_run", + goal_id: scope.goal_id, + recorded_at: "2026-08-27T06:30:00Z", + run_id: turnId, + schema_version: "loopx_rollout_event_v0", + status: "normal_run", + summary: `receipt ${turnId}`, + }); +} + +function damageLineAt(line: string, index: number): Buffer { + const bytes = Buffer.from(line, "utf8"); + return Buffer.concat([ + bytes.subarray(0, index), + Buffer.from([0xff]), + bytes.subarray(index + 1), + ]); +} + +test("a damaged quote inside details cannot hide a committed successor", async (t) => { + const cases: readonly { + name: string; + agentId?: string; + stale: boolean; + at: (line: string) => number; + }[] = [ + { + name: "damaged opening quote", + stale: true, + at: (line) => line.indexOf('"quota_command":"') + '"quota_command":'.length, + }, + { + name: "damaged closing quote", + stale: true, + at: (line) => { + const opener = line.indexOf('"quota_command":"') + '"quota_command":'.length; + return line.indexOf('"', opener + 1); + }, + }, + { + name: "damage inside the text", + stale: true, + at: (line) => line.indexOf('"should-run"') + 3, + }, + { + // A readable agent identity proves this row belongs to somebody else, + // so its damaged display field must still be tolerated. + name: "damaged opening quote in another Agent's row", + agentId: "other-agent", + stale: false, + at: (line) => line.indexOf('"quota_command":"') + '"quota_command":'.length, + }, + ]; + for (const operation of OPERATIONS) { + await t.test(operation, { timeout: 30_000 }, async (t) => { + for (const { name, agentId, stale, at } of cases) { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const path = receiptLogPath(runtimeRoot); + const line = receiptRowWithDetails("turn-followup-2", agentId); + const index = at(line); + assert.ok(index > 0, `${name}: fixture lost its damage point`); + await appendFile(path, Buffer.concat([ + damageLineAt(line, index), + Buffer.from("\n"), + ])); + const before = await readFile(path); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const result = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + if (stale) { + assert.equal( + result.error_code, + "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE", + `${name} let ${operation} write stale state`, + ); + assert.equal(result.scheduler_state_mutated, false, name); + assert.deepEqual(await readFile(path), before, name); + await assert.rejects(readFile(statePath, "utf8"), name); + } else { + assert.equal(result.ok, true, name); + assert.equal(result.scheduler_state_mutated, true, name); + assert.deepEqual(await readFile(path), before, name); + const persisted = JSON.parse(await readFile(statePath, "utf8")); + assert.equal(persisted.heartbeat_commit.operation, operation, name); + } + } + }); + } +}); + test("only an ambiguous receipt of this scope fails closed", async (t) => { const rows: readonly { row: string; From 19fdb5ceb73c5c5ee9b6661e575c425bfb3e0672 Mon Sep 17 00:00:00 2001 From: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> Date: Wed, 18 Nov 2026 09:31:13 +0800 Subject: [PATCH 3/8] fix(scheduler): keep a duplicated scope identity from hiding a successor A stray rewrite can repeat a top-level scope member of an already committed quota_should_run receipt (for example agent_id) while the only damaged byte is in its display summary. Recovery now reads every value a member carries, so a duplicate that repeats the same identity merges into one scope instead of discarding the whole row. Duplicates whose values disagree keep the row inside this scope and answer "unknown", which the freshness check already treats as stale; only a row whose every scope value is foreign stays tolerated, together with diagnostics whose key names survived. Verified with real File and SQLite scheduler-ack-current and scheduler-fail-current commands: a committed turn-2 receipt with a repeated agent_id and one damaged summary byte exits 1 with heartbeat_receipt_stale, scheduler_state_mutated=false and unchanged log bytes, while a foreign Agent's row with the same shape still lets the current turn write. The full Python receipt-fence suite passes on File and SQLite, the six scheduler TypeScript suites pass, and control-plane typecheck passes. Signed-off-by: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> --- loopx/control_plane/rollout_receipt_log.ts | 97 ++++++----- .../test_scheduler_heartbeat_receipt_fence.py | 151 ++++++++++++++++++ .../scheduler_heartbeat_receipt_fence.test.ts | 45 +++++- 3 files changed, 245 insertions(+), 48 deletions(-) diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index 84acde88ec..34be4104a6 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -245,47 +245,50 @@ function committedSuccessor( agentId: string, goalRef: JsonObject | null, ): CommittedSuccessor | null { - const fields = new Map(); - // Whether the row carries more than one Turn identity. Its scope is checked - // first: an ambiguous Turn in somebody else's row must not retire this - // transaction, and neither must a diagnostic row that is not a receipt at all. - let ambiguousTurn = false; + // Every value the row shows for a member, not just the first. A duplicate + // that names the same value as its twin says nothing new, while a duplicate + // that disagrees must not be allowed to erase the identity it contradicts. + const observed = new Map(); for (const [key, value] of damaged.members) { - if ( - fields.has(key) && - (ADMISSION_IDENTITY_MEMBERS as readonly string[]).includes(key) - ) { - if (key !== "run_id") return null; - if (fields.get(key) !== value) ambiguousTurn = true; - } - fields.set(key, value); - } - for (const key of damaged.unreadable) { - if (!(ADMISSION_IDENTITY_MEMBERS as readonly string[]).includes(key)) continue; - // Only the Turn id may be unknown: without the receipt kind or its scope the - // row cannot be attributed to this transaction at all. - if (key !== "run_id") return null; - if (fields.has("run_id")) ambiguousTurn = true; + const existing = observed.get(key); + if (existing === undefined) observed.set(key, [value]); + else existing.push(value); } - // A member that is present with another value proves the row is outside this - // scope; only then may damage elsewhere in the row be tolerated. Missing - // members are handled below, because damage can hide a member entirely. - if (fields.has("schema_version") && - fields.get("schema_version") !== ROLLOUT_EVENT_SCHEMA_VERSION) return null; - if (fields.has("event_kind") && - fields.get("event_kind") !== HEARTBEAT_RECEIPT_EVENT_KIND) return null; - if (fields.has("goal_id") && fields.get("goal_id") !== goalId) return null; - if (fields.has("agent_id") && fields.get("agent_id") !== agentId) return null; + const equal = (left: unknown, right: unknown): boolean => + JSON.stringify(left) === JSON.stringify(right); + const distinct = (key: string): unknown[] => { + const seen: unknown[] = []; + for (const value of observed.get(key) ?? []) { + if (!seen.some((item) => equal(item, value))) seen.push(value); + } + return seen; + }; + const confirms = (key: string, expected: unknown): boolean => + distinct(key).includes(expected); + const goalRefMatches = (value: unknown): boolean => { + if (goalRef === null) return false; + const eventGoalRef = jsonObject(value); + return eventGoalRef !== null && + eventGoalRef.goal_id === goalRef.goal_id && + eventGoalRef.goal_instance_id === goalRef.goal_instance_id; + }; + + // A member that shows a value of this scope keeps the row inside it. Only a + // member whose every value is foreign proves the row unrelated, which is the + // one case where damage elsewhere may be tolerated without a successor. + if (observed.has("schema_version") && + !confirms("schema_version", ROLLOUT_EVENT_SCHEMA_VERSION)) return null; + if (observed.has("event_kind") && + !confirms("event_kind", HEARTBEAT_RECEIPT_EVENT_KIND)) return null; + if (observed.has("goal_id") && !confirms("goal_id", goalId)) return null; + if (observed.has("agent_id") && !confirms("agent_id", agentId)) return null; if (goalRef === null) { - if (fields.has("goal_ref")) return null; - } else if (fields.has("goal_ref")) { - const eventGoalRef = jsonObject(fields.get("goal_ref")); - if ( - eventGoalRef === null || - eventGoalRef.goal_id !== goalRef.goal_id || - eventGoalRef.goal_instance_id !== goalRef.goal_instance_id - ) return null; + if (observed.has("goal_ref")) return null; + } else if (observed.has("goal_ref") && + !(observed.get("goal_ref") ?? []).some(goalRefMatches)) { + return null; } + // Scope members whose bytes could not be read keep the row inside this // transaction without proving which Turn admitted it. for (const key of damaged.unreadable) { @@ -293,24 +296,30 @@ function committedSuccessor( if (key !== "run_id") return { kind: "unknown" }; } const identityUncertain = damaged.lostKeyName || damaged.ambiguous; - const required = goalRef === null + const scopeKeys = goalRef === null ? ["schema_version", "event_kind", "goal_id", "agent_id"] : ["schema_version", "event_kind", "goal_id", "agent_id", "goal_ref"]; - if (required.some((key) => !fields.has(key))) { + if (scopeKeys.some((key) => distinct(key).length === 0)) { // Without damage such a row simply is not one of this scope's receipts. // Damage can have hidden the member behind a key name or separator it // could not read, so the row is unreadable instead of unrelated. return identityUncertain ? { kind: "unknown" } : null; } - // The row now proves it is one of this scope's heartbeat receipts. Only then - // does an unknown Turn mean "somebody else's turn superseded mine". + // The row proves it is one of this scope's heartbeat receipts. A duplicate + // scope member that names two different values cannot be resolved, and the + // same row with only one of them written would still be a successor. + if (scopeKeys.some((key) => distinct(key).length > 1)) { + return { kind: "unknown" }; + } + // Only then does an unknown Turn mean "somebody else's turn superseded mine". if (damaged.ambiguous) return { kind: "unknown" }; - if (ambiguousTurn) return { kind: "unattributed" }; - const runId = fields.get("run_id"); + const turnValues = distinct("run_id"); + if (turnValues.length > 1) return { kind: "unattributed" }; + if (damaged.unreadable.includes("run_id")) return { kind: "unattributed" }; + const runId = turnValues[0]; if (typeof runId === "string" && runId.trim().length > 0) { return { kind: "superseded", runId }; } - if (damaged.unreadable.includes("run_id")) return { kind: "unattributed" }; // Nothing names a Turn here. A lost key name could have been that Turn, so // only a row whose key names all survived may be read as a diagnostic. return damaged.lostKeyName ? { kind: "unknown" } : null; diff --git a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py index 68547318a9..436c623331 100644 --- a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py +++ b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py @@ -549,6 +549,157 @@ def test_damaged_committed_successor_still_supersedes_the_older_turn( assert result["scheduler_state_mutated"] is False +def _duplicate_committed_identity(path: Path, turn_id: str) -> None: + """Repeat the receipt's own top-level agent identity once. + + A stray rewrite can add a member the writer never duplicated. Repeating + the exact same value still names one identity, so it must not erase the + Turn the row already proves. + """ + lines = path.read_bytes().split(b"\n") + for index, line in enumerate(lines): + if turn_id.encode() not in line: + continue + payload = json.loads(line) + agent = payload["agent_id"] + close = line.rfind(b"}") + duplicate = b',"agent_id": ' + json.dumps(agent, ensure_ascii=False).encode() + lines[index] = line[:close] + duplicate + line[close:] + path.write_bytes(b"\n".join(lines)) + return + raise AssertionError("fixture receipt not found") + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +def test_duplicate_scope_identity_does_not_hide_the_committed_successor( + tmp_path: Path, provider: str, command: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-duplicate", scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + for turn_id in ("turn-1", "turn-2"): + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + turn_id, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + _duplicate_committed_identity(log_path, "turn-2") + _damage_committed_summary(log_path, "turn-2") + before = log_path.read_bytes() + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # A duplicate that repeats the same scope names one identity, so the Turn + # it already proves still retires the older return. + assert code == 1, result + assert result["ok"] is False + assert result["status"] == "heartbeat_receipt_stale" + assert result["scheduler_state_mutated"] is False + assert log_path.read_bytes() == before + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +def test_foreign_duplicate_identity_with_damaged_summary_is_tolerated( + tmp_path: Path, provider: str, command: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-foreign-duplicate", scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + "turn-1", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + foreign = build_rollout_event( + goal_id=goal_id, + event_kind="quota_should_run", + agent_id="another-agent", + run_id="turn-other", + summary="heartbeat quota receipt committed for turn=turn-other stall=not_applicable", + details={"quota_command": "should-run"}, + ) + with log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(foreign, sort_keys=True, ensure_ascii=False) + "\n") + _duplicate_committed_identity(log_path, "turn-other") + _damage_committed_summary(log_path, "turn-other") + before = log_path.read_bytes() + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # Every agent identity the row shows is another Agent's, so the damaged + # audit row is provably unrelated and the current return must complete. + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + assert log_path.read_bytes() == before + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize( "command", ["scheduler-ack-current", "scheduler-fail-current"], diff --git a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts index 873dec7f8f..ab4593d034 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts @@ -286,8 +286,12 @@ test("a committed successor stays superseding after its early field is damaged", }); /** A committed successor row shaped like the writer's sorted-key output. */ -function receiptRowWithDetails(turnId: string, agentId = scope.agent_id): string { - return JSON.stringify({ +function receiptRowWithDetails( + turnId: string, + agentId = scope.agent_id, + duplicateAgentIds: readonly string[] = [], +): string { + const row = JSON.stringify({ agent_id: agentId, boundary: { absolute_paths_recorded: false, credential_values_recorded: false }, details: { @@ -305,6 +309,11 @@ function receiptRowWithDetails(turnId: string, agentId = scope.agent_id): string status: "normal_run", summary: `receipt ${turnId}`, }); + if (duplicateAgentIds.length === 0) return row; + const close = row.lastIndexOf("}"); + return row.slice(0, close) + + duplicateAgentIds.map((value) => `,"agent_id":${JSON.stringify(value)}`).join("") + + row.slice(close); } function damageLineAt(line: string, index: number): Buffer { @@ -320,6 +329,7 @@ test("a damaged quote inside details cannot hide a committed successor", async ( const cases: readonly { name: string; agentId?: string; + duplicateAgentIds?: readonly string[]; stale: boolean; at: (line: string) => number; }[] = [ @@ -349,14 +359,41 @@ test("a damaged quote inside details cannot hide a committed successor", async ( stale: false, at: (line) => line.indexOf('"quota_command":"') + '"quota_command":'.length, }, + { + // A duplicate that repeats the same identity says nothing new. It must + // not erase the Turn the row already proves. + name: "duplicate identical agent identity beside damaged detail", + duplicateAgentIds: [scope.agent_id], + stale: true, + at: (line) => line.indexOf('"quota_command":"') + '"quota_command":'.length, + }, + { + // One duplicate names this scope and one names another, so the row + // cannot be resolved into a single successor identity. + name: "conflicting agent identities beside damaged detail", + duplicateAgentIds: ["other-agent"], + stale: true, + at: (line) => line.indexOf('"quota_command":"') + '"quota_command":'.length, + }, + { + // Every duplicate agent identity is somebody else's: the row is + // provably foreign and its damage may be tolerated. + name: "foreign duplicate agent identities beside damaged detail", + agentId: "other-agent", + duplicateAgentIds: ["other-agent-2"], + stale: false, + at: (line) => line.indexOf('"quota_command":"') + '"quota_command":'.length, + }, ]; for (const operation of OPERATIONS) { await t.test(operation, { timeout: 30_000 }, async (t) => { - for (const { name, agentId, stale, at } of cases) { + for (const { name, agentId, duplicateAgentIds, stale, at } of cases) { const runtimeRoot = await tempRuntime(t); await appendReceipt(runtimeRoot, "turn-followup-1"); const path = receiptLogPath(runtimeRoot); - const line = receiptRowWithDetails("turn-followup-2", agentId); + const line = receiptRowWithDetails( + "turn-followup-2", agentId, duplicateAgentIds, + ); const index = at(line); assert.ok(index > 0, `${name}: fixture lost its damage point`); await appendFile(path, Buffer.concat([ From b829fdf2f6332188fe94ac16d5197ea57cc2c153 Mon Sep 17 00:00:00 2001 From: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:49:20 +0800 Subject: [PATCH 4/8] fix(scheduler): only a readable foreign identity may exclude a successor A stray rewrite can duplicate a top-level scope member of a committed quota_should_run receipt, name another Agent first and leave this scope's copy where a single damaged byte makes it unreadable. The recovery reader reported both the readable foreign value and the damaged key, but the successor check trusted the foreign value alone and returned "unrelated", so an older Turn regained write authority. The exclusion branch now consults the same recovered key list: a member whose every readable copy is foreign still proves the row unrelated only when no copy of that member is unreadable. If the same member is also damaged, the row answers "unknown", which the freshness check already treats as stale. Fully readable foreign rows and surviving diagnostics stay tolerated. Verified with real File and SQLite scheduler-ack-current and scheduler-fail-current commands: a committed turn-2 receipt whose first agent_id names another Agent while the trailing local copy holds a raw 0xff now exits 1 with heartbeat_receipt_stale, scheduler_state_mutated=false and unchanged log bytes. The mirrored order (local readable first, foreign copy damaged) also keeps the old Turn retired. The full Python receipt-fence suite passes (62 tests, File and SQLite), the six scheduler TypeScript suites pass (99 tests), and control-plane typecheck passes. Signed-off-by: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> --- loopx/control_plane/rollout_receipt_log.ts | 34 +++-- .../test_scheduler_heartbeat_receipt_fence.py | 136 ++++++++++++++++++ .../scheduler_heartbeat_receipt_fence.test.ts | 18 +++ 3 files changed, 179 insertions(+), 9 deletions(-) diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index 34be4104a6..9157777442 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -273,20 +273,36 @@ function committedSuccessor( eventGoalRef.goal_instance_id === goalRef.goal_instance_id; }; - // A member that shows a value of this scope keeps the row inside it. Only a - // member whose every value is foreign proves the row unrelated, which is the - // one case where damage elsewhere may be tolerated without a successor. + const unreadableScope = new Set( + damaged.unreadable.filter( + (key) => (ADMISSION_IDENTITY_MEMBERS as readonly string[]).includes(key), + ), + ); + // A foreign value only proves the row is unrelated when every copy of that + // member is readable: the same member may be duplicated, and a damaged copy + // that could have named this scope leaves the row unreadable rather than + // unrelated. Absence of proof here is not proof of absence. if (observed.has("schema_version") && - !confirms("schema_version", ROLLOUT_EVENT_SCHEMA_VERSION)) return null; + !confirms("schema_version", ROLLOUT_EVENT_SCHEMA_VERSION)) { + return unreadableScope.has("schema_version") ? { kind: "unknown" } : null; + } if (observed.has("event_kind") && - !confirms("event_kind", HEARTBEAT_RECEIPT_EVENT_KIND)) return null; - if (observed.has("goal_id") && !confirms("goal_id", goalId)) return null; - if (observed.has("agent_id") && !confirms("agent_id", agentId)) return null; + !confirms("event_kind", HEARTBEAT_RECEIPT_EVENT_KIND)) { + return unreadableScope.has("event_kind") ? { kind: "unknown" } : null; + } + if (observed.has("goal_id") && !confirms("goal_id", goalId)) { + return unreadableScope.has("goal_id") ? { kind: "unknown" } : null; + } + if (observed.has("agent_id") && !confirms("agent_id", agentId)) { + return unreadableScope.has("agent_id") ? { kind: "unknown" } : null; + } if (goalRef === null) { - if (observed.has("goal_ref")) return null; + if (observed.has("goal_ref")) { + return unreadableScope.has("goal_ref") ? { kind: "unknown" } : null; + } } else if (observed.has("goal_ref") && !(observed.get("goal_ref") ?? []).some(goalRefMatches)) { - return null; + return unreadableScope.has("goal_ref") ? { kind: "unknown" } : null; } // Scope members whose bytes could not be read keep the row inside this diff --git a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py index 436c623331..298a6c253e 100644 --- a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py +++ b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py @@ -631,6 +631,142 @@ def test_duplicate_scope_identity_does_not_hide_the_committed_successor( assert log_path.read_bytes() == before +def _append_committed_agent_identity(path: Path, turn_id: str, value: str) -> None: + """Add one more top-level agent identity member to a committed row.""" + lines = path.read_bytes().split(b"\n") + for index, line in enumerate(lines): + if turn_id.encode() not in line: + continue + close = line.rfind(b"}") + duplicate = b',"agent_id": ' + json.dumps(value, ensure_ascii=False).encode() + lines[index] = line[:close] + duplicate + line[close:] + path.write_bytes(b"\n".join(lines)) + return + raise AssertionError("fixture receipt not found") + + +def _set_committed_leading_agent_identity( + path: Path, turn_id: str, value: str +) -> None: + """Rewrite the row's first top-level agent identity to another value.""" + lines = path.read_bytes().split(b"\n") + marker = b'"agent_id":' + for index, line in enumerate(lines): + if turn_id.encode() not in line: + continue + at = line.find(marker) + if at < 0: + raise AssertionError("fixture receipt has no agent identity") + cursor = at + len(marker) + while line[cursor:cursor + 1] in (b" ", b"\t"): + cursor += 1 + if line[cursor:cursor + 1] != b'"': + raise AssertionError("fixture receipt agent identity is not a string") + end = cursor + 1 + while True: + end = line.find(b'"', end) + if end < 0: + raise AssertionError("fixture receipt agent identity is unterminated") + if line[end - 1:end] != b"\\": + break + end += 1 + lines[index] = ( + line[:cursor] + json.dumps(value, ensure_ascii=False).encode() + + line[end + 1:] + ) + path.write_bytes(b"\n".join(lines)) + return + raise AssertionError("fixture receipt not found") + + +def _damage_committed_agent_member(path: Path, turn_id: str, value: str) -> None: + """Damage one byte inside the last copy of the named agent identity.""" + lines = path.read_bytes().split(b"\n") + token = json.dumps(value, ensure_ascii=False).encode() + for index, line in enumerate(lines): + if turn_id.encode() not in line: + continue + at = line.rfind(token) + if at < 0: + raise AssertionError("fixture receipt has no such agent identity") + lines[index] = line[:at + 1] + b"\xff" + line[at + 2:] + path.write_bytes(b"\n".join(lines)) + return + raise AssertionError("fixture receipt not found") + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +@pytest.mark.parametrize("order", ["foreign-first", "local-first"]) +def test_damaged_duplicate_agent_identity_still_supersedes_the_older_turn( + tmp_path: Path, provider: str, command: str, order: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-{order}-duplicate-agent", + scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + for turn_id in ("turn-1", "turn-2"): + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + turn_id, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + if order == "foreign-first": + # The row's first identity names another Agent while its trailing twin, + # which could have named this scope, is unreadable. + _duplicate_committed_identity(log_path, "turn-2") + _set_committed_leading_agent_identity(log_path, "turn-2", "another-agent") + _damage_committed_agent_member(log_path, "turn-2", SCOPED_AGENT_ID) + else: + # The readable identity names this scope while its foreign twin is + # unreadable; either way the row still proves an admitted successor. + _append_committed_agent_identity(log_path, "turn-2", "another-agent") + _damage_committed_agent_member(log_path, "turn-2", "another-agent") + before = log_path.read_bytes() + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # A damaged duplicate cannot prove the row is unrelated, so the older + # return must not regain write authority. + assert code == 1, result + assert result["ok"] is False + assert result["status"] == "heartbeat_receipt_stale" + assert result["scheduler_state_mutated"] is False + assert log_path.read_bytes() == before + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize( "command", ["scheduler-ack-current", "scheduler-fail-current"], diff --git a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts index ab4593d034..861be4dea2 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts @@ -384,6 +384,24 @@ test("a damaged quote inside details cannot hide a committed successor", async ( stale: false, at: (line) => line.indexOf('"quota_command":"') + '"quota_command":'.length, }, + { + // The first copy of the identity names somebody else, but its twin is + // unreadable and could have named this scope. A foreign value may only + // exclude a row when every copy of that member is readable. + name: "foreign agent identity ahead of damaged local duplicate", + agentId: "other-agent", + duplicateAgentIds: [scope.agent_id], + stale: true, + at: (line) => line.lastIndexOf(JSON.stringify(scope.agent_id)) + 1, + }, + { + // The readable copy names this scope while its damaged twin could + // equally have named it, so the row still proves a successor. + name: "local agent identity ahead of damaged foreign duplicate", + duplicateAgentIds: ["other-agent"], + stale: true, + at: (line) => line.lastIndexOf(JSON.stringify("other-agent")) + 1, + }, ]; for (const operation of OPERATIONS) { await t.test(operation, { timeout: 30_000 }, async (t) => { From 748f14f7c90554a16a322aafe254d05d48c37e40 Mon Sep 17 00:00:00 2001 From: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:40:12 +0800 Subject: [PATCH 5/8] fix(scheduler): finish the scope check before reporting uncertainty The previous commit treated an unreadable duplicate of a scope member as uncertainty immediately, before examining the remaining members. A row whose event kind was readable as a validation event and whose duplicate copy was damaged therefore returned "unknown" even though its Agent identity was complete and clearly another Agent's. That refused valid ACK and failure writebacks on File and SQLite. committedSuccessor now examines every identity member first. A member whose every readable copy is foreign and whose own copies are all readable settles the row as unrelated, regardless of damage in a different member. Uncertainty stays local to the damaged member: a readable foreign value with an unreadable twin of the same member still cannot exclude the row, and the unreadable copies of this scope's members still answer "unknown". Verified with real File and SQLite scheduler-ack-current and scheduler-fail-current commands: another Agent's validation row with a readable event kind and one damaged duplicate of that member no longer blocks the current writeback (exit 0, scheduler_state_mutated=true, log bytes unchanged), while this scope's damaged duplicates still return heartbeat_receipt_stale with zero writes. The full Python receipt-fence suite passes (66 tests, File and SQLite), the six scheduler TypeScript suites pass (99 tests), and control-plane typecheck passes. Signed-off-by: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> --- loopx/control_plane/rollout_receipt_log.ts | 42 +++++----- .../test_scheduler_heartbeat_receipt_fence.py | 76 +++++++++++++++++++ .../scheduler_heartbeat_receipt_fence.test.ts | 48 +++++++++++- 3 files changed, 139 insertions(+), 27 deletions(-) diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index 9157777442..080341ad0e 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -281,29 +281,25 @@ function committedSuccessor( // A foreign value only proves the row is unrelated when every copy of that // member is readable: the same member may be duplicated, and a damaged copy // that could have named this scope leaves the row unreadable rather than - // unrelated. Absence of proof here is not proof of absence. - if (observed.has("schema_version") && - !confirms("schema_version", ROLLOUT_EVENT_SCHEMA_VERSION)) { - return unreadableScope.has("schema_version") ? { kind: "unknown" } : null; - } - if (observed.has("event_kind") && - !confirms("event_kind", HEARTBEAT_RECEIPT_EVENT_KIND)) { - return unreadableScope.has("event_kind") ? { kind: "unknown" } : null; - } - if (observed.has("goal_id") && !confirms("goal_id", goalId)) { - return unreadableScope.has("goal_id") ? { kind: "unknown" } : null; - } - if (observed.has("agent_id") && !confirms("agent_id", agentId)) { - return unreadableScope.has("agent_id") ? { kind: "unknown" } : null; - } - if (goalRef === null) { - if (observed.has("goal_ref")) { - return unreadableScope.has("goal_ref") ? { kind: "unknown" } : null; - } - } else if (observed.has("goal_ref") && - !(observed.get("goal_ref") ?? []).some(goalRefMatches)) { - return unreadableScope.has("goal_ref") ? { kind: "unknown" } : null; - } + // unrelated. Every member must be examined before uncertainty is reported: + // a readable foreign Agent identity proves the row unrelated even when some + // other member is damaged, while uncertainty stays local to its own member. + const foreign = new Set(); + const noteForeign = (key: string, matches: boolean): void => { + if (!observed.has(key) || matches) return; + if (!unreadableScope.has(key)) foreign.add(key); + }; + noteForeign("schema_version", + confirms("schema_version", ROLLOUT_EVENT_SCHEMA_VERSION)); + noteForeign("event_kind", confirms("event_kind", HEARTBEAT_RECEIPT_EVENT_KIND)); + noteForeign("goal_id", confirms("goal_id", goalId)); + noteForeign("agent_id", confirms("agent_id", agentId)); + noteForeign("goal_ref", goalRef === null + ? false + : (observed.get("goal_ref") ?? []).some(goalRefMatches)); + // One conclusively foreign member settles the row; damage in a different + // member cannot pull an unrelated diagnostic back into this transaction. + if (foreign.size > 0) return null; // Scope members whose bytes could not be read keep the row inside this // transaction without proving which Turn admitted it. diff --git a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py index 298a6c253e..d8357b1420 100644 --- a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py +++ b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py @@ -836,6 +836,82 @@ def test_foreign_duplicate_identity_with_damaged_summary_is_tolerated( assert log_path.read_bytes() == before +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +def test_foreign_audit_row_with_damaged_event_kind_is_tolerated( + tmp_path: Path, provider: str, command: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-foreign-event-kind", + scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + "turn-1", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + foreign = build_rollout_event( + goal_id=goal_id, + event_kind="validation", + agent_id="another-agent", + run_id="turn-other", + summary="validation audit left by another Agent", + details={"quota_command": "should-run"}, + ) + # The row keeps a readable foreign event kind while a stray duplicate of + # the same member carries an unreadable value; its Agent identity is + # complete and clearly outside this scope. + line = json.dumps(foreign, sort_keys=True, ensure_ascii=False).encode() + close = line.rfind(b"}") + line = line[:close] + b',"event_kind": "quota_should_run"' + line[close:] + at = line.rfind(b'"quota_should_run"') + 1 + line = line[:at] + b"\xff" + line[at + 1:] + with log_path.open("ab") as handle: + handle.write(line + b"\n") + before = log_path.read_bytes() + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # A conclusively foreign Agent settles the row even when another member + # carries an unreadable duplicate, so the current return must complete. + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + assert log_path.read_bytes() == before + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize( "command", ["scheduler-ack-current", "scheduler-fail-current"], diff --git a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts index 861be4dea2..ca80cacce8 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts @@ -290,6 +290,8 @@ function receiptRowWithDetails( turnId: string, agentId = scope.agent_id, duplicateAgentIds: readonly string[] = [], + eventKind = "quota_should_run", + duplicateEventKinds: readonly string[] = [], ): string { const row = JSON.stringify({ agent_id: agentId, @@ -301,7 +303,7 @@ function receiptRowWithDetails( turn_instance_id: turnId, }, event_id: `event-${turnId}`, - event_kind: "quota_should_run", + event_kind: eventKind, goal_id: scope.goal_id, recorded_at: "2026-08-27T06:30:00Z", run_id: turnId, @@ -309,10 +311,15 @@ function receiptRowWithDetails( status: "normal_run", summary: `receipt ${turnId}`, }); - if (duplicateAgentIds.length === 0) return row; + if (duplicateAgentIds.length === 0 && duplicateEventKinds.length === 0) { + return row; + } const close = row.lastIndexOf("}"); return row.slice(0, close) + duplicateAgentIds.map((value) => `,"agent_id":${JSON.stringify(value)}`).join("") + + duplicateEventKinds.map( + (value) => `,"event_kind":${JSON.stringify(value)}`, + ).join("") + row.slice(close); } @@ -330,6 +337,8 @@ test("a damaged quote inside details cannot hide a committed successor", async ( name: string; agentId?: string; duplicateAgentIds?: readonly string[]; + eventKind?: string; + duplicateEventKinds?: readonly string[]; stale: boolean; at: (line: string) => number; }[] = [ @@ -402,15 +411,46 @@ test("a damaged quote inside details cannot hide a committed successor", async ( stale: true, at: (line) => line.lastIndexOf(JSON.stringify("other-agent")) + 1, }, + { + // A conclusively foreign Agent settles the row even when another member + // carries an unreadable duplicate; uncertainty stays local to its field. + name: "foreign agent identity beside unreadable event-kind duplicate", + agentId: "other-agent", + eventKind: "validation", + duplicateEventKinds: ["quota_should_run"], + stale: false, + at: (line) => line.lastIndexOf('"event_kind":"quota_should_run"') + + '"event_kind":"'.length + 1, + }, + { + // The same damaged event-kind duplicate beside this scope's readable + // Agent could still be a successor, so the old Turn is retired. + name: "local agent identity beside unreadable event-kind duplicate", + eventKind: "validation", + duplicateEventKinds: ["quota_should_run"], + stale: true, + at: (line) => line.lastIndexOf('"event_kind":"quota_should_run"') + + '"event_kind":"'.length + 1, + }, + { + // A row whose event kind survives only as damage still keeps the old + // Turn retired, but a foreign Agent identity finishes the scope check. + name: "foreign agent identity beside unreadable event kind", + agentId: "other-agent", + stale: false, + at: (line) => line.indexOf('"event_kind":"') + '"event_kind":"'.length + 1, + }, ]; for (const operation of OPERATIONS) { await t.test(operation, { timeout: 30_000 }, async (t) => { - for (const { name, agentId, duplicateAgentIds, stale, at } of cases) { + for (const { name, agentId, duplicateAgentIds, eventKind, + duplicateEventKinds, stale, at } of cases) { const runtimeRoot = await tempRuntime(t); await appendReceipt(runtimeRoot, "turn-followup-1"); const path = receiptLogPath(runtimeRoot); const line = receiptRowWithDetails( - "turn-followup-2", agentId, duplicateAgentIds, + "turn-followup-2", agentId, duplicateAgentIds, eventKind, + duplicateEventKinds, ); const index = at(line); assert.ok(index > 0, `${name}: fixture lost its damage point`); From 1ebd7a6febc3bb441157810749551d509c09e973 Mon Sep 17 00:00:00 2001 From: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:43:52 +0800 Subject: [PATCH 6/8] fix(scheduler): read only committed rows when deciding freshness A log write can stop between its complete JSON row and the newline that commits it. The snapshot exposed that final record, so a nearly written successor retired the old Turn and a nearly written original counted as the Turn's receipt. The next append drops those bytes, which made the same command flip from stale or missing to writable. The snapshot now names the unterminated record line, and the scheduler freshness read ignores that row for both the originating Turn and any successor. The two Python append entry points also deduplicate against newline-terminated rows only: retrying an interrupted append drops the abandoned bytes and commits the row once, while a committed row still returns the same record. General diagnostic reads keep the decodable tail. Verified with real File and SQLite scheduler-ack-current and scheduler-fail-current commands: an unterminated turn-2 successor no longer retires turn-1 (exit 0, scheduler_state_mutated=true), and an unterminated turn-1 row reports heartbeat_receipt_missing both before and after the next append drops it. The full Python suite passes (106 tests), appended-row dedup covers both entry points, the scheduler TypeScript suites pass (105 and 157 tests), and control-plane typecheck passes. Signed-off-by: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> --- loopx/control_plane/rollout_receipt_log.ts | 6 + .../runtime/receipt_log_snapshot.ts | 14 +- loopx/rollout_event_log.py | 10 +- .../test_scheduler_heartbeat_receipt_fence.py | 138 +++++++++++++++++- .../scheduler_heartbeat_receipt_fence.test.ts | 52 +++++++ tests/test_rollout_event_log_loading.py | 36 +++++ 6 files changed, 252 insertions(+), 4 deletions(-) diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index 080341ad0e..2b12858685 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -46,6 +46,8 @@ export interface GoalRolloutEventSnapshot { ambiguous: boolean; }>[]; readonly firstStrictErrorLine: number | null; + /** A complete final row that never received its committing newline. */ + readonly unterminatedRecordLine: number | null; } /** Reject a goal id that is not one path segment, before any log read. */ @@ -103,6 +105,7 @@ export async function readGoalRolloutEventSnapshot( eventLines: snapshot.recordLines, malformedLines: snapshot.malformedLines, firstStrictErrorLine: snapshot.firstErrorLine, + unterminatedRecordLine: snapshot.unterminatedRecordLine, }; } @@ -356,8 +359,11 @@ export async function heartbeatReceiptStatus(params: { params.agentId, goalRef, ); + // An unterminated final row never committed; the next append drops those + // bytes, so it can neither be this Turn's receipt nor supersede it. const receiptLines = snapshot.events .map((event, index) => ({ event, line: snapshot.eventLines[index] })) + .filter(({ line }) => line !== snapshot.unterminatedRecordLine) .filter(({ event }) => receipts.includes(event)); const firstIndex = receiptLines.findIndex( ({ event }) => event.run_id === params.turnInstanceId, diff --git a/loopx/control_plane/runtime/receipt_log_snapshot.ts b/loopx/control_plane/runtime/receipt_log_snapshot.ts index d97720af50..0d47013e41 100644 --- a/loopx/control_plane/runtime/receipt_log_snapshot.ts +++ b/loopx/control_plane/runtime/receipt_log_snapshot.ts @@ -33,6 +33,15 @@ export interface ReceiptLogSnapshot { */ readonly malformedLines: readonly MalformedLineMembers[]; readonly firstErrorLine: number | null; + /** + * Physical line of an unterminated final row that still decoded, or null. + * + * The row never received the newline its writer must append to commit it, + * and the next append drops those bytes. Deduplication and freshness may + * read the record for diagnosis, but only a newline-terminated row is a + * committed receipt. + */ + readonly unterminatedRecordLine: number | null; } interface Prefix extends ReceiptLogSnapshot { @@ -200,7 +209,7 @@ export async function readReceiptLogSnapshot( records: Object.freeze(records), recordLines: Object.freeze(recordLines), malformedLines: Object.freeze(malformedLines), - firstErrorLine, lines, + firstErrorLine, lines, unterminatedRecordLine: null, bytes: completeBytes, digest: digest(bytes.subarray(0, completeBytes)), }); } @@ -212,6 +221,7 @@ export async function readReceiptLogSnapshot( recordLines: prefix.recordLines, malformedLines: prefix.malformedLines, firstErrorLine: prefix.firstErrorLine ?? prefix.lines + 1, + unterminatedRecordLine: null, }); } if (!tail.trim()) return prefix; @@ -221,6 +231,7 @@ export async function readReceiptLogSnapshot( recordLines: Object.freeze([...prefix.recordLines, prefix.lines + 1]), malformedLines: prefix.malformedLines, firstErrorLine: prefix.firstErrorLine, + unterminatedRecordLine: prefix.lines + 1, }); } catch { return Object.freeze({ @@ -228,6 +239,7 @@ export async function readReceiptLogSnapshot( recordLines: prefix.recordLines, malformedLines: prefix.malformedLines, firstErrorLine: prefix.firstErrorLine ?? prefix.lines + 1, + unterminatedRecordLine: null, }); } } diff --git a/loopx/rollout_event_log.py b/loopx/rollout_event_log.py index e5d83352ca..49e3b66a53 100644 --- a/loopx/rollout_event_log.py +++ b/loopx/rollout_event_log.py @@ -211,6 +211,7 @@ def _iter_rollout_event_lines( log_path: Path, *, strict: bool = False, + committed_only: bool = False, ) -> Iterator[tuple[int, str]]: try: handle = log_path.open("rb") @@ -218,6 +219,11 @@ def _iter_rollout_event_lines( return with handle: for line_number, encoded_line in enumerate(handle, start=1): + # A row is committed only after its writer terminated it. Bytes + # after the last newline are abandoned by the next append, so + # deduplication must not treat them as an existing record. + if committed_only and not encoded_line.endswith(b"\n"): + continue try: line = encoded_line.decode("utf-8") except UnicodeDecodeError: @@ -476,7 +482,7 @@ def append_rollout_event(log_path: Path, event: Mapping[str, Any]) -> dict[str, with exclusive_cross_runtime_file_lock( log_path, operation="append_rollout_event" ): - for _, line in _iter_rollout_event_lines(log_path): + for _, line in _iter_rollout_event_lines(log_path, committed_only=True): if event_id not in line: continue try: @@ -527,7 +533,7 @@ def append_rollout_event_once( with exclusive_cross_runtime_file_lock( log_path, operation="append_rollout_event_once" ): - for _, line in _iter_rollout_event_lines(log_path): + for _, line in _iter_rollout_event_lines(log_path, committed_only=True): try: existing = json.loads(line) except json.JSONDecodeError: diff --git a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py index d8357b1420..485fda2195 100644 --- a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py +++ b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py @@ -13,7 +13,11 @@ from loopx.cli_commands import quota_scheduler_followup from loopx.control_plane.quota.scheduler_ack import _run_native_followup from loopx.control_plane.testing.canary_harness import run_json_cli_result -from loopx.rollout_event_log import build_rollout_event, rollout_event_log_path +from loopx.rollout_event_log import ( + append_rollout_event, + build_rollout_event, + rollout_event_log_path, +) GOAL_ID = "receipt-fence" @@ -836,6 +840,138 @@ def test_foreign_duplicate_identity_with_damaged_summary_is_tolerated( assert log_path.read_bytes() == before +def _write_unterminated_receipt( + log_path: Path, goal_id: str, turn_id: str +) -> bytes: + """Append complete JSON whose writer never wrote the committing newline.""" + row = build_rollout_event( + goal_id=goal_id, + event_kind="quota_should_run", + agent_id=SCOPED_AGENT_ID, + run_id=turn_id, + summary=f"heartbeat quota receipt committed for turn={turn_id}", + details={"quota_command": "should-run"}, + ) + encoded = json.dumps(row, sort_keys=True, ensure_ascii=False).encode() + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("ab") as handle: + handle.write(encoded) + return encoded + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +def test_uncommitted_successor_row_does_not_retire_the_older_turn( + tmp_path: Path, provider: str, command: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-uncommitted-successor", + scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + "turn-1", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + log_path = rollout_event_log_path(runtime_root, goal_id) + # The successor's bytes are complete JSON; only the committing newline is + # missing, and the next append drops those bytes. + _write_unterminated_receipt(log_path, goal_id, "turn-2") + before = log_path.read_bytes() + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + code, result = run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + # The uncommitted row never admitted a Turn, so the current return writes. + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + assert log_path.read_bytes() == before + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +def test_uncommitted_original_row_is_not_a_receipt_before_or_after_repair( + tmp_path: Path, provider: str, command: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-uncommitted-original", + scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + log_path = rollout_event_log_path(runtime_root, goal_id) + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_bytes( + _write_unterminated_receipt(log_path, goal_id, "turn-1") + ) + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + + def run(): + return run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + + code, result = run() + assert code == 1, result + assert result["status"] == "heartbeat_receipt_missing" + assert result["scheduler_state_mutated"] is False + # The next append drops the uncommitted row; the answer must not change. + append_rollout_event( + log_path, + build_rollout_event(goal_id=goal_id, event_kind="todo_complete"), + ) + code, result = run() + assert code == 1, result + assert result["status"] == "heartbeat_receipt_missing" + assert result["scheduler_state_mutated"] is False + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize( "command", ["scheduler-ack-current", "scheduler-fail-current"], diff --git a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts index ca80cacce8..42ec630eab 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_receipt_fence.test.ts @@ -687,6 +687,58 @@ test("a completed row missing only its newline stays uncommitted after repair", } }); +test("an unterminated successor does not retire the older turn", async (t) => { + for (const operation of OPERATIONS) { + await t.test(operation, { timeout: 30_000 }, async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + const path = receiptLogPath(runtimeRoot); + // Complete JSON whose writer never finished the row: the next append + // drops these bytes, so they cannot retire the committed Turn. + await appendFile(path, Buffer.from(receiptRow("turn-followup-2"))); + const before = await readFile(path); + const result = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + assert.equal(result.ok, true); + assert.equal(result.scheduler_state_mutated, true); + assert.deepEqual(await readFile(path), before); + const statePath = schedulerStatePath(runtimeRoot, { + goalId: scope.goal_id, + agentId: scope.agent_id, + surface: scope.surface, + stateKey: scope.state_key, + }); + const persisted = JSON.parse(await readFile(statePath, "utf8")); + assert.equal(persisted.heartbeat_commit.operation, operation); + }); + } +}); + +test("an unterminated original row is not a receipt before or after repair", async (t) => { + for (const operation of OPERATIONS) { + await t.test(operation, { timeout: 30_000 }, async (t) => { + const runtimeRoot = await tempRuntime(t); + const path = receiptLogPath(runtimeRoot); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, receiptRow("turn-followup-1")); + const rejected = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + assert.equal(rejected.status, "heartbeat_receipt_missing"); + assert.equal(rejected.scheduler_state_mutated, false); + // The next append drops the uncommitted row, and the answer must not + // change: it was never a receipt. + await runPython(t, PYTHON_UNRELATED_APPEND, [runtimeRoot, scope.goal_id]); + const afterRepair = await evaluateSchedulerHeartbeatFollowup( + followupRequest(runtimeRoot, operation), + ); + assert.equal(afterRepair.status, "heartbeat_receipt_missing"); + assert.equal(afterRepair.scheduler_state_mutated, false); + }); + } +}); + test("receipt append cannot split freshness from ACK or host-failure persistence", async (t) => { for (const operation of OPERATIONS) { await t.test(operation, { timeout: 30_000 }, async (t) => { diff --git a/tests/test_rollout_event_log_loading.py b/tests/test_rollout_event_log_loading.py index 43239071de..fb99d92016 100644 --- a/tests/test_rollout_event_log_loading.py +++ b/tests/test_rollout_event_log_loading.py @@ -270,6 +270,42 @@ def test_append_starts_a_new_row_after_a_torn_final_record( assert load_rollout_events(log_path) == [existing, written] +@pytest.mark.parametrize("idempotent", [False, True]) +def test_unterminated_json_row_is_not_an_existing_record( + tmp_path: Path, + idempotent: bool, +) -> None: + """A row missing its committing newline is not a deduplication record.""" + log_path = tmp_path / "rollout-event-log.jsonl" + event = build_rollout_event( + goal_id="goal-a", + event_kind="quota_should_run", + agent_id="agent-a", + run_id="turn-a", + status="run", + recorded_at="2026-09-23T00:00:00Z", + ) + encoded = json.dumps(event, sort_keys=True, ensure_ascii=False).encode("utf-8") + log_path.write_bytes(encoded) + # The diagnostic reader still sees the decodable tail row. + assert load_rollout_events(log_path) == [event] + + if idempotent: + written, appended = append_rollout_event_once( + log_path, + event, + identity_fields=("goal_id", "event_kind", "agent_id", "run_id"), + ) + assert appended is True + else: + written = append_rollout_event(log_path, event) + + # The tail was dropped and the event committed exactly once. + assert log_path.read_bytes() == encoded + b"\n" + assert load_rollout_events(log_path) == [event] + assert written == event + + def test_a_torn_record_is_dropped_before_the_next_row( tmp_path: Path, ) -> None: From 18e8fa6a479f381ef69a796ea98eb7a0a255fb14 Mon Sep 17 00:00:00 2001 From: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:02:25 +0800 Subject: [PATCH 7/8] fix(scheduler): ignore unterminated rows in prior-turn recovery A complete JSON row without its committing newline is abandoned by the next append. Use the same committed-only view for strict settlement, receipt qualification and prior-host-Turn recovery so an interrupted receipt cannot make a later admission ask for recovery before the next append and then disappear. Verify File and SQLite real CLI prior-turn recovery with committed and unterminated guard receipts, plus TypeScript settlement preflight and control-plane typecheck. Signed-off-by: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> --- .../quota/unsettled_host_turn_recovery.ts | 4 +- loopx/control_plane/rollout_receipt_log.ts | 50 ++++++++++++++--- .../test_quota_settlement_cli.py | 53 ++++++++++++++++++- .../quota_settlement_readback.test.ts | 29 ++++++++++ 4 files changed, 126 insertions(+), 10 deletions(-) diff --git a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts index 890054f1e3..3c4b224797 100644 --- a/loopx/control_plane/quota/unsettled_host_turn_recovery.ts +++ b/loopx/control_plane/quota/unsettled_host_turn_recovery.ts @@ -22,7 +22,7 @@ import { requireNonEmptyString, } from "../runtime_decode.ts"; import { - goalHeartbeatReceiptsFromSnapshot, + committedGoalHeartbeatReceiptsFromSnapshot, readGoalRolloutEventSnapshot, } from "../rollout_receipt_log.ts"; import { isCommittedMonitorPollEffect } from "./settlement_phase.ts"; @@ -230,7 +230,7 @@ async function preflightPriorHostTurnCloseoutForOwner( request.runtime_root, request.goal_id, ); - const receipts = goalHeartbeatReceiptsFromSnapshot( + const receipts = committedGoalHeartbeatReceiptsFromSnapshot( rolloutSnapshot, request.goal_id, request.agent_id, diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index 9d08a64675..74b942a147 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -109,6 +109,24 @@ export async function readGoalRolloutEventSnapshot( }; } +/** + * Rows whose writer reached the committing newline. + * + * The append owner drops an unterminated final row before writing its next + * record. Any read that decides whether work already happened must use this + * view; otherwise an interrupted write can look committed until the next + * unrelated append removes it. + */ +export function committedGoalRolloutEvents( + snapshot: GoalRolloutEventSnapshot | null, +): readonly JsonObject[] { + if (snapshot === null) return []; + if (snapshot.unterminatedRecordLine === null) return snapshot.events; + return snapshot.events.filter((_, index) => + snapshot.eventLines[index] !== snapshot.unterminatedRecordLine + ); +} + /** Return strict settlement input, or fail on the first malformed line. */ export function strictGoalRolloutEvents( snapshot: GoalRolloutEventSnapshot | null, @@ -119,25 +137,43 @@ export function strictGoalRolloutEvents( "malformed_settlement_state", ); } - return snapshot?.events ?? []; + return committedGoalRolloutEvents(snapshot); } -/** Qualify heartbeat receipts without discarding diagnostic quota audit rows. */ -export function goalHeartbeatReceiptsFromSnapshot( - snapshot: GoalRolloutEventSnapshot | null, +function assertGoalSnapshotScope( + snapshot: GoalRolloutEventSnapshot, goalId: string, - agentId?: string | null, -): JsonObject[] | null { - if (snapshot === null) return null; +): void { if (snapshot.goalId !== goalId) { throw new EffectRuntimeRequestError( "rollout event snapshot does not belong to the requested goal", "rollout_event_snapshot_scope_mismatch", ); } +} + +/** Qualify heartbeat receipts without discarding diagnostic quota audit rows. */ +export function goalHeartbeatReceiptsFromSnapshot( + snapshot: GoalRolloutEventSnapshot | null, + goalId: string, + agentId?: string | null, +): JsonObject[] | null { + if (snapshot === null) return null; + assertGoalSnapshotScope(snapshot, goalId); return heartbeatReceiptsFromEvents(snapshot.events, goalId, agentId); } +/** Heartbeat receipts whose row reached its committing newline. */ +export function committedGoalHeartbeatReceiptsFromSnapshot( + snapshot: GoalRolloutEventSnapshot | null, + goalId: string, + agentId?: string | null, +): JsonObject[] | null { + if (snapshot === null) return null; + assertGoalSnapshotScope(snapshot, goalId); + return heartbeatReceiptsFromEvents(committedGoalRolloutEvents(snapshot), goalId, agentId); +} + /** The heartbeat receipt filter over events a caller already read. */ export function heartbeatReceiptsFromEvents( events: readonly JsonObject[], diff --git a/tests/control_plane/test_quota_settlement_cli.py b/tests/control_plane/test_quota_settlement_cli.py index bc08f87f36..9301469878 100644 --- a/tests/control_plane/test_quota_settlement_cli.py +++ b/tests/control_plane/test_quota_settlement_cli.py @@ -33,7 +33,7 @@ ) from loopx.heartbeat_prompt import build_heartbeat_prompt from loopx.paths import shell_selected_global_registry -from loopx.rollout_event_log import build_rollout_event +from loopx.rollout_event_log import append_rollout_event, build_rollout_event from tests.control_plane.canonical_authority_fixture import ( initialize_canonical_source_fixture, read_fixture_todos, @@ -3076,6 +3076,57 @@ def error(run: dict) -> str | None: ) +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_unterminated_prior_guard_is_not_a_recovery_precondition( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +) -> None: + from canonical_authority_fixture import isolate_sqlite_runtime + + if provider == "sqlite": + isolate_sqlite_runtime(tmp_path, monkeypatch) + + for case, committed in (("committed", True), ("unterminated", False)): + case_root = tmp_path / case + project, runtime, registry_path = _write_fixture(case_root) + _initialize_fixture_authority(project, runtime, provider=provider) + prior_turn_id = f"turn-{case}-prior" + rc, prior = _run_cli( + registry_path, runtime, "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--turn-instance-id", prior_turn_id, "--todo-id", TODO_ID, + ) + assert rc == 0, prior + assert prior["heartbeat_receipt"]["closeout_required"] is True + log = runtime / "goals" / GOAL_ID / "rollout-event-log.jsonl" + if not committed: + raw = log.read_bytes() + assert raw.endswith(b"\n") + log.write_bytes(raw[:-1]) + # An unrelated append drops an unfinished tail before committing its + # own row. The prior-turn decision must be the same before and after + # that append. + append_rollout_event( + log, + build_rollout_event( + goal_id=GOAL_ID, + event_kind="codex_session_observed", + agent_id=AGENT_ID, + run_id=f"turn-{case}-unrelated", + summary="unrelated append", + ), + ) + admission_rc, admission = _run_cli( + registry_path, runtime, "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--begin-turn", + ) + assert admission_rc == 0, admission + if committed: + assert admission["effective_action"] == "unsettled_host_turn_recovery", admission + else: + assert admission["effective_action"] != "unsettled_host_turn_recovery", admission + assert _spend_run_count(runtime) == 0 + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_recovery_does_not_bind_current_replan_and_reenters_same_turn( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index 6a7b0786b5..cea5359c1b 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -1516,6 +1516,35 @@ test("keeps the same-turn rebind when no earlier Turn owes a closeout", async t } }); +test("an unterminated prior guard is not an owed closeout before or after repair", async () => { + const root = await fixture({turnId: priorTurnId}); + try { + const eventsPath = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const original = JSON.parse((await readFile(eventsPath, "utf8")).trim()); + original.details.closeout_required = true; + // The guard writer stopped short of its committing newline. + await writeFile(eventsPath, JSON.stringify(original)); + const preflightRequest = { + schema_version: PRIOR_HOST_TURN_CLOSEOUT_PREFLIGHT_REQUEST_SCHEMA, + runtime_root: root, goal_id: goalId, agent_id: agentId, + exclude_turn_instance_id: turnId, + }; + const before = await preflightPriorHostTurnCloseout(preflightRequest); + assert.equal(before.status, "none"); + // The next writer drops that tail before appending an unrelated row. + let later = {...original, event_id: "event-later", run_id: "later-turn", + details: {}} as Record; + await writeFile(eventsPath, `${JSON.stringify(later)}\n`); + const after = await preflightPriorHostTurnCloseout(preflightRequest); + assert.equal(after.status, "none"); + // A row that reached its newline is still a real owed closeout. + await appendFile(eventsPath, `${JSON.stringify(original)}\n`); + const committed = await preflightPriorHostTurnCloseout(preflightRequest); + assert.equal(committed.status, "candidate"); + assert.equal((committed.candidate as Record).prior_turn_instance_id, priorTurnId); + } finally {await rm(root, {recursive: true, force: true});} +}); + test("an unbound earlier Turn never reports a later Turn as its owed predecessor", async () => { const root = await fixture({guardUnbound: true}); try { From f42d8e979b0fe8270a4fad3d7e40e194a931dd44 Mon Sep 17 00:00:00 2001 From: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:26:20 +0800 Subject: [PATCH 8/8] Use committed receipts for tolerant settlement history Filter unterminated scheduler settlement rows through the existing committed rollout view in quota settlement readback and replanned history. Add TypeScript and real File/SQLite CLI regressions for interrupted tails while preserving damaged-row tolerance. Signed-off-by: ChengLong GU <88359731+gcl-coder@users.noreply.github.com> --- .../quota/settlement_readback.ts | 5 +- .../work_items/replan_history_settlement.ts | 10 ++- .../test_scheduler_heartbeat_receipt_fence.py | 77 +++++++++++++++++++ .../quota_settlement_readback.test.ts | 68 ++++++++++++++++ 4 files changed, 156 insertions(+), 4 deletions(-) diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index de65f65062..231f67965f 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -3,6 +3,7 @@ import { isAbsolute, join } from "node:path"; import { readReceiptLogSnapshot } from "../runtime/receipt_log_snapshot.ts"; import { + committedGoalRolloutEvents, heartbeatReceiptsFromEvents, readGoalRolloutEventSnapshot, strictGoalRolloutEvents, @@ -445,11 +446,13 @@ export async function readQuotaSettlementSnapshot( // Settlement-authority callers keep failing closed on the first damaged row. // The scheduler follow-up route only needs the receipts it can actually read, // so it opts into the same tolerant view the heartbeat reader already uses. + // Both views still ignore an unterminated row: the append owner will drop it + // before writing the next record. return indexSettlementSnapshot( runtimeRoot, goalId, tolerateMalformedHistory - ? snapshot?.events ?? [] + ? committedGoalRolloutEvents(snapshot) : strictGoalRolloutEvents(snapshot), runs, ); diff --git a/loopx/control_plane/work_items/replan_history_settlement.ts b/loopx/control_plane/work_items/replan_history_settlement.ts index 858afe483b..fa7286e871 100644 --- a/loopx/control_plane/work_items/replan_history_settlement.ts +++ b/loopx/control_plane/work_items/replan_history_settlement.ts @@ -7,7 +7,11 @@ import { QUOTA_SETTLEMENT_READBACK_REQUEST_SCHEMA, validateQuotaSettlementScope, } from "../quota/settlement_readback.ts"; -import { readGoalRolloutEventSnapshot, strictGoalRolloutEvents } from "../rollout_receipt_log.ts"; +import { + committedGoalRolloutEvents, + readGoalRolloutEventSnapshot, + strictGoalRolloutEvents, +} from "../rollout_receipt_log.ts"; import { parseQuotaAccountingOwner, withQuotaAccountingOwner, withBorrowedQuotaAccountingOwner, quotaOwnerOwnsProjection } from "../quota/source_admission.ts"; import { projectReplanHistory } from "./replan_history.ts"; @@ -39,9 +43,9 @@ export async function projectSettledReplanHistory(value: unknown): Promise event.event_kind === "quota_should_run" && event.goal_id === goalId && diff --git a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py index 485fda2195..51a37c010a 100644 --- a/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py +++ b/tests/control_plane/test_scheduler_heartbeat_receipt_fence.py @@ -972,6 +972,83 @@ def run(): assert result["scheduler_state_mutated"] is False +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize( + "command", ["scheduler-ack-current", "scheduler-fail-current"], +) +def test_real_unterminated_should_run_never_becomes_a_settlement_receipt( + tmp_path: Path, provider: str, command: str +) -> None: + registry, runtime_root, project = write_cli_fixture( + tmp_path / f"{provider}-{command}-real-unterminated-original", + scoped_agents=True, provider=provider, + ) + goal_id = "needs-operator" + code, guard = run_json_cli_result( + "quota", + "should-run", + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--app-automation-current-rrule", + "FREQ=MINUTELY;INTERVAL=3", + "--turn-instance-id", + "turn-1", + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + assert code == 0, guard + + # Start from the receipt the normal admission path just wrote. Keep its + # complete JSON bytes but remove only the writer's committing newline. + log_path = rollout_event_log_path(runtime_root, goal_id) + committed = log_path.read_bytes() + assert committed.endswith(b"\n") + log_path.write_bytes(committed[:-1]) + torn = log_path.read_bytes() + assert torn == committed[:-1] + + operation_args = ("--use-current-hint",) if command == "scheduler-ack-current" else () + + def run() -> tuple[int, dict[str, object]]: + return run_json_cli_result( + "quota", + command, + "--goal-id", + goal_id, + "--agent-id", + SCOPED_AGENT_ID, + "--trae_app", + "--turn-instance-id", + "turn-1", + "--execute", + *operation_args, + registry_path=registry, + runtime_root=runtime_root, + cwd=project, + ) + + code, result = run() + assert code == 1, result + assert result["status"] == "heartbeat_receipt_missing" + assert result["scheduler_state_mutated"] is False + assert log_path.read_bytes() == torn + + # The ordinary append drops the unfinished tail. The same admission + # receipt has not become a settlement fact merely because it vanished. + append_rollout_event( + log_path, + build_rollout_event(goal_id=goal_id, event_kind="todo_complete"), + ) + code, result = run() + assert code == 1, result + assert result["status"] == "heartbeat_receipt_missing" + assert result["scheduler_state_mutated"] is False + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize( "command", ["scheduler-ack-current", "scheduler-fail-current"], diff --git a/tests/control_plane_ts/quota_settlement_readback.test.ts b/tests/control_plane_ts/quota_settlement_readback.test.ts index 8b46fe0463..da5f0aae0f 100644 --- a/tests/control_plane_ts/quota_settlement_readback.test.ts +++ b/tests/control_plane_ts/quota_settlement_readback.test.ts @@ -814,6 +814,32 @@ test("only the tolerant readback route survives a damaged audit row", async () = } finally { await rm(root, {recursive: true, force: true}); } }); +test("tolerant settlement readback ignores an unterminated receipt", async () => { + const root = await fixture(); + try { + const path = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const committed = await readQuotaSettlement(request(root)); + assert.equal(committed.found, true); + const committedBytes = await readFile(path); + assert.ok(committedBytes.at(-1) === 0x0a); + await writeFile(path, committedBytes.subarray(0, -1)); + const strictUncommitted = await readQuotaSettlement(request(root)); + const tolerantUncommitted = await readQuotaSettlement(request(root, { + tolerate_malformed_history: true, + })); + assert.deepEqual(tolerantUncommitted, strictUncommitted, + "both reads ignore a receipt whose writer did not reach the newline"); + await appendFile(path, "\n"); + const strictCommitted = await readQuotaSettlement(request(root)); + const tolerantCommitted = await readQuotaSettlement(request(root, { + tolerate_malformed_history: true, + })); + assert.deepEqual(tolerantCommitted, strictCommitted); + assert.deepEqual(strictCommitted, committed, + "the ordinary append commit exposes the receipt again"); + } finally { await rm(root, {recursive: true, force: true}); } +}); + test("settlement progress requires both effects and exact receipts", async t => { const cases = [ { guard: false, state: "identity_required", next: "validation" }, @@ -2440,6 +2466,48 @@ test("agent cadence does not interpret another agent's malformed semantic receip } }); +test("tolerant replan history ignores an unterminated settlement receipt", async () => { + const { projectSettledReplanHistory } = await import( + "../../loopx/control_plane/work_items/replan_history_settlement.ts"); + const root = await fixture({writeback: true, spend: true, completion: true}); + const path = join(root, "goals", goalId, "rollout-event-log.jsonl"); + const request = { + schema_version: "replan_history_request_v0", operation: "periodic", + agent_id: agentId, monitor_agent_id: agentId, neutral_classifications: [], + stall_threshold: 2, periodic_threshold: 1, monitor_threshold: 6, + streak_threshold: 5, monitor_schema: "dead_monitor_repeat_v0", + todos: {monitors: [], advancements: [], resume: null}, + settlement_source: {runtime_root: root, goal_id: goalId, + tolerate_malformed_history: true}, + runs: [{agent_id: agentId, public_agent_id: agentId, + monitor_agent_id: agentId, turn_id: turnId, + classification: "state_refreshed", generated_at: "2026-01-01T00:00:00Z", + observed_at: 1, accepted_ack: false, progress: null, monitor: {}}], + }; + try { + const committed = await projectSettledReplanHistory(request); + assert.equal(requireJsonObject(committed.trigger, "trigger").run_count, 1); + const committedBytes = await readFile(path); + assert.ok(committedBytes.at(-1) === 0x0a); + await appendFile(path, Buffer.concat([ + Buffer.from('{"schema_version":"loopx_rollout_event_v0","event_kind":"quota_should_run","goal_id":"'), + Buffer.from(goalId), + Buffer.from('","agent_id":"unrelated","run_id":"later-turn","summary":"'), + Buffer.from([0xff]), + Buffer.from('"}\n'), + ])); + assert.deepEqual(await projectSettledReplanHistory(request), committed, + "a damaged unrelated record stays outside the committed receipt view"); + await writeFile(path, committedBytes.subarray(0, -1)); + const uncommitted = await projectSettledReplanHistory(request); + assert.equal(uncommitted.trigger, null, + "an unterminated spend receipt cannot qualify settlement history"); + await appendFile(path, "\n"); + assert.deepEqual(await projectSettledReplanHistory(request), committed, + "the append commit restores the same settlement history"); + } finally { await rm(root, {recursive: true, force: true}); } +}); + test("effective cadence skips settlement read without a matching should-run receipt", async () => { const { projectSettledReplanHistory } = await import( "../../loopx/control_plane/work_items/replan_history_settlement.ts");