diff --git a/loopx/cli_commands/quota_scheduler_followup.py b/loopx/cli_commands/quota_scheduler_followup.py index a8ebbe64a1..c8e52cf16a 100644 --- a/loopx/cli_commands/quota_scheduler_followup.py +++ b/loopx/cli_commands/quota_scheduler_followup.py @@ -228,6 +228,8 @@ def build_scheduler_followup_payload( failed_rrule=args.failed_rrule, observed_host_rrule=observed_rrule, failure_kind=args.failure_kind, + turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) return record_quota_scheduler_ack( status_payload, @@ -248,4 +250,6 @@ def build_scheduler_followup_payload( scheduler_execution_context=scheduler_context, operator_inbox_urgency_projector=operator_inbox_urgency_projector, before_decision=before_decision, + turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) diff --git a/loopx/control_plane/quota/scheduler_ack.py b/loopx/control_plane/quota/scheduler_ack.py index b9bf45a87d..06247738a2 100644 --- a/loopx/control_plane/quota/scheduler_ack.py +++ b/loopx/control_plane/quota/scheduler_ack.py @@ -4,6 +4,7 @@ import json import subprocess +from collections.abc import Mapping from pathlib import Path from ..effect_runtime import _node_executable @@ -184,14 +185,25 @@ def _run_native_followup( host_facts: dict[str, Any], use_current_hint: bool, reason_summary: str | None, + turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, ) -> dict[str, Any]: - """Decision-free compatibility transport for in-process Python callers.""" + """Decision-free compatibility transport for Python callers. + + A caller that already bound the follow-up to a Turn passes it here, so the + native transaction applies the same heartbeat-receipt fence (missing or + superseded) as the generated route. The exact GoalRef this caller resolved + travels with it, so the native successor scan compares receipts inside the + same instance instead of across every goal that shares the log. Callers + without a Turn keep the receipt-less compatibility request. + """ request = { "schema_version": _FOLLOWUP_REQUEST_SCHEMA, "runtime_root": str(runtime_root), - "turn_instance_id": None, - "require_heartbeat_receipt": False, + "turn_instance_id": turn_instance_id, + "goal_ref": dict(goal_ref) if goal_ref is not None else None, + "require_heartbeat_receipt": turn_instance_id is not None, "before": before, "host_facts": host_facts, "use_current_hint": use_current_hint, @@ -298,6 +310,8 @@ def record_quota_scheduler_ack_for_decision( generated_at: str | None = None, use_current_hint: bool = False, host_match_observed: bool = False, + turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, ) -> dict[str, Any]: state_key = _followup_state_key(before, surface=surface, state_key=state_key) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -367,6 +381,8 @@ def record_quota_scheduler_ack_for_decision( host_facts=facts, use_current_hint=use_current_hint, reason_summary=reason_summary, + turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) except ValueError as exc: return _failure_payload( @@ -396,6 +412,8 @@ def record_quota_scheduler_failure_for_decision( observed_host_rrule: str | None = None, failure_kind: str = "host_tool_failure", generated_at: str | None = None, + turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, ) -> dict[str, Any]: state_key = _followup_state_key(before, surface=surface, state_key=state_key) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -443,6 +461,8 @@ def record_quota_scheduler_failure_for_decision( host_facts=facts, use_current_hint=False, reason_summary=None, + turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) except ValueError as exc: return _failure_payload( diff --git a/loopx/control_plane/quota/source_admission.ts b/loopx/control_plane/quota/source_admission.ts index 47526e2a75..68999056fe 100644 --- a/loopx/control_plane/quota/source_admission.ts +++ b/loopx/control_plane/quota/source_admission.ts @@ -113,14 +113,56 @@ export function requireQuotaOwnerProjection( } } -export function quotaOwnerOwnsProjection( - owner: QuotaAccountingOwner, +/** + * The Goal-instance scope one persisted fact was written under. + * + * `alias` is the legacy projection: the fact carries no `goal_ref` at all. + * Anything else is the one Goal instance the writer presented, so two goals + * that share a display name stay separate even inside one shared log. + */ +export type QuotaProjectionScope = + | Readonly<{ kind: "alias" }> + | Readonly<{ kind: "exact_source"; goalRef: ExactGoalRef }>; + +/** + * Read one fact's scope, or null when its `goal_ref` is not a usable scope. + * + * Absent and unparseable refs are different facts: absent is the legacy + * projection every pre-instance writer produced, while a present-but-unusable + * ref belongs to no scope at all and must not be compared loosely. + */ +export function quotaProjectionScope( + value: unknown, +): QuotaProjectionScope | null { + if (value === undefined) return { kind: "alias" }; + const projected = parseExactGoalRef(value); + return projected.kind === "parsed" + ? { kind: "exact_source", goalRef: projected.value } + : null; +} + +/** + * One judge for "did this fact belong to that scope". + * + * Legacy facts only compare with legacy facts and an exact GoalRef only + * compares with that same instance, so a caller cannot widen a scope by + * asking a looser question than the writer answered. + */ +export function quotaScopeOwnsProjection( + scope: QuotaProjectionScope, value: unknown, ): boolean { - if (owner.kind === "alias") return value === undefined; + if (scope.kind === "alias") return value === undefined; const projected = parseExactGoalRef(value); return projected.kind === "parsed" - && sameGoalRef(projected.value, owner.goalRef); + && sameGoalRef(projected.value, scope.goalRef); +} + +export function quotaOwnerOwnsProjection( + owner: QuotaAccountingOwner, + value: unknown, +): boolean { + return quotaScopeOwnsProjection(owner, value); } function quotaSourceLock( diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index a02c8464cc..cdc5be1b91 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -11,6 +11,11 @@ import { relative, resolve, sep } from "node:path"; import type { JsonObject } from "./effect_program.ts"; import { EffectRuntimeRequestError } from "./effect_runtime_errors.ts"; +import { + quotaProjectionScope, + quotaScopeOwnsProjection, + type QuotaProjectionScope, +} from "./quota/source_admission.ts"; import { requireNonEmptyString } from "./runtime_decode.ts"; import { readReceiptLogSnapshot } from "./runtime/receipt_log_snapshot.ts"; @@ -118,6 +123,24 @@ export function goalHeartbeatReceiptsFromSnapshot( return heartbeatReceiptsFromEvents(snapshot.events, goalId, agentId); } +/** + * One receipt predicate, shared by discovery and by successor scans. + * + * A failed invocation without a real Turn identity is an audit: the host + * protocol keeps it visible in the log, but lets it neither grant host + * writeback nor expire the receipt of the Turn it follows. + */ +export function isHeartbeatReceiptEvent( + event: JsonObject, + goalId: string, + agentId?: string | null, +): boolean { + return event.event_kind === HEARTBEAT_RECEIPT_EVENT_KIND && + event.goal_id === goalId && + typeof event.run_id === "string" && event.run_id.trim().length > 0 && + (agentId === undefined || event.agent_id === agentId); +} + /** The heartbeat receipt filter over events a caller already read. */ export function heartbeatReceiptsFromEvents( events: readonly JsonObject[], @@ -125,12 +148,7 @@ export function heartbeatReceiptsFromEvents( agentId?: string | null, ): JsonObject[] { return events.filter((event) => - event.event_kind === HEARTBEAT_RECEIPT_EVENT_KIND && - event.goal_id === goalId && - // A failed invocation without a real Turn identity is an audit, not a - // receipt that can grant host writeback or supersede an admitted Turn. - typeof event.run_id === "string" && event.run_id.trim().length > 0 && - (agentId === undefined || event.agent_id === agentId) + isHeartbeatReceiptEvent(event, goalId, agentId) ); } @@ -159,7 +177,9 @@ export async function heartbeatReceiptStatus(params: { goalId: string; agentId: string; turnInstanceId: string; + goalRef?: unknown; }): Promise { + const requestedScope = requestedReceiptScope(params.goalId, params.goalRef); const snapshot = await readGoalRolloutEventSnapshot( params.runtimeRoot, params.goalId, @@ -170,28 +190,82 @@ export async function heartbeatReceiptStatus(params: { params.agentId, ); if (snapshot === null || receipts === null) return "missing"; - const firstReceipt = receipts.find( - (event) => event.run_id === params.turnInstanceId, + const firstReceiptMatches = receipts.filter((event) => + event.run_id === params.turnInstanceId && + (requestedScope === null || + quotaScopeOwnsProjection(requestedScope, event.goal_ref)) ); - const firstMatch = firstReceipt === undefined - ? -1 - : snapshot.events.indexOf(firstReceipt); + if (firstReceiptMatches.length === 0) return "missing"; + // A command generated before the origin scope travelled with its facts cannot + // say which instance its Turn belonged to. Two Goal instances may share one + // Turn id inside a log they both write, and taking whichever row landed first + // would scan the wrong instance's successors and let a stale writeback + // through, so the ambiguity is refused instead of guessed. The host re-reads + // a scoped hint and retries with the origin receipt's own scope. + if ( + requestedScope === null && + new Set(firstReceiptMatches.map((event) => receiptScopeKey(event.goal_ref))) + .size > 1 + ) { + throw new EffectRuntimeRequestError( + "heartbeat receipt Turn identity matches more than one Goal instance", + "heartbeat_receipt_ambiguous", + ); + } + const firstReceipt = firstReceiptMatches[0]; + const firstMatch = snapshot.events.indexOf(firstReceipt); if (firstMatch < 0) return "missing"; - return snapshot.events.slice(firstMatch + 1).some((event) => { - if ( - event.event_kind !== HEARTBEAT_RECEIPT_EVENT_KIND || - event.goal_id !== params.goalId || - event.agent_id !== params.agentId - ) return false; - if (typeof event.run_id === "string" && event.run_id.trim().length > 0) { - return event.run_id !== params.turnInstanceId; - } - const details = event.details; - return event.status === "normal_run" && - typeof details === "object" && details !== null && !Array.isArray(details) && - (details as JsonObject).ok === true && - (details as JsonObject).should_run === true; - }) + // The receipt itself states the scope its successor scan may use: a legacy + // receipt is only superseded by another legacy receipt, and an instance + // receipt only by that same Goal instance. A differently named Goal that + // happens to share this log therefore cannot expire an admitted Turn, and an + // unscopable `goal_ref` fails closed instead of authorizing a writeback. + const receiptScope = quotaProjectionScope(firstReceipt.goal_ref); + if (receiptScope === null) return "missing"; + // Only a real Turn receipt expires an admitted one. Identity-less quota rows + // are diagnostics: the protocol neither grants them host writeback nor lets + // them supersede the receipt they follow, so the successor scan reuses the + // discovery predicate rather than re-deciding what a receipt is. + return snapshot.events.slice(firstMatch + 1).some((event) => + isHeartbeatReceiptEvent(event, params.goalId, params.agentId) && + event.run_id !== params.turnInstanceId && + quotaScopeOwnsProjection(receiptScope, event.goal_ref) + ) ? "stale" : "fresh"; } + +/** One comparable key per scope, so ambiguity is decided by set size. */ +function receiptScopeKey(value: unknown): string { + const scope = quotaProjectionScope(value); + return scope === null ? "unscopable" : JSON.stringify(scope); +} + +/** + * Resolve the scope a caller asked the receipt fence to run in. + * + * Absent means "use the scope the matching receipt states", so the persisted + * receipt stays the only authority. A caller that does know its Goal instance + * passes it, and a malformed or foreign scope is refused rather than silently + * widening the comparison. + */ +function requestedReceiptScope( + goalId: string, + value: unknown, +): QuotaProjectionScope | null { + if (value === undefined || value === null) return null; + const scope = quotaProjectionScope(value); + if (scope === null || scope.kind === "alias") { + throw new EffectRuntimeRequestError( + "heartbeat receipt scope requires goal_id and goal_instance_id", + "heartbeat_receipt_scope_invalid", + ); + } + if (scope.goalRef.goalId.value !== goalId) { + throw new EffectRuntimeRequestError( + "heartbeat receipt scope does not belong to the requested goal", + "heartbeat_receipt_scope_mismatch", + ); + } + return scope; +} diff --git a/loopx/control_plane/scheduler/heartbeat_commit.ts b/loopx/control_plane/scheduler/heartbeat_commit.ts index 2017c07143..71ad1f41f6 100644 --- a/loopx/control_plane/scheduler/heartbeat_commit.ts +++ b/loopx/control_plane/scheduler/heartbeat_commit.ts @@ -171,6 +171,8 @@ export interface SchedulerHeartbeatCommitResult extends JsonObject { export interface SchedulerHeartbeatCommitOptions { readonly heartbeatReceiptTurnInstanceId?: string | null; + /** The Goal instance the caller bound the follow-up to, when it knows it. */ + readonly heartbeatReceiptGoalRef?: JsonObject | null; } function stableValue(value: unknown): unknown { @@ -853,6 +855,7 @@ export async function evaluateSchedulerHeartbeatCommit( goalId: request.scope.goalId, agentId: request.scope.agentId, turnInstanceId: receiptTurnInstanceId, + goalRef: options.heartbeatReceiptGoalRef, }); if (status !== "fresh") { return result( diff --git a/loopx/control_plane/scheduler/heartbeat_followup.ts b/loopx/control_plane/scheduler/heartbeat_followup.ts index d1047f0454..b98ed942c0 100644 --- a/loopx/control_plane/scheduler/heartbeat_followup.ts +++ b/loopx/control_plane/scheduler/heartbeat_followup.ts @@ -33,6 +33,8 @@ export interface SchedulerHeartbeatFollowupRequest extends JsonObject { schema_version: typeof SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA; runtime_root: string; turn_instance_id: string | null; + /** The Goal instance the Python transport resolved for this follow-up. */ + goal_ref: JsonObject | null; require_heartbeat_receipt: boolean; before: JsonObject; host_facts: JsonObject; @@ -95,6 +97,9 @@ function requestObject(value: unknown): SchedulerHeartbeatFollowupRequest { schema_version: SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA, runtime_root: runtimeRoot, turn_instance_id: turnInstanceId, + goal_ref: input.goal_ref === undefined || input.goal_ref === null + ? null + : requireJsonObject(input.goal_ref, "goal_ref"), require_heartbeat_receipt: requireReceipt, before: requireJsonObject(input.before ?? {}, "before"), host_facts: hostFacts, @@ -341,6 +346,7 @@ export async function evaluateSchedulerHeartbeatFollowup( goalId: String(facts.goal_id), agentId: String(facts.agent_id), turnInstanceId: String(request.turn_instance_id), + goalRef: request.goal_ref, }); if (status !== "fresh") return receiptFailure(request, status); } @@ -351,6 +357,9 @@ export async function evaluateSchedulerHeartbeatFollowup( heartbeatReceiptTurnInstanceId: request.require_heartbeat_receipt ? request.turn_instance_id : null, + heartbeatReceiptGoalRef: request.require_heartbeat_receipt + ? request.goal_ref + : null, }); const receiptStatus = commitReceiptFailureStatus(commit); if (receiptStatus !== null) return receiptFailure(request, receiptStatus); diff --git a/loopx/control_plane/scheduler/heartbeat_followup_cli.ts b/loopx/control_plane/scheduler/heartbeat_followup_cli.ts index d7c448debf..955ee7f0b0 100644 --- a/loopx/control_plane/scheduler/heartbeat_followup_cli.ts +++ b/loopx/control_plane/scheduler/heartbeat_followup_cli.ts @@ -4,7 +4,7 @@ import { effectRuntimeErrorPayload, EffectRuntimeRequestError, } from "../effect_runtime_errors.ts"; -import { requireJsonObject } from "../runtime_decode.ts"; +import { jsonObject, requireJsonObject } from "../runtime_decode.ts"; import { isSupportedNodeVersion } from "../runtime/node_version.ts"; import { evaluateSchedulerHeartbeatFollowup, @@ -214,6 +214,12 @@ function requestFromArgs(argv: string[]): { request: Record; fo const facts = requireJsonObject(hint.host_facts, "host_facts"); matchingText(facts, "goal_id", parsed.goal_id, "--goal-id"); matchingText(facts, "agent_id", parsed.agent_id, "--agent-id"); + // The origin receipt's scope travels with the facts it was generated for. + // Passing it into the transaction is what keeps the pre-flight check and the + // in-lock recheck on that receipt instead of resolving the bare Turn id again. + const goalRef = jsonObject(facts.goal_ref); + if (goalRef !== null) + matchingText(goalRef, "goal_id", parsed.goal_id, "--goal-id"); const expectedOperation = parsed.command === "scheduler-ack-current" ? "ack" : "host_failure"; @@ -291,6 +297,7 @@ function requestFromArgs(argv: string[]): { request: Record; fo runtime_root: parsed.runtime_root, turn_instance_id: parsed.turn_instance_id, require_heartbeat_receipt: true, + ...(goalRef === null ? {} : { goal_ref: goalRef }), before: hint.before ?? {}, use_current_hint: hint.use_current_hint ?? expectedOperation === "ack", reason_summary: values["--reason-summary"] ?? null, diff --git a/loopx/control_plane/scheduler/hint.ts b/loopx/control_plane/scheduler/hint.ts index f472872568..62a4c596ec 100644 --- a/loopx/control_plane/scheduler/hint.ts +++ b/loopx/control_plane/scheduler/hint.ts @@ -192,8 +192,12 @@ function appAutomation(decision: SchedulerBackoffTransitionResult, cadence: Cade function hostFacts(request: JsonObject, profile: HintProfile, decision: SchedulerBackoffTransitionResult, scope: JsonObject): JsonObject | null { const payload = request.payload as JsonObject, agent = jsonObject(payload.agent_identity)?.agent_id; if (!payload.goal_id || !agent) return null; + // The follow-up the host repeats verbatim must stay bound to the Goal instance + // whose Turn produced the receipt. Without it a native command can only + // resolve a Turn id, and two instances sharing this log would collide. + const goalRef = jsonObject(request.goal_ref); return {schema_version: "loopx_scheduler_heartbeat_host_facts_v0", goal_id: String(payload.goal_id), agent_id: String(agent), - ...scope, progression_index: decision.current_index, progression_minutes: scope.progression_minutes, + ...(goalRef === null ? {} : {goal_ref: goalRef}), ...scope, progression_index: decision.current_index, progression_minutes: scope.progression_minutes, expected_rrule: decision.current_rrule, cadence_class: profile.cadence_class, stale_tolerance_minutes: 2, generated_at: request.generated_at, ack_needed: ["apply_required", "host_match_ack_required"].includes(decision.host_transition), apply_needed: decision.host_transition === "apply_required"}; diff --git a/loopx/control_plane/scheduler/scheduler_hint.py b/loopx/control_plane/scheduler/scheduler_hint.py index 356ea03b99..0c53aca901 100644 --- a/loopx/control_plane/scheduler/scheduler_hint.py +++ b/loopx/control_plane/scheduler/scheduler_hint.py @@ -326,8 +326,13 @@ def build_scheduler_hint( for key in ("agent_todo_summary", "user_todo_summary") ] facts = _projection_facts(payload) + # The generated follow-up commands must stay bound to the Goal instance whose + # Turn produced the receipt, so the owner scope travels beside the facts + # rather than being re-resolved from a bare Turn id at execution time. + goal_ref = payload.get("goal_ref") params = { "payload": facts, + "goal_ref": dict(goal_ref) if isinstance(goal_ref, Mapping) else None, "identity_json": _identity_json(payload), "context": context.projection(), "arbitration": asdict(arbitration) if arbitration is not None else None, diff --git a/loopx/control_plane/scheduler/state.py b/loopx/control_plane/scheduler/state.py index c00c5a6d84..ec1c1d2b13 100644 --- a/loopx/control_plane/scheduler/state.py +++ b/loopx/control_plane/scheduler/state.py @@ -252,6 +252,7 @@ def _store_result( agent_id: str, surface: str, state_key: str, + migrate_legacy: bool | None = None, ) -> Mapping[str, Any]: params: dict[str, Any] = { "schema_version": SCHEDULER_STATE_STORE_REQUEST_SCHEMA, @@ -265,6 +266,8 @@ def _store_result( } if state is not None: params["state"] = state + if migrate_legacy is not None: + params["migrate_legacy"] = migrate_legacy try: result = effect_runtime_result(f"scheduler.state.{operation}", params) except EffectRuntimeRejected as exc: @@ -290,6 +293,11 @@ def load_scheduler_state( ) -> dict[str, Any] | None: if not agent_id: return None + # Python reads are projections (quota should-run hints and the scheduler + # follow-up hint rebuild). Legacy state stays read-only until the native + # commit holds every applicable authority fence and migrates it under the + # scheduler lock; a read here must not move or delete the legacy file + # before a receipt/authority fence can reject the caller. result = _store_result( "load", runtime_root, @@ -298,6 +306,7 @@ def load_scheduler_state( agent_id=agent_id, surface=surface, state_key=state_key, + migrate_legacy=False, ) state = result.get("state") if state is not None and not isinstance(state, dict): diff --git a/loopx/quota.py b/loopx/quota.py index 450115daaf..5ec04954ca 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1244,6 +1244,8 @@ def record_quota_scheduler_ack( | None = None, operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None = None, before_decision: Mapping[str, Any] | None = None, + turn_instance_id: str | None = None, + goal_ref: Mapping[str, Any] | None = None, ) -> dict[str, Any]: safe_goal_id = _validate_goal_id_path_segment(str(goal_id or "")) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -1278,6 +1280,8 @@ def record_quota_scheduler_ack( reason_summary=reason_summary, use_current_hint=use_current_hint, host_match_observed=host_match_observed, + turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) diff --git a/tests/control_plane/test_scheduler_hint_single_request.py b/tests/control_plane/test_scheduler_hint_single_request.py index 5ee1543d39..ed362b7324 100644 --- a/tests/control_plane/test_scheduler_hint_single_request.py +++ b/tests/control_plane/test_scheduler_hint_single_request.py @@ -176,3 +176,54 @@ def test_invalid_host_view_preserves_python_whitespace_rules(surface, view): assert hint["action"] == "repair_scheduler_execution_context" assert hint[view]["applicability"] == "blocked_invalid_context" assert ("codex_app" if view == "app_automation" else "app_automation") not in hint + + +def _followup_facts(cli_args: list[str]) -> dict: + import base64 + import json + import zlib + + encoded = "".join( + cli_args[index + 1] + for index, value in enumerate(cli_args[:-1]) + if value == "--scheduler-host-facts-chunk" + ) + padded = encoded + "=" * (-len(encoded) % 4) + return json.loads(zlib.decompress(base64.b64decode(padded)))["host_facts"] + + +def _ack_hint_facts(payload: dict) -> dict: + arbitration = SchedulerArbitration( + SchedulerDisposition("active_work"), "active_work", "normal_run" + ) + hint = scheduler_hint.build_scheduler_hint( + payload, + arbitration=arbitration, + scheduler_execution_context=scheduler_execution_context_for_runtime_profile( + "codex_app_heartbeat" + ), + ) + return _followup_facts(hint["app_automation"]["ack_hint"]["cli_args"]) + + +def _hint_payload() -> dict: + return { + "goal_id": "hint-fixture", + "agent_identity": {"agent_id": "agent-fixture"}, + "heartbeat_recommendation": {"recommended_mode": "normal_run"}, + "recommended_action": "Perform the bounded work.", + "effective_action": "normal_run", + "interaction_contract": {"mode": "normal_run"}, + } + + +def test_generated_ack_hint_carries_the_origin_goal_instance(): + goal_ref = {"goal_id": "hint-fixture", "goal_instance_id": "ginst_" + "a" * 32} + facts = _ack_hint_facts({**_hint_payload(), "goal_ref": goal_ref}) + # A shared log can hold another instance's row under the same Turn id, so the + # generated command must not resolve its receipt from the Turn id alone. + assert facts["goal_ref"] == goal_ref + + +def test_generated_ack_hint_omits_scope_for_a_legacy_owner(): + assert "goal_ref" not in _ack_hint_facts(_hint_payload()) diff --git a/tests/control_plane/test_scheduler_stale_receipt_gate.py b/tests/control_plane/test_scheduler_stale_receipt_gate.py new file mode 100644 index 0000000000..e453d9e34f --- /dev/null +++ b/tests/control_plane/test_scheduler_stale_receipt_gate.py @@ -0,0 +1,461 @@ +"""The Python follow-up route applies the same receipt fence as the native route. + +A Turn id binds the follow-up to that Turn's heartbeat receipt. The native +transaction rejects a receipt that is missing *or* superseded; the compatibility +route used to check only for absence, so an ACK for a Turn whose receipt a later +`quota should-run` had superseded still rewrote scheduler state. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +from examples.control_plane.quota_plan_fixtures import ( + SCOPED_AGENT_ID, + write_cli_fixture, +) +from loopx.control_plane.goals.goal_ref_validation import exact_goal_ref +from loopx.control_plane.scheduler.state import ( + APP_AUTOMATION_STATEFUL_BACKOFF_STATE_KEY as APP_KEY, + build_scheduler_state, + load_scheduler_state, + scheduler_state_path, +) +from loopx.control_plane.testing.canary_harness import run_json_cli_result +from loopx.rollout_event_log import ( + append_rollout_event, + build_rollout_event, + rollout_event_log_path, +) + +GOAL = "needs-operator" +TURN_1 = "stale-receipt-turn-1" +TURN_2 = "stale-receipt-turn-2" +# A different Goal that shares this display name and Agent, so it writes into +# the same rollout-event log. +OTHER_INSTANCE = "ginst_" + "b" * 32 +PROVIDERS = ["file", "sqlite"] + + +def _fixture( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, provider: str = "file" +) -> tuple[Path, Path, Path]: + registry, runtime, project = write_cli_fixture( + tmp_path / "fixture", scoped_agents=True, provider=provider + ) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "codex-home")) + monkeypatch.setenv("CODEX_THREAD_ID", "scheduler-fixture") + return registry, runtime, project + + +def _should_run( + registry: Path, runtime: Path, project: Path, turn: str +) -> dict: + code, result = run_json_cli_result( + "quota", "should-run", "--goal-id", GOAL, + "--agent-id", SCOPED_AGENT_ID, "--codex-app", + "--turn-instance-id", turn, + registry_path=registry, runtime_root=runtime, cwd=project, + ) + assert code == 0, result + return result + + +def _ack( + registry: Path, runtime: Path, project: Path, turn: str, *, execute: bool = True +) -> tuple[int, dict]: + return run_json_cli_result( + "quota", "scheduler-ack-current", "--goal-id", GOAL, + "--agent-id", SCOPED_AGENT_ID, "--codex-app", + "--turn-instance-id", turn, *(["--execute"] if execute else []), + registry_path=registry, runtime_root=runtime, cwd=project, + ) + + +def _fail( + registry: Path, + runtime: Path, + project: Path, + turn: str, + *, + failed_rrule: str, + execute: bool = True, +) -> tuple[int, dict]: + return run_json_cli_result( + "quota", "scheduler-fail-current", "--goal-id", GOAL, + "--agent-id", SCOPED_AGENT_ID, "--codex-app", + "--turn-instance-id", turn, + "--failed-rrule", failed_rrule, + *(["--execute"] if execute else []), + registry_path=registry, runtime_root=runtime, cwd=project, + ) + + +def _supercede( + runtime: Path, turn: str, goal_ref: dict[str, str] | None = None +) -> None: + """Append a later receipt for ``turn`` on the same goal and agent.""" + + append_rollout_event( + rollout_event_log_path(runtime, GOAL), + build_rollout_event( + goal_id=GOAL, + event_kind="quota_should_run", + goal_ref=goal_ref, + agent_id=SCOPED_AGENT_ID, + run_id=turn, + status="normal_run", + details={"ok": True, "should_run": True}, + ), + ) + + +def test_superseded_receipt_cannot_commit_a_scheduler_ack( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + _supercede(runtime, TURN_2) + + code, result = _ack(registry, runtime, project, TURN_1) + + assert code == 1, result + assert result["ok"] is False + assert result["status"] == "heartbeat_receipt_stale" + assert result["state"] == "blocked_receipt" + assert result["scheduler_state_mutated"] is False + assert not scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ).exists() + + +def test_fresh_receipt_still_commits_the_ack( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + + code, result = _ack(registry, runtime, project, TURN_1) + + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + state = load_scheduler_state( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ) + assert state is not None + assert state["last_applied_rrule"] + + +def _legacy_segment(value: str) -> str: + """Mirror the legacy (unbounded) scheduler path segment normalization.""" + + safe = re.sub(r"[^0-9A-Za-z_.-]+", "-", value.strip()) + return re.sub(r"^[-._]+|[-._]+$", "", safe) or "default" + + +def _legacy_state_path( + runtime: Path, *, goal_id: str, agent_id: str, surface: str, state_key: str +) -> Path: + return ( + runtime + / "goals" + / _legacy_segment(goal_id) + / "scheduler-state" + / _legacy_segment(agent_id) + / _legacy_segment(surface) + / f"{hashlib.sha256(state_key.encode('utf-8')).hexdigest()[:16]}.json" + ) + + +@pytest.mark.parametrize("execute", [True, False]) +def test_stale_receipt_rejection_does_not_migrate_legacy_scheduler_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, execute: bool +) -> None: + """The rejected follow-up rebuilds a hint, but the hint read is read-only. + + Migration belongs to the native commit that holds the scheduler lock after + every receipt fence passes; a blocked preview or execute must leave the + legacy layout byte-identical and create no canonical state file. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + legacy_path = _legacy_state_path( + runtime, + goal_id=GOAL, + agent_id=SCOPED_AGENT_ID, + surface="codex_app", + state_key=APP_KEY, + ) + legacy_path.parent.mkdir(parents=True, exist_ok=True) + legacy_state = build_scheduler_state( + goal_id=GOAL, + agent_id=SCOPED_AGENT_ID, + state_key=APP_KEY, + reset_token="legacy-reset", + identity_signature="legacy-identity", + progression_index=1, + progression_minutes=[30, 60], + last_applied_rrule="FREQ=MINUTELY;INTERVAL=30", + updated_at="2026-01-01T12:00:00Z", + ) + legacy_bytes = json.dumps(legacy_state, sort_keys=True).encode("utf-8") + legacy_path.write_bytes(legacy_bytes) + canonical_path = scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ) + assert legacy_path != canonical_path + assert not canonical_path.exists() + + _supercede(runtime, TURN_2) + code, result = _ack(registry, runtime, project, TURN_1, execute=execute) + + assert code == 1, result + assert result["status"] == "heartbeat_receipt_stale" + assert result["state"] == "blocked_receipt" + assert result["scheduler_state_mutated"] is False + assert legacy_path.read_bytes() == legacy_bytes + assert not canonical_path.exists() + + +def _diagnostic_row(runtime: Path) -> None: + """Append one identity-less successful quota row: a host-protocol audit.""" + + append_rollout_event( + rollout_event_log_path(runtime, GOAL), + build_rollout_event( + goal_id=GOAL, + event_kind="quota_should_run", + agent_id=SCOPED_AGENT_ID, + status="normal_run", + details={"ok": True, "should_run": True}, + ), + ) + + +def test_identity_less_diagnostic_row_cannot_expire_the_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A diagnostic row after the receipt is audit, not a successor. + + The host protocol requires a nonempty Turn identity for a heartbeat + receipt; an identity-less quota row must neither grant host writeback nor + expire the admitted Turn's receipt, on either backend. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + _diagnostic_row(runtime) + + code, result = _ack(registry, runtime, project, TURN_1) + + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + state = load_scheduler_state( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ) + assert state is not None + assert state["last_applied_rrule"] + + +def test_a_real_successor_receipt_still_expires_the_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Strategy: the relaxation above must not weaken real successors.""" + + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + _diagnostic_row(runtime) + _supercede(runtime, TURN_2) + + code, result = _ack(registry, runtime, project, TURN_1) + + assert code == 1, result + assert result["status"] == "heartbeat_receipt_stale" + assert result["scheduler_state_mutated"] is False + assert not scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ).exists() + + +def _writeback( + operation: str, + registry: Path, + runtime: Path, + project: Path, + turn: str, + hint: dict, +) -> tuple[int, dict]: + """Run the ACK or the host-failure writeback this Turn is bound to.""" + + if operation == "ack": + return _ack(registry, runtime, project, turn) + return _fail( + registry, + runtime, + project, + turn, + failed_rrule=hint["scheduler_hint"]["app_automation"]["recommended_rrule"], + ) + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("operation", ["ack", "failure"]) +def test_same_scope_successor_receipt_still_expires_the_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation: str +) -> None: + """Control: a real successor in the same scope still blocks both writebacks. + + Nothing in the instance-scoping relaxation may let an older Turn report + over a newer one that genuinely follows it. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + _supercede(runtime, TURN_2) + + code, result = _writeback(operation, registry, runtime, project, TURN_1, hint) + + assert code == 1, result + assert result["status"] == "heartbeat_receipt_stale" + assert result["state"] == "blocked_receipt" + assert result["scheduler_state_mutated"] is False + assert not scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ).exists() + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("operation", ["ack", "failure"]) +def test_another_goal_instance_receipt_cannot_expire_the_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation: str +) -> None: + """A receipt from another Goal instance is not this Turn's successor. + + Both Goals share the display name and Agent, so they append to one log. + Only receipts of the admitted Turn's own scope may expire its receipt. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + _supercede(runtime, TURN_2, exact_goal_ref(GOAL, OTHER_INSTANCE)) + + code, result = _writeback(operation, registry, runtime, project, TURN_1, hint) + + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + state = load_scheduler_state( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ) + assert state is not None + + +def _prepend_receipt( + runtime: Path, turn: str, goal_ref: dict[str, str] +) -> None: + """Place another Goal instance's row before the admitted Turn's receipt.""" + + path = rollout_event_log_path(runtime, GOAL) + row = json.dumps( + build_rollout_event( + goal_id=GOAL, + event_kind="quota_should_run", + goal_ref=goal_ref, + agent_id=SCOPED_AGENT_ID, + run_id=turn, + status="normal_run", + details={"ok": True, "should_run": True}, + ), + sort_keys=True, + ) + path.write_bytes(row.encode("utf-8") + b"\n" + path.read_bytes()) + + +def _run_generated( + hint: dict, operation: str, project: Path +) -> tuple[int, dict]: + """Run the native follow-up command the hint generated for this Turn.""" + + packet = hint["scheduler_hint"]["app_automation"][f"{operation}_hint"] + result = subprocess.run( + [sys.executable, "-m", "loopx.cli", "--format", "json", *packet["cli_args"]], + cwd=project, + check=False, + env={**os.environ, "LOOPX_CONTEXT_DELIVERY": "off"}, + text=True, + encoding="utf-8", + errors="replace", + capture_output=True, + ) + return result.returncode, json.loads(result.stdout) + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("operation", ["ack", "failure"]) +def test_generated_native_writeback_refuses_a_shared_turn_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation: str +) -> None: + """The generated command cannot resolve its receipt from the Turn id alone. + + Two Goal instances may write one Turn id into this log. When another + instance's row lands first, picking it as the origin would scan the wrong + instance's successors and let the admitted Turn's stale writeback through. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + _prepend_receipt(runtime, TURN_1, exact_goal_ref(GOAL, OTHER_INSTANCE)) + _supercede(runtime, TURN_2) + + code, result = _run_generated(hint, operation, project) + + assert code == 1, result + assert result["status"] == "error" + assert result["error"]["code"] == "heartbeat_receipt_ambiguous" + assert not scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ).exists() + + +@pytest.mark.parametrize("provider", PROVIDERS) +def test_generated_native_failure_writeback_commits_without_a_shared_turn_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + """Control: the same generated command still writes when nothing collides.""" + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + + code, result = _run_generated(hint, "failure", project) + + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + + +@pytest.mark.parametrize("provider", PROVIDERS) +def test_generated_native_ack_only_refuses_on_a_shared_turn_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + """The ambiguity is decided by the log, not by the writeback's own gates.""" + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + + code, result = _run_generated(hint, "ack", project) + + assert code == 1, result + assert result["reason_code"] == "rrule_mismatch" + assert "error" not in result + diff --git a/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts b/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts index a636ce3df3..2f1c0acb48 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts @@ -68,7 +68,11 @@ function receiptPath(runtimeRoot: string): string { return join(runtimeRoot, "goals", scope.goal_id, "rollout-event-log.jsonl"); } -async function appendReceipt(runtimeRoot: string, turnId: string): Promise { +async function appendReceipt( + runtimeRoot: string, + turnId: string, + goalRef?: Record, +): Promise { const path = receiptPath(runtimeRoot); await mkdir(dirname(path), { recursive: true }); await writeFile(path, `${JSON.stringify({ @@ -77,9 +81,18 @@ async function appendReceipt(runtimeRoot: string, turnId: string): Promise goal_id: scope.goal_id, agent_id: scope.agent_id, run_id: turnId, + ...(goalRef === undefined ? {} : { goal_ref: goalRef }), })}\n`, { encoding: "utf8", flag: "a" }); } +const instanceScope = (suffix: string): Record => ({ + goal_id: scope.goal_id, + goal_instance_id: `ginst_${suffix.repeat(32)}`, +}); + +const ownInstance = instanceScope("a"); +const otherInstance = instanceScope("b"); + function request( runtimeRoot: string, extra: Record = {}, @@ -325,7 +338,7 @@ test("another Agent receipt cannot supersede ACK or host-failure writeback", asy assert.equal(failed.failure_count, 1); }); -test("a successful identity-less quota guard supersedes an older host receipt", async (t) => { +test("a successful identity-less quota guard cannot supersede an older host receipt", async (t) => { const runtimeRoot = await tempRuntime(t); await appendReceipt(runtimeRoot, "turn-followup-1"); await writeFile(receiptPath(runtimeRoot), `${JSON.stringify({ @@ -337,6 +350,14 @@ test("a successful identity-less quota guard supersedes an older host receipt", details: { ok: true, should_run: true }, })}\n`, { flag: "a" }); + // The host protocol keeps identity-less quota rows as diagnostic audits: they + // never expire the admitted Turn's receipt, so this ACK still commits. + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); + assert.equal(acknowledged.ok, true); + assert.equal((acknowledged.scheduler_commit as Record).status, "written"); + + // A real newer Turn still supersedes the older receipt, diagnostics or not. + await appendReceipt(runtimeRoot, "turn-followup-2"); const stale = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); assert.equal(stale.error_code, "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE"); assert.equal(stale.scheduler_state_mutated, false); @@ -381,3 +402,93 @@ test("a quota audit cannot manufacture a missing heartbeat receipt", async (t) = }); await assert.rejects(readFile(statePath, "utf8"), { code: "ENOENT" }); }); + +test("a receipt from another Goal instance cannot supersede a legacy receipt", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + // Same goal id and Agent, different Goal instance: the shared log must not + // let that instance expire the admitted Turn's receipt. + await appendReceipt(runtimeRoot, "turn-followup-other", otherInstance); + + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); + assert.equal(acknowledged.ok, true); + assert.equal((acknowledged.scheduler_commit as Record).status, "written"); + + const base = request(runtimeRoot); + const failed = await evaluateSchedulerHeartbeatFollowup({ + ...base, + host_facts: { + ...(base.host_facts as Record), + operation: "host_failure", + failure_kind: "timeout", + observed_host_rrule: "FREQ=MINUTELY;INTERVAL=3", + }, + }); + assert.equal(failed.ok, true); + assert.equal(failed.failure_count, 1); +}); + +test("a scoped follow-up still rejects a successor of its own Goal instance", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1", ownInstance); + await appendReceipt(runtimeRoot, "turn-followup-2", ownInstance); + + const stale = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: ownInstance, + })); + assert.equal(stale.error_code, "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE"); + assert.equal(stale.scheduler_state_mutated, false); +}); + +test("a scoped follow-up ignores a successor from another Goal instance", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1", ownInstance); + await appendReceipt(runtimeRoot, "turn-followup-other", otherInstance); + + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: ownInstance, + })); + assert.equal(acknowledged.ok, true); + assert.equal((acknowledged.scheduler_commit as Record).status, "written"); +}); + +test("a scoped follow-up ignores a legacy successor in the same log", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1", ownInstance); + await appendReceipt(runtimeRoot, "turn-followup-2"); + + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: ownInstance, + })); + assert.equal(acknowledged.ok, true); + assert.equal((acknowledged.scheduler_commit as Record).status, "written"); +}); + +test("a scoped follow-up does not adopt a legacy receipt", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + + const missing = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: ownInstance, + })); + assert.equal(missing.error_code, "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_MISSING"); + assert.equal(missing.scheduler_state_mutated, false); +}); + +test("a follow-up scope without a Goal instance is refused", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1", ownInstance); + + await assert.rejects( + evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: { goal_id: scope.goal_id }, + })), + { code: "heartbeat_receipt_scope_invalid" }, + ); + await assert.rejects( + evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: { ...ownInstance, goal_id: "another-goal" }, + })), + { code: "heartbeat_receipt_scope_mismatch" }, + ); +}); diff --git a/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts b/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts index baa30efd6c..d63a67cb60 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts @@ -367,3 +367,95 @@ test("native host-followup transport rejects inflated facts beyond its bound", ( "invalid_scheduler_host_facts", ); }); + +const ownInstance = { goal_id: goalId, goal_instance_id: `ginst_${"a".repeat(32)}` }; +const otherInstance = { goal_id: goalId, goal_instance_id: `ginst_${"b".repeat(32)}` }; + +async function writeReceiptRows( + runtimeRoot: string, + rows: readonly { turn: string; goalRef?: Record }[], +): Promise { + const path = join(runtimeRoot, "goals", goalId, "rollout-event-log.jsonl"); + await mkdir(dirname(path), { recursive: true }); + const lines = rows.map((row) => JSON.stringify({ + schema_version: "loopx_rollout_event_v0", + event_kind: "quota_should_run", + goal_id: goalId, + agent_id: agentId, + run_id: row.turn, + ...(row.goalRef === undefined ? {} : { goal_ref: row.goalRef }), + })); + await writeFile(path, `${lines.join("\n")}\n`, "utf8"); +} + +function scopedHint(): Record { + const hint = hintPayload(); + hint.host_facts = { ...(hint.host_facts as Record), goal_ref: ownInstance }; + return hint; +} + +// A shared log can hold another Goal instance's row under the same Turn id. A +// command that only resolves that id would pick whichever row landed first and +// scan the wrong instance's successors, so the generated facts carry the origin +// scope and the command keeps it. +test("native follow-up keeps the receipt scope its facts bound", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceiptRows(runtimeRoot, [ + { turn: turnId, goalRef: otherInstance }, + { turn: turnId, goalRef: ownInstance }, + { turn: "turn-followup-cli-next", goalRef: ownInstance }, + ]); + + const scoped = runCli(argv(runtimeRoot, "json", scopedHint())); + const scopedResult = JSON.parse(scoped.stdout) as Record; + assert.equal(scoped.status, 1, scoped.stderr); + assert.equal( + scopedResult.error_code, + "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE", + ); + assert.equal(scopedResult.scheduler_state_mutated, false); +}); + +// The same log with no successor is legal for the instance that owns the Turn. +test("a bound scope accepts its own fresh receipt beside another instance", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceiptRows(runtimeRoot, [ + { turn: turnId, goalRef: otherInstance }, + { turn: turnId, goalRef: ownInstance }, + ]); + + const scoped = runCli(argv(runtimeRoot, "json", scopedHint())); + assert.equal(scoped.status, 0, scoped.stderr); + assert.equal( + ((JSON.parse(scoped.stdout) as Record).scheduler_commit as Record).status, + "written", + ); +}); + +// A historical command generated before the scope travelled in its facts cannot +// tell the two rows apart; refusing is the only answer that cannot write back to +// the wrong instance. +test("an unscoped command refuses a Turn id shared by two Goal instances", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceiptRows(runtimeRoot, [ + { turn: turnId, goalRef: otherInstance }, + { turn: turnId, goalRef: ownInstance }, + ]); + + const unscoped = runCli(argv(runtimeRoot)); + const result = JSON.parse(unscoped.stdout) as Record; + assert.equal(unscoped.status, 1, unscoped.stderr); + assert.equal(result.status, "error"); + assert.equal( + (result.error as Record).code, + "heartbeat_receipt_ambiguous", + ); +}); + +test("an unscoped legacy log still accepts its own single Turn receipt", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceiptRows(runtimeRoot, [{ turn: turnId }]); + + const unscoped = runCli(argv(runtimeRoot)); + assert.equal(unscoped.status, 0, unscoped.stderr); +});