From a4504ebae072075afb0f5b649925def92becacea Mon Sep 17 00:00:00 2001 From: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:47:33 +0800 Subject: [PATCH 1/5] fix(scheduler): apply the heartbeat receipt fence to Python follow-ups A Turn id already bound the follow-up to one heartbeat receipt, but the compatibility transport sent turn_instance_id=null and require_heartbeat_receipt=false to the native transaction. The Python route therefore rejected a missing receipt and accepted a superseded one, while the generated route rejects both. Forward the Turn id when the caller has one, so the native transaction owns the missing/superseded decision for both routes. Callers without a Turn (the in-process API) keep the receipt-less compatibility request. Refs #8605 Signed-off-by: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> --- .../cli_commands/quota_scheduler_followup.py | 2 + loopx/control_plane/quota/scheduler_ack.py | 17 ++- loopx/quota.py | 2 + .../test_scheduler_stale_receipt_gate.py | 121 ++++++++++++++++++ 4 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 tests/control_plane/test_scheduler_stale_receipt_gate.py diff --git a/loopx/cli_commands/quota_scheduler_followup.py b/loopx/cli_commands/quota_scheduler_followup.py index a8ebbe64a1..8bbe506bba 100644 --- a/loopx/cli_commands/quota_scheduler_followup.py +++ b/loopx/cli_commands/quota_scheduler_followup.py @@ -228,6 +228,7 @@ def build_scheduler_followup_payload( failed_rrule=args.failed_rrule, observed_host_rrule=observed_rrule, failure_kind=args.failure_kind, + turn_instance_id=turn_instance_id, ) return record_quota_scheduler_ack( status_payload, @@ -248,4 +249,5 @@ def build_scheduler_followup_payload( scheduler_execution_context=scheduler_context, operator_inbox_urgency_projector=operator_inbox_urgency_projector, before_decision=before_decision, + turn_instance_id=turn_instance_id, ) diff --git a/loopx/control_plane/quota/scheduler_ack.py b/loopx/control_plane/quota/scheduler_ack.py index b9bf45a87d..2fc8939725 100644 --- a/loopx/control_plane/quota/scheduler_ack.py +++ b/loopx/control_plane/quota/scheduler_ack.py @@ -184,14 +184,21 @@ def _run_native_followup( host_facts: dict[str, Any], use_current_hint: bool, reason_summary: str | None, + turn_instance_id: str | None = None, ) -> dict[str, Any]: - """Decision-free compatibility transport for in-process Python callers.""" + """Decision-free compatibility transport for Python callers. + + A caller that already bound the follow-up to a Turn passes it here, so the + native transaction applies the same heartbeat-receipt fence (missing or + superseded) as the generated route. Callers without a Turn keep the + receipt-less compatibility request. + """ request = { "schema_version": _FOLLOWUP_REQUEST_SCHEMA, "runtime_root": str(runtime_root), - "turn_instance_id": None, - "require_heartbeat_receipt": False, + "turn_instance_id": turn_instance_id, + "require_heartbeat_receipt": turn_instance_id is not None, "before": before, "host_facts": host_facts, "use_current_hint": use_current_hint, @@ -298,6 +305,7 @@ def record_quota_scheduler_ack_for_decision( generated_at: str | None = None, use_current_hint: bool = False, host_match_observed: bool = False, + turn_instance_id: str | None = None, ) -> dict[str, Any]: state_key = _followup_state_key(before, surface=surface, state_key=state_key) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -367,6 +375,7 @@ def record_quota_scheduler_ack_for_decision( host_facts=facts, use_current_hint=use_current_hint, reason_summary=reason_summary, + turn_instance_id=turn_instance_id, ) except ValueError as exc: return _failure_payload( @@ -396,6 +405,7 @@ def record_quota_scheduler_failure_for_decision( observed_host_rrule: str | None = None, failure_kind: str = "host_tool_failure", generated_at: str | None = None, + turn_instance_id: str | None = None, ) -> dict[str, Any]: state_key = _followup_state_key(before, surface=surface, state_key=state_key) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -443,6 +453,7 @@ def record_quota_scheduler_failure_for_decision( host_facts=facts, use_current_hint=False, reason_summary=None, + turn_instance_id=turn_instance_id, ) except ValueError as exc: return _failure_payload( diff --git a/loopx/quota.py b/loopx/quota.py index 450115daaf..1a983d0ded 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1244,6 +1244,7 @@ def record_quota_scheduler_ack( | None = None, operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None = None, before_decision: Mapping[str, Any] | None = None, + turn_instance_id: str | None = None, ) -> dict[str, Any]: safe_goal_id = _validate_goal_id_path_segment(str(goal_id or "")) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -1278,6 +1279,7 @@ def record_quota_scheduler_ack( reason_summary=reason_summary, use_current_hint=use_current_hint, host_match_observed=host_match_observed, + turn_instance_id=turn_instance_id, ) diff --git a/tests/control_plane/test_scheduler_stale_receipt_gate.py b/tests/control_plane/test_scheduler_stale_receipt_gate.py new file mode 100644 index 0000000000..4196b3f940 --- /dev/null +++ b/tests/control_plane/test_scheduler_stale_receipt_gate.py @@ -0,0 +1,121 @@ +"""The Python follow-up route applies the same receipt fence as the native route. + +A Turn id binds the follow-up to that Turn's heartbeat receipt. The native +transaction rejects a receipt that is missing *or* superseded; the compatibility +route used to check only for absence, so an ACK for a Turn whose receipt a later +`quota should-run` had superseded still rewrote scheduler state. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from examples.control_plane.quota_plan_fixtures import ( + SCOPED_AGENT_ID, + write_cli_fixture, +) +from loopx.control_plane.scheduler.state import ( + APP_AUTOMATION_STATEFUL_BACKOFF_STATE_KEY as APP_KEY, + load_scheduler_state, + scheduler_state_path, +) +from loopx.control_plane.testing.canary_harness import run_json_cli_result +from loopx.rollout_event_log import ( + append_rollout_event, + build_rollout_event, + rollout_event_log_path, +) + +GOAL = "needs-operator" +TURN_1 = "stale-receipt-turn-1" +TURN_2 = "stale-receipt-turn-2" + + +def _fixture( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[Path, Path, Path]: + registry, runtime, project = write_cli_fixture( + tmp_path / "fixture", scoped_agents=True, provider="file" + ) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "codex-home")) + monkeypatch.setenv("CODEX_THREAD_ID", "scheduler-fixture") + return registry, runtime, project + + +def _should_run( + registry: Path, runtime: Path, project: Path, turn: str +) -> dict: + code, result = run_json_cli_result( + "quota", "should-run", "--goal-id", GOAL, + "--agent-id", SCOPED_AGENT_ID, "--codex-app", + "--turn-instance-id", turn, + registry_path=registry, runtime_root=runtime, cwd=project, + ) + assert code == 0, result + return result + + +def _ack( + registry: Path, runtime: Path, project: Path, turn: str +) -> tuple[int, dict]: + return run_json_cli_result( + "quota", "scheduler-ack-current", "--goal-id", GOAL, + "--agent-id", SCOPED_AGENT_ID, "--codex-app", + "--turn-instance-id", turn, "--execute", + registry_path=registry, runtime_root=runtime, cwd=project, + ) + + +def _supercede(runtime: Path, turn: str) -> None: + """Append a later receipt for ``turn`` on the same goal and agent.""" + + append_rollout_event( + rollout_event_log_path(runtime, GOAL), + build_rollout_event( + goal_id=GOAL, + event_kind="quota_should_run", + agent_id=SCOPED_AGENT_ID, + run_id=turn, + status="normal_run", + details={"ok": True, "should_run": True}, + ), + ) + + +def test_superseded_receipt_cannot_commit_a_scheduler_ack( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + _supercede(runtime, TURN_2) + + code, result = _ack(registry, runtime, project, TURN_1) + + assert code == 1, result + assert result["ok"] is False + assert result["status"] == "heartbeat_receipt_stale" + assert result["state"] == "blocked_receipt" + assert result["scheduler_state_mutated"] is False + assert not scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ).exists() + + +def test_fresh_receipt_still_commits_the_ack( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + + code, result = _ack(registry, runtime, project, TURN_1) + + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + state = load_scheduler_state( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ) + assert state is not None + assert state["last_applied_rrule"] From 7eba7658d871240079eedb257ea793376d9a2ede Mon Sep 17 00:00:00 2001 From: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> Date: Sat, 14 Nov 2026 12:09:39 +0800 Subject: [PATCH 2/5] fix(scheduler): keep Python scheduler reads from migrating legacy state A Python read of scheduler state is a projection: the follow-up hint rebuild reads the stored state before the native transaction applies the heartbeat receipt fence. That read used the store's default, which migrates a legacy scheduler-state file into the canonical hashed path, so a follow-up that is then rejected as `heartbeat_receipt_stale` had already deleted the legacy file and written a new one. A blocked preview or execute must leave the legacy layout untouched; migration belongs to the native commit that holds the scheduler lock after every fence passes, which already asks for `migrate_legacy: false`. `load_scheduler_state` now reads legacy state without migrating it. Readers see the same value; only the write under the lock moves the file. Refs #8605 Signed-off-by: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> --- loopx/control_plane/scheduler/state.py | 9 +++ .../test_scheduler_stale_receipt_gate.py | 80 ++++++++++++++++++- 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/loopx/control_plane/scheduler/state.py b/loopx/control_plane/scheduler/state.py index c00c5a6d84..ec1c1d2b13 100644 --- a/loopx/control_plane/scheduler/state.py +++ b/loopx/control_plane/scheduler/state.py @@ -252,6 +252,7 @@ def _store_result( agent_id: str, surface: str, state_key: str, + migrate_legacy: bool | None = None, ) -> Mapping[str, Any]: params: dict[str, Any] = { "schema_version": SCHEDULER_STATE_STORE_REQUEST_SCHEMA, @@ -265,6 +266,8 @@ def _store_result( } if state is not None: params["state"] = state + if migrate_legacy is not None: + params["migrate_legacy"] = migrate_legacy try: result = effect_runtime_result(f"scheduler.state.{operation}", params) except EffectRuntimeRejected as exc: @@ -290,6 +293,11 @@ def load_scheduler_state( ) -> dict[str, Any] | None: if not agent_id: return None + # Python reads are projections (quota should-run hints and the scheduler + # follow-up hint rebuild). Legacy state stays read-only until the native + # commit holds every applicable authority fence and migrates it under the + # scheduler lock; a read here must not move or delete the legacy file + # before a receipt/authority fence can reject the caller. result = _store_result( "load", runtime_root, @@ -298,6 +306,7 @@ def load_scheduler_state( agent_id=agent_id, surface=surface, state_key=state_key, + migrate_legacy=False, ) state = result.get("state") if state is not None and not isinstance(state, dict): diff --git a/tests/control_plane/test_scheduler_stale_receipt_gate.py b/tests/control_plane/test_scheduler_stale_receipt_gate.py index 4196b3f940..50cf0a4239 100644 --- a/tests/control_plane/test_scheduler_stale_receipt_gate.py +++ b/tests/control_plane/test_scheduler_stale_receipt_gate.py @@ -8,6 +8,9 @@ from __future__ import annotations +import hashlib +import json +import re from pathlib import Path import pytest @@ -18,6 +21,7 @@ ) from loopx.control_plane.scheduler.state import ( APP_AUTOMATION_STATEFUL_BACKOFF_STATE_KEY as APP_KEY, + build_scheduler_state, load_scheduler_state, scheduler_state_path, ) @@ -58,12 +62,12 @@ def _should_run( def _ack( - registry: Path, runtime: Path, project: Path, turn: str + registry: Path, runtime: Path, project: Path, turn: str, *, execute: bool = True ) -> tuple[int, dict]: return run_json_cli_result( "quota", "scheduler-ack-current", "--goal-id", GOAL, "--agent-id", SCOPED_AGENT_ID, "--codex-app", - "--turn-instance-id", turn, "--execute", + "--turn-instance-id", turn, *(["--execute"] if execute else []), registry_path=registry, runtime_root=runtime, cwd=project, ) @@ -119,3 +123,75 @@ def test_fresh_receipt_still_commits_the_ack( ) assert state is not None assert state["last_applied_rrule"] + + +def _legacy_segment(value: str) -> str: + """Mirror the legacy (unbounded) scheduler path segment normalization.""" + + safe = re.sub(r"[^0-9A-Za-z_.-]+", "-", value.strip()) + return re.sub(r"^[-._]+|[-._]+$", "", safe) or "default" + + +def _legacy_state_path( + runtime: Path, *, goal_id: str, agent_id: str, surface: str, state_key: str +) -> Path: + return ( + runtime + / "goals" + / _legacy_segment(goal_id) + / "scheduler-state" + / _legacy_segment(agent_id) + / _legacy_segment(surface) + / f"{hashlib.sha256(state_key.encode('utf-8')).hexdigest()[:16]}.json" + ) + + +@pytest.mark.parametrize("execute", [True, False]) +def test_stale_receipt_rejection_does_not_migrate_legacy_scheduler_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, execute: bool +) -> None: + """The rejected follow-up rebuilds a hint, but the hint read is read-only. + + Migration belongs to the native commit that holds the scheduler lock after + every receipt fence passes; a blocked preview or execute must leave the + legacy layout byte-identical and create no canonical state file. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + legacy_path = _legacy_state_path( + runtime, + goal_id=GOAL, + agent_id=SCOPED_AGENT_ID, + surface="codex_app", + state_key=APP_KEY, + ) + legacy_path.parent.mkdir(parents=True, exist_ok=True) + legacy_state = build_scheduler_state( + goal_id=GOAL, + agent_id=SCOPED_AGENT_ID, + state_key=APP_KEY, + reset_token="legacy-reset", + identity_signature="legacy-identity", + progression_index=1, + progression_minutes=[30, 60], + last_applied_rrule="FREQ=MINUTELY;INTERVAL=30", + updated_at="2026-01-01T12:00:00Z", + ) + legacy_bytes = json.dumps(legacy_state, sort_keys=True).encode("utf-8") + legacy_path.write_bytes(legacy_bytes) + canonical_path = scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ) + assert legacy_path != canonical_path + assert not canonical_path.exists() + + _supercede(runtime, TURN_2) + code, result = _ack(registry, runtime, project, TURN_1, execute=execute) + + assert code == 1, result + assert result["status"] == "heartbeat_receipt_stale" + assert result["state"] == "blocked_receipt" + assert result["scheduler_state_mutated"] is False + assert legacy_path.read_bytes() == legacy_bytes + assert not canonical_path.exists() From d47cdfe29a27d872018fb9a32c35bc6de2dbedcc Mon Sep 17 00:00:00 2001 From: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:21:16 +0800 Subject: [PATCH 3/5] fix(scheduler): keep identity-less quota audits from expiring a receipt heartbeatReceiptStatus() re-decided what counts as a successor receipt: after a run_id-carrying receipt it also treated an identity-less `normal_run` row with details.ok/should_run as a superseding receipt. The host protocol (host-integration-plugin-plan-v0) is explicit that only a quota event with a nonempty string Turn identity is a heartbeat receipt, and that identity-less invocations stay diagnostic audits: they neither grant host writeback nor expire a valid receipt. The discovery predicate already encoded that rule, so the successor scan now reuses it (isHeartbeatReceiptEvent) instead of keeping a second, looser definition. Without this, the Python compatibility route reports heartbeat_receipt_stale and refuses to write scheduler state whenever such an audit row follows the admitted Turn's own receipt. Tests: replace the TypeScript case that locked the old behaviour with one proving the ACK still commits and that a real newer Turn still expires the older receipt, and add the same diagnostic-row scenario to the Python compatibility-route gate. Signed-off-by: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> --- loopx/control_plane/rollout_receipt_log.ts | 48 ++++++++------- .../test_scheduler_stale_receipt_gate.py | 61 +++++++++++++++++++ .../scheduler_heartbeat_followup.test.ts | 10 ++- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index a02c8464cc..fd82ae9f8d 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -118,6 +118,24 @@ export function goalHeartbeatReceiptsFromSnapshot( return heartbeatReceiptsFromEvents(snapshot.events, goalId, agentId); } +/** + * One receipt predicate, shared by discovery and by successor scans. + * + * A failed invocation without a real Turn identity is an audit: the host + * protocol keeps it visible in the log, but lets it neither grant host + * writeback nor expire the receipt of the Turn it follows. + */ +export function isHeartbeatReceiptEvent( + event: JsonObject, + goalId: string, + agentId?: string | null, +): boolean { + return event.event_kind === HEARTBEAT_RECEIPT_EVENT_KIND && + event.goal_id === goalId && + typeof event.run_id === "string" && event.run_id.trim().length > 0 && + (agentId === undefined || event.agent_id === agentId); +} + /** The heartbeat receipt filter over events a caller already read. */ export function heartbeatReceiptsFromEvents( events: readonly JsonObject[], @@ -125,12 +143,7 @@ export function heartbeatReceiptsFromEvents( agentId?: string | null, ): JsonObject[] { return events.filter((event) => - event.event_kind === HEARTBEAT_RECEIPT_EVENT_KIND && - event.goal_id === goalId && - // A failed invocation without a real Turn identity is an audit, not a - // receipt that can grant host writeback or supersede an admitted Turn. - typeof event.run_id === "string" && event.run_id.trim().length > 0 && - (agentId === undefined || event.agent_id === agentId) + isHeartbeatReceiptEvent(event, goalId, agentId) ); } @@ -177,21 +190,14 @@ export async function heartbeatReceiptStatus(params: { ? -1 : snapshot.events.indexOf(firstReceipt); if (firstMatch < 0) return "missing"; - return snapshot.events.slice(firstMatch + 1).some((event) => { - if ( - event.event_kind !== HEARTBEAT_RECEIPT_EVENT_KIND || - event.goal_id !== params.goalId || - event.agent_id !== params.agentId - ) return false; - if (typeof event.run_id === "string" && event.run_id.trim().length > 0) { - return event.run_id !== params.turnInstanceId; - } - const details = event.details; - return event.status === "normal_run" && - typeof details === "object" && details !== null && !Array.isArray(details) && - (details as JsonObject).ok === true && - (details as JsonObject).should_run === true; - }) + // Only a real Turn receipt expires an admitted one. Identity-less quota rows + // are diagnostics: the protocol neither grants them host writeback nor lets + // them supersede the receipt they follow, so the successor scan reuses the + // discovery predicate rather than re-deciding what a receipt is. + return snapshot.events.slice(firstMatch + 1).some((event) => + isHeartbeatReceiptEvent(event, params.goalId, params.agentId) && + event.run_id !== params.turnInstanceId + ) ? "stale" : "fresh"; } diff --git a/tests/control_plane/test_scheduler_stale_receipt_gate.py b/tests/control_plane/test_scheduler_stale_receipt_gate.py index 50cf0a4239..63607d4b47 100644 --- a/tests/control_plane/test_scheduler_stale_receipt_gate.py +++ b/tests/control_plane/test_scheduler_stale_receipt_gate.py @@ -195,3 +195,64 @@ def test_stale_receipt_rejection_does_not_migrate_legacy_scheduler_state( assert result["scheduler_state_mutated"] is False assert legacy_path.read_bytes() == legacy_bytes assert not canonical_path.exists() + + +def _diagnostic_row(runtime: Path) -> None: + """Append one identity-less successful quota row: a host-protocol audit.""" + + append_rollout_event( + rollout_event_log_path(runtime, GOAL), + build_rollout_event( + goal_id=GOAL, + event_kind="quota_should_run", + agent_id=SCOPED_AGENT_ID, + status="normal_run", + details={"ok": True, "should_run": True}, + ), + ) + + +def test_identity_less_diagnostic_row_cannot_expire_the_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A diagnostic row after the receipt is audit, not a successor. + + The host protocol requires a nonempty Turn identity for a heartbeat + receipt; an identity-less quota row must neither grant host writeback nor + expire the admitted Turn's receipt, on either backend. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + _diagnostic_row(runtime) + + code, result = _ack(registry, runtime, project, TURN_1) + + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + state = load_scheduler_state( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ) + assert state is not None + assert state["last_applied_rrule"] + + +def test_a_real_successor_receipt_still_expires_the_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Strategy: the relaxation above must not weaken real successors.""" + + registry, runtime, project = _fixture(tmp_path, monkeypatch) + _should_run(registry, runtime, project, TURN_1) + _diagnostic_row(runtime) + _supercede(runtime, TURN_2) + + code, result = _ack(registry, runtime, project, TURN_1) + + assert code == 1, result + assert result["status"] == "heartbeat_receipt_stale" + assert result["scheduler_state_mutated"] is False + assert not scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ).exists() diff --git a/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts b/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts index a636ce3df3..bfcde38027 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts @@ -325,7 +325,7 @@ test("another Agent receipt cannot supersede ACK or host-failure writeback", asy assert.equal(failed.failure_count, 1); }); -test("a successful identity-less quota guard supersedes an older host receipt", async (t) => { +test("a successful identity-less quota guard cannot supersede an older host receipt", async (t) => { const runtimeRoot = await tempRuntime(t); await appendReceipt(runtimeRoot, "turn-followup-1"); await writeFile(receiptPath(runtimeRoot), `${JSON.stringify({ @@ -337,6 +337,14 @@ test("a successful identity-less quota guard supersedes an older host receipt", details: { ok: true, should_run: true }, })}\n`, { flag: "a" }); + // The host protocol keeps identity-less quota rows as diagnostic audits: they + // never expire the admitted Turn's receipt, so this ACK still commits. + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); + assert.equal(acknowledged.ok, true); + assert.equal((acknowledged.scheduler_commit as Record).status, "written"); + + // A real newer Turn still supersedes the older receipt, diagnostics or not. + await appendReceipt(runtimeRoot, "turn-followup-2"); const stale = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); assert.equal(stale.error_code, "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE"); assert.equal(stale.scheduler_state_mutated, false); From e2237ccc50e18ecf1932c7fafd9a96a8759d6a61 Mon Sep 17 00:00:00 2001 From: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:57:35 +0800 Subject: [PATCH 4/5] fix(scheduler): scope heartbeat receipt scans to one Goal instance A rollout-event log is shared by every Goal that reuses a display name, so the successor scan could let another instance's receipt expire an admitted Turn's receipt. Resolve the follower's exact GoalRef in the native transport and compare receipts only inside the writer's projection scope: legacy facts compare with legacy facts, and an exact GoalRef only with the same instance. An explicit scope that is malformed or names another goal fails closed with heartbeat_receipt_scope_invalid / heartbeat_receipt_scope_mismatch. Python decision entry points forward the resolved goal_ref, and tests cover ACK and host-failure writebacks on both providers plus the unscoped legacy path. Signed-off-by: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> --- .../cli_commands/quota_scheduler_followup.py | 2 + loopx/control_plane/quota/scheduler_ack.py | 13 ++- loopx/control_plane/quota/source_admission.ts | 50 ++++++++- loopx/control_plane/rollout_receipt_log.ts | 57 +++++++++- .../scheduler/heartbeat_commit.ts | 3 + .../scheduler/heartbeat_followup.ts | 9 ++ loopx/quota.py | 2 + .../test_scheduler_stale_receipt_gate.py | 106 +++++++++++++++++- .../scheduler_heartbeat_followup.test.ts | 105 ++++++++++++++++- 9 files changed, 331 insertions(+), 16 deletions(-) diff --git a/loopx/cli_commands/quota_scheduler_followup.py b/loopx/cli_commands/quota_scheduler_followup.py index 8bbe506bba..c8e52cf16a 100644 --- a/loopx/cli_commands/quota_scheduler_followup.py +++ b/loopx/cli_commands/quota_scheduler_followup.py @@ -229,6 +229,7 @@ def build_scheduler_followup_payload( observed_host_rrule=observed_rrule, failure_kind=args.failure_kind, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) return record_quota_scheduler_ack( status_payload, @@ -250,4 +251,5 @@ def build_scheduler_followup_payload( operator_inbox_urgency_projector=operator_inbox_urgency_projector, before_decision=before_decision, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) diff --git a/loopx/control_plane/quota/scheduler_ack.py b/loopx/control_plane/quota/scheduler_ack.py index 2fc8939725..06247738a2 100644 --- a/loopx/control_plane/quota/scheduler_ack.py +++ b/loopx/control_plane/quota/scheduler_ack.py @@ -4,6 +4,7 @@ import json import subprocess +from collections.abc import Mapping from pathlib import Path from ..effect_runtime import _node_executable @@ -185,19 +186,23 @@ def _run_native_followup( use_current_hint: bool, reason_summary: str | None, turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, ) -> dict[str, Any]: """Decision-free compatibility transport for Python callers. A caller that already bound the follow-up to a Turn passes it here, so the native transaction applies the same heartbeat-receipt fence (missing or - superseded) as the generated route. Callers without a Turn keep the - receipt-less compatibility request. + superseded) as the generated route. The exact GoalRef this caller resolved + travels with it, so the native successor scan compares receipts inside the + same instance instead of across every goal that shares the log. Callers + without a Turn keep the receipt-less compatibility request. """ request = { "schema_version": _FOLLOWUP_REQUEST_SCHEMA, "runtime_root": str(runtime_root), "turn_instance_id": turn_instance_id, + "goal_ref": dict(goal_ref) if goal_ref is not None else None, "require_heartbeat_receipt": turn_instance_id is not None, "before": before, "host_facts": host_facts, @@ -306,6 +311,7 @@ def record_quota_scheduler_ack_for_decision( use_current_hint: bool = False, host_match_observed: bool = False, turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, ) -> dict[str, Any]: state_key = _followup_state_key(before, surface=surface, state_key=state_key) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -376,6 +382,7 @@ def record_quota_scheduler_ack_for_decision( use_current_hint=use_current_hint, reason_summary=reason_summary, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) except ValueError as exc: return _failure_payload( @@ -406,6 +413,7 @@ def record_quota_scheduler_failure_for_decision( failure_kind: str = "host_tool_failure", generated_at: str | None = None, turn_instance_id: str | None = None, + goal_ref: Mapping[str, object] | None = None, ) -> dict[str, Any]: state_key = _followup_state_key(before, surface=surface, state_key=state_key) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -454,6 +462,7 @@ def record_quota_scheduler_failure_for_decision( use_current_hint=False, reason_summary=None, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) except ValueError as exc: return _failure_payload( diff --git a/loopx/control_plane/quota/source_admission.ts b/loopx/control_plane/quota/source_admission.ts index 47526e2a75..68999056fe 100644 --- a/loopx/control_plane/quota/source_admission.ts +++ b/loopx/control_plane/quota/source_admission.ts @@ -113,14 +113,56 @@ export function requireQuotaOwnerProjection( } } -export function quotaOwnerOwnsProjection( - owner: QuotaAccountingOwner, +/** + * The Goal-instance scope one persisted fact was written under. + * + * `alias` is the legacy projection: the fact carries no `goal_ref` at all. + * Anything else is the one Goal instance the writer presented, so two goals + * that share a display name stay separate even inside one shared log. + */ +export type QuotaProjectionScope = + | Readonly<{ kind: "alias" }> + | Readonly<{ kind: "exact_source"; goalRef: ExactGoalRef }>; + +/** + * Read one fact's scope, or null when its `goal_ref` is not a usable scope. + * + * Absent and unparseable refs are different facts: absent is the legacy + * projection every pre-instance writer produced, while a present-but-unusable + * ref belongs to no scope at all and must not be compared loosely. + */ +export function quotaProjectionScope( + value: unknown, +): QuotaProjectionScope | null { + if (value === undefined) return { kind: "alias" }; + const projected = parseExactGoalRef(value); + return projected.kind === "parsed" + ? { kind: "exact_source", goalRef: projected.value } + : null; +} + +/** + * One judge for "did this fact belong to that scope". + * + * Legacy facts only compare with legacy facts and an exact GoalRef only + * compares with that same instance, so a caller cannot widen a scope by + * asking a looser question than the writer answered. + */ +export function quotaScopeOwnsProjection( + scope: QuotaProjectionScope, value: unknown, ): boolean { - if (owner.kind === "alias") return value === undefined; + if (scope.kind === "alias") return value === undefined; const projected = parseExactGoalRef(value); return projected.kind === "parsed" - && sameGoalRef(projected.value, owner.goalRef); + && sameGoalRef(projected.value, scope.goalRef); +} + +export function quotaOwnerOwnsProjection( + owner: QuotaAccountingOwner, + value: unknown, +): boolean { + return quotaScopeOwnsProjection(owner, value); } function quotaSourceLock( diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index fd82ae9f8d..33d9c68541 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -11,6 +11,11 @@ import { relative, resolve, sep } from "node:path"; import type { JsonObject } from "./effect_program.ts"; import { EffectRuntimeRequestError } from "./effect_runtime_errors.ts"; +import { + quotaProjectionScope, + quotaScopeOwnsProjection, + type QuotaProjectionScope, +} from "./quota/source_admission.ts"; import { requireNonEmptyString } from "./runtime_decode.ts"; import { readReceiptLogSnapshot } from "./runtime/receipt_log_snapshot.ts"; @@ -172,7 +177,9 @@ export async function heartbeatReceiptStatus(params: { goalId: string; agentId: string; turnInstanceId: string; + goalRef?: unknown; }): Promise { + const requestedScope = requestedReceiptScope(params.goalId, params.goalRef); const snapshot = await readGoalRolloutEventSnapshot( params.runtimeRoot, params.goalId, @@ -183,21 +190,59 @@ export async function heartbeatReceiptStatus(params: { params.agentId, ); if (snapshot === null || receipts === null) return "missing"; - const firstReceipt = receipts.find( - (event) => event.run_id === params.turnInstanceId, + const firstReceipt = receipts.find((event) => + event.run_id === params.turnInstanceId && + (requestedScope === null || + quotaScopeOwnsProjection(requestedScope, event.goal_ref)) ); - const firstMatch = firstReceipt === undefined - ? -1 - : snapshot.events.indexOf(firstReceipt); + if (firstReceipt === undefined) return "missing"; + const firstMatch = snapshot.events.indexOf(firstReceipt); if (firstMatch < 0) return "missing"; + // The receipt itself states the scope its successor scan may use: a legacy + // receipt is only superseded by another legacy receipt, and an instance + // receipt only by that same Goal instance. A differently named Goal that + // happens to share this log therefore cannot expire an admitted Turn, and an + // unscopable `goal_ref` fails closed instead of authorizing a writeback. + const receiptScope = quotaProjectionScope(firstReceipt.goal_ref); + if (receiptScope === null) return "missing"; // Only a real Turn receipt expires an admitted one. Identity-less quota rows // are diagnostics: the protocol neither grants them host writeback nor lets // them supersede the receipt they follow, so the successor scan reuses the // discovery predicate rather than re-deciding what a receipt is. return snapshot.events.slice(firstMatch + 1).some((event) => isHeartbeatReceiptEvent(event, params.goalId, params.agentId) && - event.run_id !== params.turnInstanceId + event.run_id !== params.turnInstanceId && + quotaScopeOwnsProjection(receiptScope, event.goal_ref) ) ? "stale" : "fresh"; } + +/** + * Resolve the scope a caller asked the receipt fence to run in. + * + * Absent means "use the scope the matching receipt states", so the persisted + * receipt stays the only authority. A caller that does know its Goal instance + * passes it, and a malformed or foreign scope is refused rather than silently + * widening the comparison. + */ +function requestedReceiptScope( + goalId: string, + value: unknown, +): QuotaProjectionScope | null { + if (value === undefined || value === null) return null; + const scope = quotaProjectionScope(value); + if (scope === null || scope.kind === "alias") { + throw new EffectRuntimeRequestError( + "heartbeat receipt scope requires goal_id and goal_instance_id", + "heartbeat_receipt_scope_invalid", + ); + } + if (scope.goalRef.goalId.value !== goalId) { + throw new EffectRuntimeRequestError( + "heartbeat receipt scope does not belong to the requested goal", + "heartbeat_receipt_scope_mismatch", + ); + } + return scope; +} diff --git a/loopx/control_plane/scheduler/heartbeat_commit.ts b/loopx/control_plane/scheduler/heartbeat_commit.ts index 2017c07143..71ad1f41f6 100644 --- a/loopx/control_plane/scheduler/heartbeat_commit.ts +++ b/loopx/control_plane/scheduler/heartbeat_commit.ts @@ -171,6 +171,8 @@ export interface SchedulerHeartbeatCommitResult extends JsonObject { export interface SchedulerHeartbeatCommitOptions { readonly heartbeatReceiptTurnInstanceId?: string | null; + /** The Goal instance the caller bound the follow-up to, when it knows it. */ + readonly heartbeatReceiptGoalRef?: JsonObject | null; } function stableValue(value: unknown): unknown { @@ -853,6 +855,7 @@ export async function evaluateSchedulerHeartbeatCommit( goalId: request.scope.goalId, agentId: request.scope.agentId, turnInstanceId: receiptTurnInstanceId, + goalRef: options.heartbeatReceiptGoalRef, }); if (status !== "fresh") { return result( diff --git a/loopx/control_plane/scheduler/heartbeat_followup.ts b/loopx/control_plane/scheduler/heartbeat_followup.ts index d1047f0454..b98ed942c0 100644 --- a/loopx/control_plane/scheduler/heartbeat_followup.ts +++ b/loopx/control_plane/scheduler/heartbeat_followup.ts @@ -33,6 +33,8 @@ export interface SchedulerHeartbeatFollowupRequest extends JsonObject { schema_version: typeof SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA; runtime_root: string; turn_instance_id: string | null; + /** The Goal instance the Python transport resolved for this follow-up. */ + goal_ref: JsonObject | null; require_heartbeat_receipt: boolean; before: JsonObject; host_facts: JsonObject; @@ -95,6 +97,9 @@ function requestObject(value: unknown): SchedulerHeartbeatFollowupRequest { schema_version: SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA, runtime_root: runtimeRoot, turn_instance_id: turnInstanceId, + goal_ref: input.goal_ref === undefined || input.goal_ref === null + ? null + : requireJsonObject(input.goal_ref, "goal_ref"), require_heartbeat_receipt: requireReceipt, before: requireJsonObject(input.before ?? {}, "before"), host_facts: hostFacts, @@ -341,6 +346,7 @@ export async function evaluateSchedulerHeartbeatFollowup( goalId: String(facts.goal_id), agentId: String(facts.agent_id), turnInstanceId: String(request.turn_instance_id), + goalRef: request.goal_ref, }); if (status !== "fresh") return receiptFailure(request, status); } @@ -351,6 +357,9 @@ export async function evaluateSchedulerHeartbeatFollowup( heartbeatReceiptTurnInstanceId: request.require_heartbeat_receipt ? request.turn_instance_id : null, + heartbeatReceiptGoalRef: request.require_heartbeat_receipt + ? request.goal_ref + : null, }); const receiptStatus = commitReceiptFailureStatus(commit); if (receiptStatus !== null) return receiptFailure(request, receiptStatus); diff --git a/loopx/quota.py b/loopx/quota.py index 1a983d0ded..5ec04954ca 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1245,6 +1245,7 @@ def record_quota_scheduler_ack( operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None = None, before_decision: Mapping[str, Any] | None = None, turn_instance_id: str | None = None, + goal_ref: Mapping[str, Any] | None = None, ) -> dict[str, Any]: safe_goal_id = _validate_goal_id_path_segment(str(goal_id or "")) safe_agent_id = normalize_todo_claimed_by(agent_id) @@ -1280,6 +1281,7 @@ def record_quota_scheduler_ack( use_current_hint=use_current_hint, host_match_observed=host_match_observed, turn_instance_id=turn_instance_id, + goal_ref=goal_ref, ) diff --git a/tests/control_plane/test_scheduler_stale_receipt_gate.py b/tests/control_plane/test_scheduler_stale_receipt_gate.py index 63607d4b47..b580778779 100644 --- a/tests/control_plane/test_scheduler_stale_receipt_gate.py +++ b/tests/control_plane/test_scheduler_stale_receipt_gate.py @@ -19,6 +19,7 @@ SCOPED_AGENT_ID, write_cli_fixture, ) +from loopx.control_plane.goals.goal_ref_validation import exact_goal_ref from loopx.control_plane.scheduler.state import ( APP_AUTOMATION_STATEFUL_BACKOFF_STATE_KEY as APP_KEY, build_scheduler_state, @@ -35,13 +36,17 @@ GOAL = "needs-operator" TURN_1 = "stale-receipt-turn-1" TURN_2 = "stale-receipt-turn-2" +# A different Goal that shares this display name and Agent, so it writes into +# the same rollout-event log. +OTHER_INSTANCE = "ginst_" + "b" * 32 +PROVIDERS = ["file", "sqlite"] def _fixture( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, provider: str = "file" ) -> tuple[Path, Path, Path]: registry, runtime, project = write_cli_fixture( - tmp_path / "fixture", scoped_agents=True, provider="file" + tmp_path / "fixture", scoped_agents=True, provider=provider ) monkeypatch.setenv("CODEX_HOME", str(tmp_path / "codex-home")) monkeypatch.setenv("CODEX_THREAD_ID", "scheduler-fixture") @@ -72,7 +77,28 @@ def _ack( ) -def _supercede(runtime: Path, turn: str) -> None: +def _fail( + registry: Path, + runtime: Path, + project: Path, + turn: str, + *, + failed_rrule: str, + execute: bool = True, +) -> tuple[int, dict]: + return run_json_cli_result( + "quota", "scheduler-fail-current", "--goal-id", GOAL, + "--agent-id", SCOPED_AGENT_ID, "--codex-app", + "--turn-instance-id", turn, + "--failed-rrule", failed_rrule, + *(["--execute"] if execute else []), + registry_path=registry, runtime_root=runtime, cwd=project, + ) + + +def _supercede( + runtime: Path, turn: str, goal_ref: dict[str, str] | None = None +) -> None: """Append a later receipt for ``turn`` on the same goal and agent.""" append_rollout_event( @@ -80,6 +106,7 @@ def _supercede(runtime: Path, turn: str) -> None: build_rollout_event( goal_id=GOAL, event_kind="quota_should_run", + goal_ref=goal_ref, agent_id=SCOPED_AGENT_ID, run_id=turn, status="normal_run", @@ -256,3 +283,76 @@ def test_a_real_successor_receipt_still_expires_the_receipt( assert not scheduler_state_path( runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY ).exists() + + +def _writeback( + operation: str, + registry: Path, + runtime: Path, + project: Path, + turn: str, + hint: dict, +) -> tuple[int, dict]: + """Run the ACK or the host-failure writeback this Turn is bound to.""" + + if operation == "ack": + return _ack(registry, runtime, project, turn) + return _fail( + registry, + runtime, + project, + turn, + failed_rrule=hint["scheduler_hint"]["app_automation"]["recommended_rrule"], + ) + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("operation", ["ack", "failure"]) +def test_same_scope_successor_receipt_still_expires_the_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation: str +) -> None: + """Control: a real successor in the same scope still blocks both writebacks. + + Nothing in the instance-scoping relaxation may let an older Turn report + over a newer one that genuinely follows it. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + _supercede(runtime, TURN_2) + + code, result = _writeback(operation, registry, runtime, project, TURN_1, hint) + + assert code == 1, result + assert result["status"] == "heartbeat_receipt_stale" + assert result["state"] == "blocked_receipt" + assert result["scheduler_state_mutated"] is False + assert not scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ).exists() + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("operation", ["ack", "failure"]) +def test_another_goal_instance_receipt_cannot_expire_the_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation: str +) -> None: + """A receipt from another Goal instance is not this Turn's successor. + + Both Goals share the display name and Agent, so they append to one log. + Only receipts of the admitted Turn's own scope may expire its receipt. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + _supercede(runtime, TURN_2, exact_goal_ref(GOAL, OTHER_INSTANCE)) + + code, result = _writeback(operation, registry, runtime, project, TURN_1, hint) + + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + state = load_scheduler_state( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ) + assert state is not None diff --git a/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts b/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts index bfcde38027..2f1c0acb48 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_followup.test.ts @@ -68,7 +68,11 @@ function receiptPath(runtimeRoot: string): string { return join(runtimeRoot, "goals", scope.goal_id, "rollout-event-log.jsonl"); } -async function appendReceipt(runtimeRoot: string, turnId: string): Promise { +async function appendReceipt( + runtimeRoot: string, + turnId: string, + goalRef?: Record, +): Promise { const path = receiptPath(runtimeRoot); await mkdir(dirname(path), { recursive: true }); await writeFile(path, `${JSON.stringify({ @@ -77,9 +81,18 @@ async function appendReceipt(runtimeRoot: string, turnId: string): Promise goal_id: scope.goal_id, agent_id: scope.agent_id, run_id: turnId, + ...(goalRef === undefined ? {} : { goal_ref: goalRef }), })}\n`, { encoding: "utf8", flag: "a" }); } +const instanceScope = (suffix: string): Record => ({ + goal_id: scope.goal_id, + goal_instance_id: `ginst_${suffix.repeat(32)}`, +}); + +const ownInstance = instanceScope("a"); +const otherInstance = instanceScope("b"); + function request( runtimeRoot: string, extra: Record = {}, @@ -389,3 +402,93 @@ test("a quota audit cannot manufacture a missing heartbeat receipt", async (t) = }); await assert.rejects(readFile(statePath, "utf8"), { code: "ENOENT" }); }); + +test("a receipt from another Goal instance cannot supersede a legacy receipt", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + // Same goal id and Agent, different Goal instance: the shared log must not + // let that instance expire the admitted Turn's receipt. + await appendReceipt(runtimeRoot, "turn-followup-other", otherInstance); + + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot)); + assert.equal(acknowledged.ok, true); + assert.equal((acknowledged.scheduler_commit as Record).status, "written"); + + const base = request(runtimeRoot); + const failed = await evaluateSchedulerHeartbeatFollowup({ + ...base, + host_facts: { + ...(base.host_facts as Record), + operation: "host_failure", + failure_kind: "timeout", + observed_host_rrule: "FREQ=MINUTELY;INTERVAL=3", + }, + }); + assert.equal(failed.ok, true); + assert.equal(failed.failure_count, 1); +}); + +test("a scoped follow-up still rejects a successor of its own Goal instance", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1", ownInstance); + await appendReceipt(runtimeRoot, "turn-followup-2", ownInstance); + + const stale = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: ownInstance, + })); + assert.equal(stale.error_code, "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE"); + assert.equal(stale.scheduler_state_mutated, false); +}); + +test("a scoped follow-up ignores a successor from another Goal instance", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1", ownInstance); + await appendReceipt(runtimeRoot, "turn-followup-other", otherInstance); + + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: ownInstance, + })); + assert.equal(acknowledged.ok, true); + assert.equal((acknowledged.scheduler_commit as Record).status, "written"); +}); + +test("a scoped follow-up ignores a legacy successor in the same log", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1", ownInstance); + await appendReceipt(runtimeRoot, "turn-followup-2"); + + const acknowledged = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: ownInstance, + })); + assert.equal(acknowledged.ok, true); + assert.equal((acknowledged.scheduler_commit as Record).status, "written"); +}); + +test("a scoped follow-up does not adopt a legacy receipt", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1"); + + const missing = await evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: ownInstance, + })); + assert.equal(missing.error_code, "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_MISSING"); + assert.equal(missing.scheduler_state_mutated, false); +}); + +test("a follow-up scope without a Goal instance is refused", async (t) => { + const runtimeRoot = await tempRuntime(t); + await appendReceipt(runtimeRoot, "turn-followup-1", ownInstance); + + await assert.rejects( + evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: { goal_id: scope.goal_id }, + })), + { code: "heartbeat_receipt_scope_invalid" }, + ); + await assert.rejects( + evaluateSchedulerHeartbeatFollowup(request(runtimeRoot, { + goal_ref: { ...ownInstance, goal_id: "another-goal" }, + })), + { code: "heartbeat_receipt_scope_mismatch" }, + ); +}); From 983fdec8c7a966d32df8718604e878fd989c9157 Mon Sep 17 00:00:00 2001 From: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:23:38 +0800 Subject: [PATCH 5/5] fix(scheduler): bind the origin receipt scope into generated follow-ups The native ACK and host-failure commands were generated from facts that only carried the Turn id, so the transaction resolved the receipt by scanning the shared rollout log for the first row with that run_id. Two Goal instances may both write one Turn id into that log; when the other instance's row landed first, the scan adopted it, never saw the real successor, and let a stale writeback through on both File and SQLite. Carry the origin receipt's GoalRef through the scheduler hint into the host facts the host repeats verbatim, and forward it as the follow-up request's goal_ref so the pre-flight check and the in-lock recheck stay on the receipt the command was generated for. A historical command without that scope cannot tell the instances apart, so an unscoped scan that matches one Turn id under more than one scope now fails closed with heartbeat_receipt_ambiguous instead of guessing; a log with a single scope keeps working unchanged. Tests run the command the hint actually generated: with another instance's row placed first, both writebacks are refused on File and SQLite and no scheduler state is written, while the same commands still commit when nothing collides. The TypeScript cases cover the bound scope, the ambiguous refusal and the single-scope legacy log; a projection test pins goal_ref into the generated facts. Signed-off-by: GZY-SUPER-HACKER <162807803+GZY-SUPER-HACKER@users.noreply.github.com> --- loopx/control_plane/rollout_receipt_log.ts | 27 ++++- .../scheduler/heartbeat_followup_cli.ts | 9 +- loopx/control_plane/scheduler/hint.ts | 6 +- .../control_plane/scheduler/scheduler_hint.py | 5 + .../test_scheduler_hint_single_request.py | 51 +++++++++ .../test_scheduler_stale_receipt_gate.py | 103 ++++++++++++++++++ .../scheduler_heartbeat_followup_cli.test.ts | 92 ++++++++++++++++ 7 files changed, 289 insertions(+), 4 deletions(-) diff --git a/loopx/control_plane/rollout_receipt_log.ts b/loopx/control_plane/rollout_receipt_log.ts index 33d9c68541..cdc5be1b91 100644 --- a/loopx/control_plane/rollout_receipt_log.ts +++ b/loopx/control_plane/rollout_receipt_log.ts @@ -190,12 +190,29 @@ export async function heartbeatReceiptStatus(params: { params.agentId, ); if (snapshot === null || receipts === null) return "missing"; - const firstReceipt = receipts.find((event) => + const firstReceiptMatches = receipts.filter((event) => event.run_id === params.turnInstanceId && (requestedScope === null || quotaScopeOwnsProjection(requestedScope, event.goal_ref)) ); - if (firstReceipt === undefined) return "missing"; + if (firstReceiptMatches.length === 0) return "missing"; + // A command generated before the origin scope travelled with its facts cannot + // say which instance its Turn belonged to. Two Goal instances may share one + // Turn id inside a log they both write, and taking whichever row landed first + // would scan the wrong instance's successors and let a stale writeback + // through, so the ambiguity is refused instead of guessed. The host re-reads + // a scoped hint and retries with the origin receipt's own scope. + if ( + requestedScope === null && + new Set(firstReceiptMatches.map((event) => receiptScopeKey(event.goal_ref))) + .size > 1 + ) { + throw new EffectRuntimeRequestError( + "heartbeat receipt Turn identity matches more than one Goal instance", + "heartbeat_receipt_ambiguous", + ); + } + const firstReceipt = firstReceiptMatches[0]; const firstMatch = snapshot.events.indexOf(firstReceipt); if (firstMatch < 0) return "missing"; // The receipt itself states the scope its successor scan may use: a legacy @@ -218,6 +235,12 @@ export async function heartbeatReceiptStatus(params: { : "fresh"; } +/** One comparable key per scope, so ambiguity is decided by set size. */ +function receiptScopeKey(value: unknown): string { + const scope = quotaProjectionScope(value); + return scope === null ? "unscopable" : JSON.stringify(scope); +} + /** * Resolve the scope a caller asked the receipt fence to run in. * diff --git a/loopx/control_plane/scheduler/heartbeat_followup_cli.ts b/loopx/control_plane/scheduler/heartbeat_followup_cli.ts index d7c448debf..955ee7f0b0 100644 --- a/loopx/control_plane/scheduler/heartbeat_followup_cli.ts +++ b/loopx/control_plane/scheduler/heartbeat_followup_cli.ts @@ -4,7 +4,7 @@ import { effectRuntimeErrorPayload, EffectRuntimeRequestError, } from "../effect_runtime_errors.ts"; -import { requireJsonObject } from "../runtime_decode.ts"; +import { jsonObject, requireJsonObject } from "../runtime_decode.ts"; import { isSupportedNodeVersion } from "../runtime/node_version.ts"; import { evaluateSchedulerHeartbeatFollowup, @@ -214,6 +214,12 @@ function requestFromArgs(argv: string[]): { request: Record; fo const facts = requireJsonObject(hint.host_facts, "host_facts"); matchingText(facts, "goal_id", parsed.goal_id, "--goal-id"); matchingText(facts, "agent_id", parsed.agent_id, "--agent-id"); + // The origin receipt's scope travels with the facts it was generated for. + // Passing it into the transaction is what keeps the pre-flight check and the + // in-lock recheck on that receipt instead of resolving the bare Turn id again. + const goalRef = jsonObject(facts.goal_ref); + if (goalRef !== null) + matchingText(goalRef, "goal_id", parsed.goal_id, "--goal-id"); const expectedOperation = parsed.command === "scheduler-ack-current" ? "ack" : "host_failure"; @@ -291,6 +297,7 @@ function requestFromArgs(argv: string[]): { request: Record; fo runtime_root: parsed.runtime_root, turn_instance_id: parsed.turn_instance_id, require_heartbeat_receipt: true, + ...(goalRef === null ? {} : { goal_ref: goalRef }), before: hint.before ?? {}, use_current_hint: hint.use_current_hint ?? expectedOperation === "ack", reason_summary: values["--reason-summary"] ?? null, diff --git a/loopx/control_plane/scheduler/hint.ts b/loopx/control_plane/scheduler/hint.ts index f472872568..62a4c596ec 100644 --- a/loopx/control_plane/scheduler/hint.ts +++ b/loopx/control_plane/scheduler/hint.ts @@ -192,8 +192,12 @@ function appAutomation(decision: SchedulerBackoffTransitionResult, cadence: Cade function hostFacts(request: JsonObject, profile: HintProfile, decision: SchedulerBackoffTransitionResult, scope: JsonObject): JsonObject | null { const payload = request.payload as JsonObject, agent = jsonObject(payload.agent_identity)?.agent_id; if (!payload.goal_id || !agent) return null; + // The follow-up the host repeats verbatim must stay bound to the Goal instance + // whose Turn produced the receipt. Without it a native command can only + // resolve a Turn id, and two instances sharing this log would collide. + const goalRef = jsonObject(request.goal_ref); return {schema_version: "loopx_scheduler_heartbeat_host_facts_v0", goal_id: String(payload.goal_id), agent_id: String(agent), - ...scope, progression_index: decision.current_index, progression_minutes: scope.progression_minutes, + ...(goalRef === null ? {} : {goal_ref: goalRef}), ...scope, progression_index: decision.current_index, progression_minutes: scope.progression_minutes, expected_rrule: decision.current_rrule, cadence_class: profile.cadence_class, stale_tolerance_minutes: 2, generated_at: request.generated_at, ack_needed: ["apply_required", "host_match_ack_required"].includes(decision.host_transition), apply_needed: decision.host_transition === "apply_required"}; diff --git a/loopx/control_plane/scheduler/scheduler_hint.py b/loopx/control_plane/scheduler/scheduler_hint.py index 356ea03b99..0c53aca901 100644 --- a/loopx/control_plane/scheduler/scheduler_hint.py +++ b/loopx/control_plane/scheduler/scheduler_hint.py @@ -326,8 +326,13 @@ def build_scheduler_hint( for key in ("agent_todo_summary", "user_todo_summary") ] facts = _projection_facts(payload) + # The generated follow-up commands must stay bound to the Goal instance whose + # Turn produced the receipt, so the owner scope travels beside the facts + # rather than being re-resolved from a bare Turn id at execution time. + goal_ref = payload.get("goal_ref") params = { "payload": facts, + "goal_ref": dict(goal_ref) if isinstance(goal_ref, Mapping) else None, "identity_json": _identity_json(payload), "context": context.projection(), "arbitration": asdict(arbitration) if arbitration is not None else None, diff --git a/tests/control_plane/test_scheduler_hint_single_request.py b/tests/control_plane/test_scheduler_hint_single_request.py index 5ee1543d39..ed362b7324 100644 --- a/tests/control_plane/test_scheduler_hint_single_request.py +++ b/tests/control_plane/test_scheduler_hint_single_request.py @@ -176,3 +176,54 @@ def test_invalid_host_view_preserves_python_whitespace_rules(surface, view): assert hint["action"] == "repair_scheduler_execution_context" assert hint[view]["applicability"] == "blocked_invalid_context" assert ("codex_app" if view == "app_automation" else "app_automation") not in hint + + +def _followup_facts(cli_args: list[str]) -> dict: + import base64 + import json + import zlib + + encoded = "".join( + cli_args[index + 1] + for index, value in enumerate(cli_args[:-1]) + if value == "--scheduler-host-facts-chunk" + ) + padded = encoded + "=" * (-len(encoded) % 4) + return json.loads(zlib.decompress(base64.b64decode(padded)))["host_facts"] + + +def _ack_hint_facts(payload: dict) -> dict: + arbitration = SchedulerArbitration( + SchedulerDisposition("active_work"), "active_work", "normal_run" + ) + hint = scheduler_hint.build_scheduler_hint( + payload, + arbitration=arbitration, + scheduler_execution_context=scheduler_execution_context_for_runtime_profile( + "codex_app_heartbeat" + ), + ) + return _followup_facts(hint["app_automation"]["ack_hint"]["cli_args"]) + + +def _hint_payload() -> dict: + return { + "goal_id": "hint-fixture", + "agent_identity": {"agent_id": "agent-fixture"}, + "heartbeat_recommendation": {"recommended_mode": "normal_run"}, + "recommended_action": "Perform the bounded work.", + "effective_action": "normal_run", + "interaction_contract": {"mode": "normal_run"}, + } + + +def test_generated_ack_hint_carries_the_origin_goal_instance(): + goal_ref = {"goal_id": "hint-fixture", "goal_instance_id": "ginst_" + "a" * 32} + facts = _ack_hint_facts({**_hint_payload(), "goal_ref": goal_ref}) + # A shared log can hold another instance's row under the same Turn id, so the + # generated command must not resolve its receipt from the Turn id alone. + assert facts["goal_ref"] == goal_ref + + +def test_generated_ack_hint_omits_scope_for_a_legacy_owner(): + assert "goal_ref" not in _ack_hint_facts(_hint_payload()) diff --git a/tests/control_plane/test_scheduler_stale_receipt_gate.py b/tests/control_plane/test_scheduler_stale_receipt_gate.py index b580778779..e453d9e34f 100644 --- a/tests/control_plane/test_scheduler_stale_receipt_gate.py +++ b/tests/control_plane/test_scheduler_stale_receipt_gate.py @@ -10,7 +10,10 @@ import hashlib import json +import os import re +import subprocess +import sys from pathlib import Path import pytest @@ -356,3 +359,103 @@ def test_another_goal_instance_receipt_cannot_expire_the_receipt( runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY ) assert state is not None + + +def _prepend_receipt( + runtime: Path, turn: str, goal_ref: dict[str, str] +) -> None: + """Place another Goal instance's row before the admitted Turn's receipt.""" + + path = rollout_event_log_path(runtime, GOAL) + row = json.dumps( + build_rollout_event( + goal_id=GOAL, + event_kind="quota_should_run", + goal_ref=goal_ref, + agent_id=SCOPED_AGENT_ID, + run_id=turn, + status="normal_run", + details={"ok": True, "should_run": True}, + ), + sort_keys=True, + ) + path.write_bytes(row.encode("utf-8") + b"\n" + path.read_bytes()) + + +def _run_generated( + hint: dict, operation: str, project: Path +) -> tuple[int, dict]: + """Run the native follow-up command the hint generated for this Turn.""" + + packet = hint["scheduler_hint"]["app_automation"][f"{operation}_hint"] + result = subprocess.run( + [sys.executable, "-m", "loopx.cli", "--format", "json", *packet["cli_args"]], + cwd=project, + check=False, + env={**os.environ, "LOOPX_CONTEXT_DELIVERY": "off"}, + text=True, + encoding="utf-8", + errors="replace", + capture_output=True, + ) + return result.returncode, json.loads(result.stdout) + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("operation", ["ack", "failure"]) +def test_generated_native_writeback_refuses_a_shared_turn_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation: str +) -> None: + """The generated command cannot resolve its receipt from the Turn id alone. + + Two Goal instances may write one Turn id into this log. When another + instance's row lands first, picking it as the origin would scan the wrong + instance's successors and let the admitted Turn's stale writeback through. + """ + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + _prepend_receipt(runtime, TURN_1, exact_goal_ref(GOAL, OTHER_INSTANCE)) + _supercede(runtime, TURN_2) + + code, result = _run_generated(hint, operation, project) + + assert code == 1, result + assert result["status"] == "error" + assert result["error"]["code"] == "heartbeat_receipt_ambiguous" + assert not scheduler_state_path( + runtime, goal_id=GOAL, agent_id=SCOPED_AGENT_ID, state_key=APP_KEY + ).exists() + + +@pytest.mark.parametrize("provider", PROVIDERS) +def test_generated_native_failure_writeback_commits_without_a_shared_turn_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + """Control: the same generated command still writes when nothing collides.""" + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + + code, result = _run_generated(hint, "failure", project) + + assert code == 0, result + assert result["ok"] is True + assert result["scheduler_state_mutated"] is True + + +@pytest.mark.parametrize("provider", PROVIDERS) +def test_generated_native_ack_only_refuses_on_a_shared_turn_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + """The ambiguity is decided by the log, not by the writeback's own gates.""" + + registry, runtime, project = _fixture(tmp_path, monkeypatch, provider=provider) + hint = _should_run(registry, runtime, project, TURN_1) + + code, result = _run_generated(hint, "ack", project) + + assert code == 1, result + assert result["reason_code"] == "rrule_mismatch" + assert "error" not in result + diff --git a/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts b/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts index baa30efd6c..d63a67cb60 100644 --- a/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts +++ b/tests/control_plane_ts/scheduler_heartbeat_followup_cli.test.ts @@ -367,3 +367,95 @@ test("native host-followup transport rejects inflated facts beyond its bound", ( "invalid_scheduler_host_facts", ); }); + +const ownInstance = { goal_id: goalId, goal_instance_id: `ginst_${"a".repeat(32)}` }; +const otherInstance = { goal_id: goalId, goal_instance_id: `ginst_${"b".repeat(32)}` }; + +async function writeReceiptRows( + runtimeRoot: string, + rows: readonly { turn: string; goalRef?: Record }[], +): Promise { + const path = join(runtimeRoot, "goals", goalId, "rollout-event-log.jsonl"); + await mkdir(dirname(path), { recursive: true }); + const lines = rows.map((row) => JSON.stringify({ + schema_version: "loopx_rollout_event_v0", + event_kind: "quota_should_run", + goal_id: goalId, + agent_id: agentId, + run_id: row.turn, + ...(row.goalRef === undefined ? {} : { goal_ref: row.goalRef }), + })); + await writeFile(path, `${lines.join("\n")}\n`, "utf8"); +} + +function scopedHint(): Record { + const hint = hintPayload(); + hint.host_facts = { ...(hint.host_facts as Record), goal_ref: ownInstance }; + return hint; +} + +// A shared log can hold another Goal instance's row under the same Turn id. A +// command that only resolves that id would pick whichever row landed first and +// scan the wrong instance's successors, so the generated facts carry the origin +// scope and the command keeps it. +test("native follow-up keeps the receipt scope its facts bound", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceiptRows(runtimeRoot, [ + { turn: turnId, goalRef: otherInstance }, + { turn: turnId, goalRef: ownInstance }, + { turn: "turn-followup-cli-next", goalRef: ownInstance }, + ]); + + const scoped = runCli(argv(runtimeRoot, "json", scopedHint())); + const scopedResult = JSON.parse(scoped.stdout) as Record; + assert.equal(scoped.status, 1, scoped.stderr); + assert.equal( + scopedResult.error_code, + "SCHEDULER_FOLLOWUP_HEARTBEAT_RECEIPT_STALE", + ); + assert.equal(scopedResult.scheduler_state_mutated, false); +}); + +// The same log with no successor is legal for the instance that owns the Turn. +test("a bound scope accepts its own fresh receipt beside another instance", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceiptRows(runtimeRoot, [ + { turn: turnId, goalRef: otherInstance }, + { turn: turnId, goalRef: ownInstance }, + ]); + + const scoped = runCli(argv(runtimeRoot, "json", scopedHint())); + assert.equal(scoped.status, 0, scoped.stderr); + assert.equal( + ((JSON.parse(scoped.stdout) as Record).scheduler_commit as Record).status, + "written", + ); +}); + +// A historical command generated before the scope travelled in its facts cannot +// tell the two rows apart; refusing is the only answer that cannot write back to +// the wrong instance. +test("an unscoped command refuses a Turn id shared by two Goal instances", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceiptRows(runtimeRoot, [ + { turn: turnId, goalRef: otherInstance }, + { turn: turnId, goalRef: ownInstance }, + ]); + + const unscoped = runCli(argv(runtimeRoot)); + const result = JSON.parse(unscoped.stdout) as Record; + assert.equal(unscoped.status, 1, unscoped.stderr); + assert.equal(result.status, "error"); + assert.equal( + (result.error as Record).code, + "heartbeat_receipt_ambiguous", + ); +}); + +test("an unscoped legacy log still accepts its own single Turn receipt", async (t) => { + const runtimeRoot = await tempRuntime(t); + await writeReceiptRows(runtimeRoot, [{ turn: turnId }]); + + const unscoped = runCli(argv(runtimeRoot)); + assert.equal(unscoped.status, 0, unscoped.stderr); +});