From 2a2740368658bed7d255660cabf6144286595993 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:43:38 +0800 Subject: [PATCH 1/2] Preserve requested scheduler detail during recovery waits Prior-Turn recovery rebuilds the scheduler hint when the unchanged admission limit is reached. Forward the CLI detail selector to that projection so the DSH Driver receives the bounded recheck cadence it requested. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/control_plane/quota/live_decision.py | 2 ++ loopx/control_plane/quota/unsettled_host_turn.py | 11 ++++++++--- .../test_unsettled_recovery_admission_bound.py | 13 ++++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index 4105e66a1a..9d85bb3bfb 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -928,6 +928,7 @@ def build_live_quota_should_run_decision( agent_id=agent_id, current_turn_instance_id=turn_instance_id, available_capabilities=available_capabilities, + include_scheduler_detail=include_scheduler_detail, scheduler_execution_context=resolved_context, goal_ref=goal_ref, owner_action_id=goal_owner_resume_id(registered_goal), @@ -1068,6 +1069,7 @@ def _build_remote_authority_wait_decision( payload, registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id, agent_id=agent_id, current_turn_instance_id=turn_instance_id, available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, + include_scheduler_detail=include_scheduler_detail, goal_ref=goal_ref, owner_action_id=goal_owner_resume_id(configured_goal), ): diff --git a/loopx/control_plane/quota/unsettled_host_turn.py b/loopx/control_plane/quota/unsettled_host_turn.py index 44ee6885f7..0d24776d01 100644 --- a/loopx/control_plane/quota/unsettled_host_turn.py +++ b/loopx/control_plane/quota/unsettled_host_turn.py @@ -410,6 +410,7 @@ def apply_unsettled_host_turn_recovery_if_required( agent_id: str | None, current_turn_instance_id: str | None, available_capabilities: list[str] | None, + include_scheduler_detail: bool, scheduler_execution_context: ( Mapping[str, Any] | SchedulerExecutionContextResolution | None ), @@ -432,9 +433,12 @@ def apply_unsettled_host_turn_recovery_if_required( ) if verdict is None: return False - return _apply_recovery_projection(payload, verdict=verdict, registry_path=registry_path, runtime_root=runtime_root, + return _apply_recovery_projection( + payload, verdict=verdict, registry_path=registry_path, runtime_root=runtime_root, current_turn_instance_id=current_turn_instance_id, available_capabilities=available_capabilities, - scheduler_execution_context=scheduler_execution_context) + scheduler_execution_context=scheduler_execution_context, + include_scheduler_detail=include_scheduler_detail, + ) def apply_receipt_bound_wait_recovery( @@ -525,6 +529,7 @@ def _apply_recovery_projection( payload: dict[str, Any], *, verdict: Mapping[str, Any], registry_path: Path, runtime_root: Path, current_turn_instance_id: str | None, available_capabilities: list[str] | None, scheduler_execution_context: Mapping[str, Any] | SchedulerExecutionContextResolution | None, + include_scheduler_detail: bool = False, ) -> bool: recovery = verdict["recovery"] obligation = verdict["obligation"] @@ -619,6 +624,6 @@ def _apply_recovery_projection( payload["scheduler_hint"] = build_scheduler_hint( payload, arbitration=arbitration, scheduler_execution_context=scheduler_execution_context, - available_capabilities=available_capabilities, + available_capabilities=available_capabilities, include_detail=include_scheduler_detail, ) return True diff --git a/tests/control_plane/test_unsettled_recovery_admission_bound.py b/tests/control_plane/test_unsettled_recovery_admission_bound.py index 2f1dad56b7..2629b800e3 100644 --- a/tests/control_plane/test_unsettled_recovery_admission_bound.py +++ b/tests/control_plane/test_unsettled_recovery_admission_bound.py @@ -41,13 +41,20 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( # Repeating a committed recovery identity is not a fresh admission. rc, replay = _run_cli(registry, runtime, *guard, "--turn-instance-id", "recovery-2") assert rc == 0 and replay["should_run"] is True, replay - for turn in ("wait-1", "wait-2"): - rc, wait = _run_cli(registry, runtime, *guard, "--turn-instance-id", turn) + for turn, detail in (("wait-1", ()), ("wait-2", ("--include-detail", "scheduler"))): + rc, wait = _run_cli(registry, runtime, *guard, "--turn-instance-id", turn, *detail) assert rc == 0 and wait["should_run"] is False, wait assert wait["reason_code"] == "unsettled_recovery_admission_limit" assert wait["unsettled_host_turn_recovery"]["state"] == "attention_required" assert wait["unsettled_host_turn_recovery"]["admissions_without_progress"] == 3 - assert wait["scheduler_hint"]["action"] != "run_now" + hint = wait["scheduler_hint"] + assert hint["action"] == "backoff_waiting_for_user" + if detail: + scheduler = hint["cold_path_detail"]["local_scheduler"] + assert scheduler["recommended_interval_minutes"] > 0 + assert hint["unchanged_poll"]["limits"]["local_scheduler"] == 3 + else: + assert "cold_path_detail" not in hint channel = wait["interaction_contract"]["agent_channel"] assert channel["must_attempt"] is False and channel["delivery_allowed"] is False assert wait["interaction_contract"]["cli_channel"]["spend_allowed_now"] is False From c28bccb2256c9caa754307b7ab3e6079e14548ef Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:20:50 +0800 Subject: [PATCH 2/2] Keep App scheduler reconciliation executable during recovery waits Reuse observed host cadence and persisted backoff when projecting a prior-Turn recovery wait. Preserve the registered scheduler scope and place CLI options after the standalone interpreter prefix so the matched host can be acknowledged using the original receipt. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/control_plane/quota/live_decision.py | 65 +++++++++++--- loopx/control_plane/quota/should_run.py | 6 +- .../quota/unsettled_host_turn.py | 21 ++--- loopx/quota.py | 2 + .../test_scheduler_fallback_hint.py | 30 +++++-- ...test_unsettled_recovery_admission_bound.py | 84 +++++++++++++++++++ 6 files changed, 177 insertions(+), 31 deletions(-) diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index 9d85bb3bfb..fde3e9c491 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -3,12 +3,15 @@ import shlex from collections.abc import Callable, Mapping, Sequence +from functools import partial from pathlib import Path from typing import Any from ...quota import build_quota_should_run from ...agent_registry import load_goal_from_registry, registered_agent_ids_for_goal +from ...command_invocation import loopx_cli_argv from ..agent_context import project_agent_context, project_goal_agent_context +from ..agents.identity import build_quota_agent_identity from ..runtime.time import now_utc_iso from ..goals.activation import goal_owner_resume_id from ..goals.goal_frontier.fallback_source import live_fallback_authority_items @@ -34,10 +37,13 @@ replan_obligation_id_from_packet, ) from ..todos.contract import normalize_todo_id +from ..scheduler.arbitration import SchedulerArbitration from ..scheduler.execution_context import ( SchedulerExecutionContextResolution, resolve_scheduler_execution_context, ) +from ..scheduler.scheduler_hint import build_scheduler_hint +from ..scheduler.state import load_app_automation_scheduler_state from .unsettled_host_turn import ( apply_unsettled_host_turn_recovery_if_required, apply_receipt_bound_wait_recovery, @@ -486,6 +492,7 @@ def bind_scheduler_followup_cli_routes( scheduler_hint = payload.get("scheduler_hint") if not isinstance(scheduler_hint, dict): return + launcher_args = loopx_cli_argv()[1:] app_packets = [ packet for packet_key in ("app_automation", "codex_app") @@ -532,13 +539,13 @@ def bind_scheduler_followup_cli_routes( } continue bound_cli_args = list(cli_args) - if bound_cli_args[0] != "--registry": - bound_cli_args = [ - "--registry", - str(registry_path.expanduser().resolve()), - "--runtime-root", - str(runtime_root.expanduser().resolve()), - *bound_cli_args, + # Standalone Core emits an interpreter route. Its launcher + # arguments must precede the CLI's global options. + option_index = len(launcher_args) if cli_args[:len(launcher_args)] == launcher_args else 0 + if bound_cli_args[option_index:option_index + 1] != ["--registry"]: + bound_cli_args[option_index:option_index] = [ + "--registry", str(registry_path.expanduser().resolve()), + "--runtime-root", str(runtime_root.expanduser().resolve()), ] safe_turn_instance_id = str(turn_instance_id or "").strip() if safe_turn_instance_id and "--turn-instance-id" not in bound_cli_args: @@ -737,7 +744,7 @@ def build_live_quota_should_run_decision( if remote_config is not None: # Use the same hard-pause precedence and keep fresh local selection # outside the remote route, even when a TLS read succeeds. - return _build_remote_authority_wait_decision( + payload = _build_remote_authority_wait_decision( decision_status_payload, configured_goal=registered_goal or {}, goal_id=goal_id, agent_id=agent_id, remote_config=remote_config, registry_path=registry_path, runtime_root=runtime_root, goal_ref=goal_ref, @@ -747,6 +754,11 @@ def build_live_quota_should_run_decision( turn_instance_id=turn_instance_id, receipt_bound_todo_id=receipt_bound_todo_id, receipt_bound_replan_obligation_id=receipt_bound_replan_obligation_id, ) + bind_scheduler_followup_cli_routes( + payload, registry_path=registry_path, runtime_root=runtime_root, + turn_instance_id=turn_instance_id, source=route_source, + ) + return payload settlement_readback = read_heartbeat_settlement( runtime_root, goal_id=goal_id, @@ -831,6 +843,13 @@ def build_live_quota_should_run_decision( goal_id=goal_id, agent_id=agent_id, ) + app_scheduler_state: dict[str, Any] | None = None + + def observe_app_scheduler_state(readback: Mapping[str, Any]) -> None: + # A later recovery wait uses the same state this decision actually read. + nonlocal app_scheduler_state + app_scheduler_state = readback.get("state") + payload: dict[str, Any] = build_quota_should_run( decision_status_payload, goal_id=goal_id, @@ -858,6 +877,7 @@ def build_live_quota_should_run_decision( **receipt_bound_guard_fields, turn_instance_id=turn_instance_id, runtime_root=runtime_root, + app_scheduler_state_observer=observe_app_scheduler_state, workspace_path=workspace_path, goal_ref=goal_ref, authoritative_fallback_todo_items=authoritative_fallback_todo_items, @@ -928,7 +948,14 @@ def build_live_quota_should_run_decision( agent_id=agent_id, current_turn_instance_id=turn_instance_id, available_capabilities=available_capabilities, - include_scheduler_detail=include_scheduler_detail, + project_scheduler_hint=partial( + build_scheduler_hint, include_detail=include_scheduler_detail, + scheduler_execution_context=resolved_context, + available_capabilities=available_capabilities, + codex_app_scheduler_state=app_scheduler_state, + codex_app_current_rrule=observed_rrule, + codex_app_automation_id=observed_automation_id or None, + ), scheduler_execution_context=resolved_context, goal_ref=goal_ref, owner_action_id=goal_owner_resume_id(registered_goal), @@ -1051,6 +1078,24 @@ def _build_remote_authority_wait_decision( } if goal_ref is not None: payload["goal_ref"] = dict(goal_ref) + + def project_recovery_scheduler_hint( + decision: dict[str, Any], *, arbitration: SchedulerArbitration, + ) -> dict[str, Any]: + decision["agent_identity"] = build_quota_agent_identity(configured_goal, agent_id=agent_id) + host = scheduler_context.context + state = None + if scheduler_context.ok and host is not None and host.app_automation_applicable: + state = load_app_automation_scheduler_state( + runtime_root, goal_id=goal_id, agent_id=agent_id, surface=host.host_surface.value, + ) + return build_scheduler_hint( + decision, arbitration=arbitration, include_detail=include_scheduler_detail, + available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, + codex_app_scheduler_state=state, codex_app_current_rrule=codex_app_rrule, + codex_app_automation_id=codex_app_automation_id, + ) + if scope_matches and not settled: if (receipt_bound_todo_id and agent_id and turn_instance_id and readback is not None and readback.replay_phase is ReceiptBoundReplayPhase.SETTLEMENT_PENDING): @@ -1069,7 +1114,7 @@ def _build_remote_authority_wait_decision( payload, registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id, agent_id=agent_id, current_turn_instance_id=turn_instance_id, available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, - include_scheduler_detail=include_scheduler_detail, + project_scheduler_hint=project_recovery_scheduler_hint, goal_ref=goal_ref, owner_action_id=goal_owner_resume_id(configured_goal), ): diff --git a/loopx/control_plane/quota/should_run.py b/loopx/control_plane/quota/should_run.py index 1f85af9df8..962f4540e7 100644 --- a/loopx/control_plane/quota/should_run.py +++ b/loopx/control_plane/quota/should_run.py @@ -194,6 +194,7 @@ def _attach_quota_should_run_envelope( def _app_scheduler_state_read_observer( direct_sources: dict[str, dict[str, Any]], + observe_state: Callable[[Mapping[str, Any]], None] | None, ) -> Callable[[Mapping[str, Any]], None]: def record(readback: Mapping[str, Any]) -> None: previous = direct_sources.get("app_scheduler_state", {}) @@ -211,6 +212,8 @@ def record(readback: Mapping[str, Any]) -> None: item_count=item_count, unreadable_count=unreadable_count, ) + if observe_state is not None: + observe_state(readback) return record @@ -577,6 +580,7 @@ def build_quota_should_run( goal_ref: Mapping[str, object] | None = None, authoritative_fallback_todo_items: FallbackTodoSource = None, replan_explore_reader: Callable[[str, str | None], Mapping[str, Any] | None] | None = None, + app_scheduler_state_observer: Callable[[Mapping[str, Any]], None] | None = None, ) -> dict[str, Any]: safe_goal_id = str(goal_id or "").strip() direct_sources: dict[str, dict[str, Any]] = {} @@ -687,7 +691,7 @@ def build_quota_should_run( runtime_root=runtime_root, goal_ref=goal_ref, scheduler_state_read_observer=_app_scheduler_state_read_observer( - direct_sources + direct_sources, app_scheduler_state_observer, ), ) _apply_missing_state_health_guidance( diff --git a/loopx/control_plane/quota/unsettled_host_turn.py b/loopx/control_plane/quota/unsettled_host_turn.py index 0d24776d01..eda16f9d2c 100644 --- a/loopx/control_plane/quota/unsettled_host_turn.py +++ b/loopx/control_plane/quota/unsettled_host_turn.py @@ -22,6 +22,7 @@ EffectRuntimeResponseAmbiguous, effect_runtime_result, ) +from ..scheduler.arbitration import SchedulerArbitration from ..scheduler.execution_context import SchedulerExecutionContextResolution from ..agents.agent_scope_frontier import AgentScopeFrontierAction from ..todos.todo_semantics import todo_item_task_class @@ -410,7 +411,7 @@ def apply_unsettled_host_turn_recovery_if_required( agent_id: str | None, current_turn_instance_id: str | None, available_capabilities: list[str] | None, - include_scheduler_detail: bool, + project_scheduler_hint: Callable[..., dict[str, Any]], scheduler_execution_context: ( Mapping[str, Any] | SchedulerExecutionContextResolution | None ), @@ -433,12 +434,14 @@ def apply_unsettled_host_turn_recovery_if_required( ) if verdict is None: return False - return _apply_recovery_projection( + arbitration = _apply_recovery_projection( payload, verdict=verdict, registry_path=registry_path, runtime_root=runtime_root, current_turn_instance_id=current_turn_instance_id, available_capabilities=available_capabilities, scheduler_execution_context=scheduler_execution_context, - include_scheduler_detail=include_scheduler_detail, ) + if verdict["recovery"].get("state") == "attention_required": + payload["scheduler_hint"] = project_scheduler_hint(payload, arbitration=arbitration) + return True def apply_receipt_bound_wait_recovery( @@ -529,8 +532,7 @@ def _apply_recovery_projection( payload: dict[str, Any], *, verdict: Mapping[str, Any], registry_path: Path, runtime_root: Path, current_turn_instance_id: str | None, available_capabilities: list[str] | None, scheduler_execution_context: Mapping[str, Any] | SchedulerExecutionContextResolution | None, - include_scheduler_detail: bool = False, -) -> bool: +) -> SchedulerArbitration: recovery = verdict["recovery"] obligation = verdict["obligation"] attention = recovery.get("state") == "attention_required" @@ -619,11 +621,4 @@ def _apply_recovery_projection( cli_channel["spend_allowed_now"] = obligation.get("spend_allowed_now") is True cli_channel["recovery_ref"] = "$.unsettled_host_turn_recovery" payload["interaction_contract"] = interaction_contract - if attention: - from ..scheduler.scheduler_hint import build_scheduler_hint - - payload["scheduler_hint"] = build_scheduler_hint( - payload, arbitration=arbitration, scheduler_execution_context=scheduler_execution_context, - available_capabilities=available_capabilities, include_detail=include_scheduler_detail, - ) - return True + return arbitration diff --git a/loopx/quota.py b/loopx/quota.py index 68f27bd509..4d774c4980 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1088,6 +1088,7 @@ def build_quota_should_run( workspace_path: Path | None = None, goal_ref: Mapping[str, object] | None = None, authoritative_fallback_todo_items: FallbackTodoSource = None, + app_scheduler_state_observer: Callable[[Mapping[str, Any]], None] | None = None, ) -> dict[str, Any]: from .control_plane.quota.should_run import ( build_quota_should_run as _build_quota_should_run, @@ -1134,6 +1135,7 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None) -> dict[s goal_ref=goal_ref, authoritative_fallback_todo_items=authoritative_fallback_todo_items, replan_explore_reader=read_explore_context, + app_scheduler_state_observer=app_scheduler_state_observer, ) diff --git a/tests/control_plane/test_scheduler_fallback_hint.py b/tests/control_plane/test_scheduler_fallback_hint.py index 2c20e5eeed..1ef74314e8 100644 --- a/tests/control_plane/test_scheduler_fallback_hint.py +++ b/tests/control_plane/test_scheduler_fallback_hint.py @@ -1,8 +1,12 @@ from __future__ import annotations +import sys from datetime import datetime, timezone from pathlib import Path +import pytest + +from loopx.command_invocation import loopx_cli_route from loopx.control_plane.heartbeat.rules import ( SCHEDULER_HINT_APPLICATION_RULE, SCHEDULER_HINT_COMPACT_RULE, @@ -345,12 +349,16 @@ def test_fallback_hint_keeps_a_concrete_projected_turn(tmp_path: Path) -> None: assert fallback["route_binding"]["turn_instance_bound"] is False -def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None: +@pytest.mark.parametrize("launcher", [(), ("-I", "-m", "loopx.entrypoint")]) +def test_scheduler_followup_routes_preserve_turn_lineage( + tmp_path: Path, launcher: tuple[str, ...], +) -> None: payload = { "scheduler_hint": { "app_automation": { "ack_hint": { "cli_args": [ + *launcher, "quota", "scheduler-ack-current", "--execute", @@ -359,6 +367,7 @@ def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None }, "failure_hint": { "cli_args": [ + *launcher, "quota", "scheduler-fail-current", "--execute", @@ -369,16 +378,23 @@ def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None } turn_instance_id = "turn-scheduler-followup-001" - bind_scheduler_followup_cli_routes( - payload, - registry_path=tmp_path / "registry.json", - runtime_root=tmp_path / "runtime", - turn_instance_id=turn_instance_id, - ) + with loopx_cli_route(None, default_argv=[sys.executable, *launcher]): + bind_scheduler_followup_cli_routes( + payload, + registry_path=tmp_path / "registry.json", + runtime_root=tmp_path / "runtime", + turn_instance_id=turn_instance_id, + ) app_automation = payload["scheduler_hint"]["app_automation"] for hint_name in ("ack_hint", "failure_hint"): hint = app_automation[hint_name] + assert hint["cli_args"][:len(launcher) + 5] == [ + *launcher, + "--registry", str((tmp_path / "registry.json").resolve()), + "--runtime-root", str((tmp_path / "runtime").resolve()), + "quota", + ] assert hint["cli_args"][-3:] == [ "--turn-instance-id", turn_instance_id, diff --git a/tests/control_plane/test_unsettled_recovery_admission_bound.py b/tests/control_plane/test_unsettled_recovery_admission_bound.py index 2629b800e3..e75dfefa25 100644 --- a/tests/control_plane/test_unsettled_recovery_admission_bound.py +++ b/tests/control_plane/test_unsettled_recovery_admission_bound.py @@ -1,4 +1,5 @@ """Recovery admissions are bounded per original debt, never treated as spend.""" +import json from concurrent.futures import ThreadPoolExecutor from pathlib import Path from threading import Barrier @@ -115,6 +116,89 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( assert _spend_run_count(runtime) == 0 +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("remote_binding", [False, True]) +def test_recovery_wait_keeps_observed_app_schedule_and_acknowledgement( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, remote_binding: bool, +) -> None: + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path, provider=provider) + codex_home = tmp_path / "codex" + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + monkeypatch.setenv("CODEX_THREAD_ID", "recovery-thread") + automation = codex_home / "automations" / "recovery-heartbeat" / "automation.toml" + automation.parent.mkdir(parents=True) + + def set_host_rrule(rrule: str) -> None: + automation.write_text( + 'version = 1\nid = "recovery-heartbeat"\nkind = "heartbeat"\n' + 'name = "Recovery schedule fixture"\nstatus = "ACTIVE"\n' + f'prompt = "Advance `{GOAL_ID}` from active state. Agent: `{AGENT_ID}`."\n' + f'rrule = "{rrule}"\ntarget_thread_id = "recovery-thread"\n', + encoding="utf-8", + ) + + def guard(turn: str, *extra: str) -> dict: + rc, result = _run_cli( + registry, runtime, "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--turn-instance-id", turn, *extra, cwd=project, + ) + assert rc == 0, result + return result + + matching_rrule = "FREQ=MINUTELY;INTERVAL=30" + short_rrule = "FREQ=MINUTELY;INTERVAL=3" + set_host_rrule(matching_rrule) + original = guard("owed-original", "--todo-id", TODO_ID) + assert original["heartbeat_receipt"]["closeout_required"] + for index in range(3): + assert guard(f"recovery-{index}")["should_run"] + + if remote_binding: + # A local recovery receipt still owns closeout after the Goal opts into + # the remote route. This fixture has no remote credentials or endpoint. + document = json.loads(registry.read_text(encoding="utf-8")) + goal, = (goal for goal in document["goals"] if goal["id"] == GOAL_ID) + goal["remote_authority"] = {"enabled": True} + registry.write_text(json.dumps(document), encoding="utf-8") + + wait = guard("matching-wait", "--include-detail", "scheduler") + assert wait["should_run"] is False + assert wait["reason_code"] == "unsettled_recovery_admission_limit" + hint = wait["scheduler_hint"] + assert hint["cold_path_detail"]["local_scheduler"]["recommended_interval_minutes"] == 30 + app = hint["app_automation"] + backoff = app["stateful_backoff"] + assert backoff["host_observation"]["current_rrule"] == matching_rrule + assert backoff["host_observation"]["status"] == "matches_recommended" + assert backoff["apply_needed"] is False and backoff["ack_needed"] is True + assert app["host_action"] == "ack_observed_rrule_without_update" + ack_hint = app["ack_hint"] + assert ack_hint["route_binding"]["turn_instance_bound"] is True + assert ack_hint["args"]["turn_instance_id"] == "matching-wait" + host_config = automation.read_bytes() + rc, ack = _run_cli(registry, runtime, *ack_hint["cli_args"], cwd=project) + assert rc == 0 and ack["scheduler_state_mutated"] is True, ack + assert automation.read_bytes() == host_config + + replay = guard("matching-wait") + assert replay["heartbeat_receipt"]["event_id"] == wait["heartbeat_receipt"]["event_id"] + app = replay["scheduler_hint"]["app_automation"] + backoff = app["stateful_backoff"] + assert backoff["apply_needed"] is False and backoff["ack_needed"] is False + assert app["host_action"] == "none" + assert "cold_path_detail" not in replay["scheduler_hint"] + + set_host_rrule(short_rrule) + drifted = guard("drifted-wait")["scheduler_hint"]["app_automation"] + assert drifted["stateful_backoff"]["host_observation"]["current_rrule"] == short_rrule + assert drifted["stateful_backoff"]["host_observation"]["status"] == "drift_detected" + assert drifted["host_action"] == "update_current_heartbeat_rrule" + assert drifted["recommended_rrule"] == matching_rrule + assert _spend_run_count(runtime) == 0 + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize("intermediate_guard", [False, True]) def test_owner_resume_survives_controller_lifecycle(