diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index 4105e66a1a..fde3e9c491 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -3,12 +3,15 @@ import shlex from collections.abc import Callable, Mapping, Sequence +from functools import partial from pathlib import Path from typing import Any from ...quota import build_quota_should_run from ...agent_registry import load_goal_from_registry, registered_agent_ids_for_goal +from ...command_invocation import loopx_cli_argv from ..agent_context import project_agent_context, project_goal_agent_context +from ..agents.identity import build_quota_agent_identity from ..runtime.time import now_utc_iso from ..goals.activation import goal_owner_resume_id from ..goals.goal_frontier.fallback_source import live_fallback_authority_items @@ -34,10 +37,13 @@ replan_obligation_id_from_packet, ) from ..todos.contract import normalize_todo_id +from ..scheduler.arbitration import SchedulerArbitration from ..scheduler.execution_context import ( SchedulerExecutionContextResolution, resolve_scheduler_execution_context, ) +from ..scheduler.scheduler_hint import build_scheduler_hint +from ..scheduler.state import load_app_automation_scheduler_state from .unsettled_host_turn import ( apply_unsettled_host_turn_recovery_if_required, apply_receipt_bound_wait_recovery, @@ -486,6 +492,7 @@ def bind_scheduler_followup_cli_routes( scheduler_hint = payload.get("scheduler_hint") if not isinstance(scheduler_hint, dict): return + launcher_args = loopx_cli_argv()[1:] app_packets = [ packet for packet_key in ("app_automation", "codex_app") @@ -532,13 +539,13 @@ def bind_scheduler_followup_cli_routes( } continue bound_cli_args = list(cli_args) - if bound_cli_args[0] != "--registry": - bound_cli_args = [ - "--registry", - str(registry_path.expanduser().resolve()), - "--runtime-root", - str(runtime_root.expanduser().resolve()), - *bound_cli_args, + # Standalone Core emits an interpreter route. Its launcher + # arguments must precede the CLI's global options. + option_index = len(launcher_args) if cli_args[:len(launcher_args)] == launcher_args else 0 + if bound_cli_args[option_index:option_index + 1] != ["--registry"]: + bound_cli_args[option_index:option_index] = [ + "--registry", str(registry_path.expanduser().resolve()), + "--runtime-root", str(runtime_root.expanduser().resolve()), ] safe_turn_instance_id = str(turn_instance_id or "").strip() if safe_turn_instance_id and "--turn-instance-id" not in bound_cli_args: @@ -737,7 +744,7 @@ def build_live_quota_should_run_decision( if remote_config is not None: # Use the same hard-pause precedence and keep fresh local selection # outside the remote route, even when a TLS read succeeds. - return _build_remote_authority_wait_decision( + payload = _build_remote_authority_wait_decision( decision_status_payload, configured_goal=registered_goal or {}, goal_id=goal_id, agent_id=agent_id, remote_config=remote_config, registry_path=registry_path, runtime_root=runtime_root, goal_ref=goal_ref, @@ -747,6 +754,11 @@ def build_live_quota_should_run_decision( turn_instance_id=turn_instance_id, receipt_bound_todo_id=receipt_bound_todo_id, receipt_bound_replan_obligation_id=receipt_bound_replan_obligation_id, ) + bind_scheduler_followup_cli_routes( + payload, registry_path=registry_path, runtime_root=runtime_root, + turn_instance_id=turn_instance_id, source=route_source, + ) + return payload settlement_readback = read_heartbeat_settlement( runtime_root, goal_id=goal_id, @@ -831,6 +843,13 @@ def build_live_quota_should_run_decision( goal_id=goal_id, agent_id=agent_id, ) + app_scheduler_state: dict[str, Any] | None = None + + def observe_app_scheduler_state(readback: Mapping[str, Any]) -> None: + # A later recovery wait uses the same state this decision actually read. + nonlocal app_scheduler_state + app_scheduler_state = readback.get("state") + payload: dict[str, Any] = build_quota_should_run( decision_status_payload, goal_id=goal_id, @@ -858,6 +877,7 @@ def build_live_quota_should_run_decision( **receipt_bound_guard_fields, turn_instance_id=turn_instance_id, runtime_root=runtime_root, + app_scheduler_state_observer=observe_app_scheduler_state, workspace_path=workspace_path, goal_ref=goal_ref, authoritative_fallback_todo_items=authoritative_fallback_todo_items, @@ -928,6 +948,14 @@ def build_live_quota_should_run_decision( agent_id=agent_id, current_turn_instance_id=turn_instance_id, available_capabilities=available_capabilities, + project_scheduler_hint=partial( + build_scheduler_hint, include_detail=include_scheduler_detail, + scheduler_execution_context=resolved_context, + available_capabilities=available_capabilities, + codex_app_scheduler_state=app_scheduler_state, + codex_app_current_rrule=observed_rrule, + codex_app_automation_id=observed_automation_id or None, + ), scheduler_execution_context=resolved_context, goal_ref=goal_ref, owner_action_id=goal_owner_resume_id(registered_goal), @@ -1050,6 +1078,24 @@ def _build_remote_authority_wait_decision( } if goal_ref is not None: payload["goal_ref"] = dict(goal_ref) + + def project_recovery_scheduler_hint( + decision: dict[str, Any], *, arbitration: SchedulerArbitration, + ) -> dict[str, Any]: + decision["agent_identity"] = build_quota_agent_identity(configured_goal, agent_id=agent_id) + host = scheduler_context.context + state = None + if scheduler_context.ok and host is not None and host.app_automation_applicable: + state = load_app_automation_scheduler_state( + runtime_root, goal_id=goal_id, agent_id=agent_id, surface=host.host_surface.value, + ) + return build_scheduler_hint( + decision, arbitration=arbitration, include_detail=include_scheduler_detail, + available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, + codex_app_scheduler_state=state, codex_app_current_rrule=codex_app_rrule, + codex_app_automation_id=codex_app_automation_id, + ) + if scope_matches and not settled: if (receipt_bound_todo_id and agent_id and turn_instance_id and readback is not None and readback.replay_phase is ReceiptBoundReplayPhase.SETTLEMENT_PENDING): @@ -1068,6 +1114,7 @@ def _build_remote_authority_wait_decision( payload, registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id, agent_id=agent_id, current_turn_instance_id=turn_instance_id, available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, + project_scheduler_hint=project_recovery_scheduler_hint, goal_ref=goal_ref, owner_action_id=goal_owner_resume_id(configured_goal), ): diff --git a/loopx/control_plane/quota/should_run.py b/loopx/control_plane/quota/should_run.py index 1f85af9df8..962f4540e7 100644 --- a/loopx/control_plane/quota/should_run.py +++ b/loopx/control_plane/quota/should_run.py @@ -194,6 +194,7 @@ def _attach_quota_should_run_envelope( def _app_scheduler_state_read_observer( direct_sources: dict[str, dict[str, Any]], + observe_state: Callable[[Mapping[str, Any]], None] | None, ) -> Callable[[Mapping[str, Any]], None]: def record(readback: Mapping[str, Any]) -> None: previous = direct_sources.get("app_scheduler_state", {}) @@ -211,6 +212,8 @@ def record(readback: Mapping[str, Any]) -> None: item_count=item_count, unreadable_count=unreadable_count, ) + if observe_state is not None: + observe_state(readback) return record @@ -577,6 +580,7 @@ def build_quota_should_run( goal_ref: Mapping[str, object] | None = None, authoritative_fallback_todo_items: FallbackTodoSource = None, replan_explore_reader: Callable[[str, str | None], Mapping[str, Any] | None] | None = None, + app_scheduler_state_observer: Callable[[Mapping[str, Any]], None] | None = None, ) -> dict[str, Any]: safe_goal_id = str(goal_id or "").strip() direct_sources: dict[str, dict[str, Any]] = {} @@ -687,7 +691,7 @@ def build_quota_should_run( runtime_root=runtime_root, goal_ref=goal_ref, scheduler_state_read_observer=_app_scheduler_state_read_observer( - direct_sources + direct_sources, app_scheduler_state_observer, ), ) _apply_missing_state_health_guidance( diff --git a/loopx/control_plane/quota/unsettled_host_turn.py b/loopx/control_plane/quota/unsettled_host_turn.py index 44ee6885f7..eda16f9d2c 100644 --- a/loopx/control_plane/quota/unsettled_host_turn.py +++ b/loopx/control_plane/quota/unsettled_host_turn.py @@ -22,6 +22,7 @@ EffectRuntimeResponseAmbiguous, effect_runtime_result, ) +from ..scheduler.arbitration import SchedulerArbitration from ..scheduler.execution_context import SchedulerExecutionContextResolution from ..agents.agent_scope_frontier import AgentScopeFrontierAction from ..todos.todo_semantics import todo_item_task_class @@ -410,6 +411,7 @@ def apply_unsettled_host_turn_recovery_if_required( agent_id: str | None, current_turn_instance_id: str | None, available_capabilities: list[str] | None, + project_scheduler_hint: Callable[..., dict[str, Any]], scheduler_execution_context: ( Mapping[str, Any] | SchedulerExecutionContextResolution | None ), @@ -432,9 +434,14 @@ def apply_unsettled_host_turn_recovery_if_required( ) if verdict is None: return False - return _apply_recovery_projection(payload, verdict=verdict, registry_path=registry_path, runtime_root=runtime_root, + arbitration = _apply_recovery_projection( + payload, verdict=verdict, registry_path=registry_path, runtime_root=runtime_root, current_turn_instance_id=current_turn_instance_id, available_capabilities=available_capabilities, - scheduler_execution_context=scheduler_execution_context) + scheduler_execution_context=scheduler_execution_context, + ) + if verdict["recovery"].get("state") == "attention_required": + payload["scheduler_hint"] = project_scheduler_hint(payload, arbitration=arbitration) + return True def apply_receipt_bound_wait_recovery( @@ -525,7 +532,7 @@ def _apply_recovery_projection( payload: dict[str, Any], *, verdict: Mapping[str, Any], registry_path: Path, runtime_root: Path, current_turn_instance_id: str | None, available_capabilities: list[str] | None, scheduler_execution_context: Mapping[str, Any] | SchedulerExecutionContextResolution | None, -) -> bool: +) -> SchedulerArbitration: recovery = verdict["recovery"] obligation = verdict["obligation"] attention = recovery.get("state") == "attention_required" @@ -614,11 +621,4 @@ def _apply_recovery_projection( cli_channel["spend_allowed_now"] = obligation.get("spend_allowed_now") is True cli_channel["recovery_ref"] = "$.unsettled_host_turn_recovery" payload["interaction_contract"] = interaction_contract - if attention: - from ..scheduler.scheduler_hint import build_scheduler_hint - - payload["scheduler_hint"] = build_scheduler_hint( - payload, arbitration=arbitration, scheduler_execution_context=scheduler_execution_context, - available_capabilities=available_capabilities, - ) - return True + return arbitration diff --git a/loopx/quota.py b/loopx/quota.py index 68f27bd509..4d774c4980 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1088,6 +1088,7 @@ def build_quota_should_run( workspace_path: Path | None = None, goal_ref: Mapping[str, object] | None = None, authoritative_fallback_todo_items: FallbackTodoSource = None, + app_scheduler_state_observer: Callable[[Mapping[str, Any]], None] | None = None, ) -> dict[str, Any]: from .control_plane.quota.should_run import ( build_quota_should_run as _build_quota_should_run, @@ -1134,6 +1135,7 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None) -> dict[s goal_ref=goal_ref, authoritative_fallback_todo_items=authoritative_fallback_todo_items, replan_explore_reader=read_explore_context, + app_scheduler_state_observer=app_scheduler_state_observer, ) diff --git a/tests/control_plane/test_scheduler_fallback_hint.py b/tests/control_plane/test_scheduler_fallback_hint.py index 2c20e5eeed..1ef74314e8 100644 --- a/tests/control_plane/test_scheduler_fallback_hint.py +++ b/tests/control_plane/test_scheduler_fallback_hint.py @@ -1,8 +1,12 @@ from __future__ import annotations +import sys from datetime import datetime, timezone from pathlib import Path +import pytest + +from loopx.command_invocation import loopx_cli_route from loopx.control_plane.heartbeat.rules import ( SCHEDULER_HINT_APPLICATION_RULE, SCHEDULER_HINT_COMPACT_RULE, @@ -345,12 +349,16 @@ def test_fallback_hint_keeps_a_concrete_projected_turn(tmp_path: Path) -> None: assert fallback["route_binding"]["turn_instance_bound"] is False -def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None: +@pytest.mark.parametrize("launcher", [(), ("-I", "-m", "loopx.entrypoint")]) +def test_scheduler_followup_routes_preserve_turn_lineage( + tmp_path: Path, launcher: tuple[str, ...], +) -> None: payload = { "scheduler_hint": { "app_automation": { "ack_hint": { "cli_args": [ + *launcher, "quota", "scheduler-ack-current", "--execute", @@ -359,6 +367,7 @@ def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None }, "failure_hint": { "cli_args": [ + *launcher, "quota", "scheduler-fail-current", "--execute", @@ -369,16 +378,23 @@ def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None } turn_instance_id = "turn-scheduler-followup-001" - bind_scheduler_followup_cli_routes( - payload, - registry_path=tmp_path / "registry.json", - runtime_root=tmp_path / "runtime", - turn_instance_id=turn_instance_id, - ) + with loopx_cli_route(None, default_argv=[sys.executable, *launcher]): + bind_scheduler_followup_cli_routes( + payload, + registry_path=tmp_path / "registry.json", + runtime_root=tmp_path / "runtime", + turn_instance_id=turn_instance_id, + ) app_automation = payload["scheduler_hint"]["app_automation"] for hint_name in ("ack_hint", "failure_hint"): hint = app_automation[hint_name] + assert hint["cli_args"][:len(launcher) + 5] == [ + *launcher, + "--registry", str((tmp_path / "registry.json").resolve()), + "--runtime-root", str((tmp_path / "runtime").resolve()), + "quota", + ] assert hint["cli_args"][-3:] == [ "--turn-instance-id", turn_instance_id, diff --git a/tests/control_plane/test_unsettled_recovery_admission_bound.py b/tests/control_plane/test_unsettled_recovery_admission_bound.py index 2f1dad56b7..e75dfefa25 100644 --- a/tests/control_plane/test_unsettled_recovery_admission_bound.py +++ b/tests/control_plane/test_unsettled_recovery_admission_bound.py @@ -1,4 +1,5 @@ """Recovery admissions are bounded per original debt, never treated as spend.""" +import json from concurrent.futures import ThreadPoolExecutor from pathlib import Path from threading import Barrier @@ -41,13 +42,20 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( # Repeating a committed recovery identity is not a fresh admission. rc, replay = _run_cli(registry, runtime, *guard, "--turn-instance-id", "recovery-2") assert rc == 0 and replay["should_run"] is True, replay - for turn in ("wait-1", "wait-2"): - rc, wait = _run_cli(registry, runtime, *guard, "--turn-instance-id", turn) + for turn, detail in (("wait-1", ()), ("wait-2", ("--include-detail", "scheduler"))): + rc, wait = _run_cli(registry, runtime, *guard, "--turn-instance-id", turn, *detail) assert rc == 0 and wait["should_run"] is False, wait assert wait["reason_code"] == "unsettled_recovery_admission_limit" assert wait["unsettled_host_turn_recovery"]["state"] == "attention_required" assert wait["unsettled_host_turn_recovery"]["admissions_without_progress"] == 3 - assert wait["scheduler_hint"]["action"] != "run_now" + hint = wait["scheduler_hint"] + assert hint["action"] == "backoff_waiting_for_user" + if detail: + scheduler = hint["cold_path_detail"]["local_scheduler"] + assert scheduler["recommended_interval_minutes"] > 0 + assert hint["unchanged_poll"]["limits"]["local_scheduler"] == 3 + else: + assert "cold_path_detail" not in hint channel = wait["interaction_contract"]["agent_channel"] assert channel["must_attempt"] is False and channel["delivery_allowed"] is False assert wait["interaction_contract"]["cli_channel"]["spend_allowed_now"] is False @@ -108,6 +116,89 @@ def test_unchanged_recovery_waits_then_accepts_progress_or_owner_resume( assert _spend_run_count(runtime) == 0 +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("remote_binding", [False, True]) +def test_recovery_wait_keeps_observed_app_schedule_and_acknowledgement( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, remote_binding: bool, +) -> None: + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path, provider=provider) + codex_home = tmp_path / "codex" + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + monkeypatch.setenv("CODEX_THREAD_ID", "recovery-thread") + automation = codex_home / "automations" / "recovery-heartbeat" / "automation.toml" + automation.parent.mkdir(parents=True) + + def set_host_rrule(rrule: str) -> None: + automation.write_text( + 'version = 1\nid = "recovery-heartbeat"\nkind = "heartbeat"\n' + 'name = "Recovery schedule fixture"\nstatus = "ACTIVE"\n' + f'prompt = "Advance `{GOAL_ID}` from active state. Agent: `{AGENT_ID}`."\n' + f'rrule = "{rrule}"\ntarget_thread_id = "recovery-thread"\n', + encoding="utf-8", + ) + + def guard(turn: str, *extra: str) -> dict: + rc, result = _run_cli( + registry, runtime, "quota", "should-run", "--codex-app", + "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--turn-instance-id", turn, *extra, cwd=project, + ) + assert rc == 0, result + return result + + matching_rrule = "FREQ=MINUTELY;INTERVAL=30" + short_rrule = "FREQ=MINUTELY;INTERVAL=3" + set_host_rrule(matching_rrule) + original = guard("owed-original", "--todo-id", TODO_ID) + assert original["heartbeat_receipt"]["closeout_required"] + for index in range(3): + assert guard(f"recovery-{index}")["should_run"] + + if remote_binding: + # A local recovery receipt still owns closeout after the Goal opts into + # the remote route. This fixture has no remote credentials or endpoint. + document = json.loads(registry.read_text(encoding="utf-8")) + goal, = (goal for goal in document["goals"] if goal["id"] == GOAL_ID) + goal["remote_authority"] = {"enabled": True} + registry.write_text(json.dumps(document), encoding="utf-8") + + wait = guard("matching-wait", "--include-detail", "scheduler") + assert wait["should_run"] is False + assert wait["reason_code"] == "unsettled_recovery_admission_limit" + hint = wait["scheduler_hint"] + assert hint["cold_path_detail"]["local_scheduler"]["recommended_interval_minutes"] == 30 + app = hint["app_automation"] + backoff = app["stateful_backoff"] + assert backoff["host_observation"]["current_rrule"] == matching_rrule + assert backoff["host_observation"]["status"] == "matches_recommended" + assert backoff["apply_needed"] is False and backoff["ack_needed"] is True + assert app["host_action"] == "ack_observed_rrule_without_update" + ack_hint = app["ack_hint"] + assert ack_hint["route_binding"]["turn_instance_bound"] is True + assert ack_hint["args"]["turn_instance_id"] == "matching-wait" + host_config = automation.read_bytes() + rc, ack = _run_cli(registry, runtime, *ack_hint["cli_args"], cwd=project) + assert rc == 0 and ack["scheduler_state_mutated"] is True, ack + assert automation.read_bytes() == host_config + + replay = guard("matching-wait") + assert replay["heartbeat_receipt"]["event_id"] == wait["heartbeat_receipt"]["event_id"] + app = replay["scheduler_hint"]["app_automation"] + backoff = app["stateful_backoff"] + assert backoff["apply_needed"] is False and backoff["ack_needed"] is False + assert app["host_action"] == "none" + assert "cold_path_detail" not in replay["scheduler_hint"] + + set_host_rrule(short_rrule) + drifted = guard("drifted-wait")["scheduler_hint"]["app_automation"] + assert drifted["stateful_backoff"]["host_observation"]["current_rrule"] == short_rrule + assert drifted["stateful_backoff"]["host_observation"]["status"] == "drift_detected" + assert drifted["host_action"] == "update_current_heartbeat_rrule" + assert drifted["recommended_rrule"] == matching_rrule + assert _spend_run_count(runtime) == 0 + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) @pytest.mark.parametrize("intermediate_guard", [False, True]) def test_owner_resume_survives_controller_lifecycle(