Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 55 additions & 8 deletions loopx/control_plane/quota/live_decision.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,15 @@

import shlex
from collections.abc import Callable, Mapping, Sequence
from functools import partial
from pathlib import Path
from typing import Any

from ...quota import build_quota_should_run
from ...agent_registry import load_goal_from_registry, registered_agent_ids_for_goal
from ...command_invocation import loopx_cli_argv
from ..agent_context import project_agent_context, project_goal_agent_context
from ..agents.identity import build_quota_agent_identity
from ..runtime.time import now_utc_iso
from ..goals.activation import goal_owner_resume_id
from ..goals.goal_frontier.fallback_source import live_fallback_authority_items
Expand All @@ -34,10 +37,13 @@
replan_obligation_id_from_packet,
)
from ..todos.contract import normalize_todo_id
from ..scheduler.arbitration import SchedulerArbitration
from ..scheduler.execution_context import (
SchedulerExecutionContextResolution,
resolve_scheduler_execution_context,
)
from ..scheduler.scheduler_hint import build_scheduler_hint
from ..scheduler.state import load_app_automation_scheduler_state
from .unsettled_host_turn import (
apply_unsettled_host_turn_recovery_if_required,
apply_receipt_bound_wait_recovery,
Expand Down Expand Up @@ -486,6 +492,7 @@ def bind_scheduler_followup_cli_routes(
scheduler_hint = payload.get("scheduler_hint")
if not isinstance(scheduler_hint, dict):
return
launcher_args = loopx_cli_argv()[1:]
app_packets = [
packet
for packet_key in ("app_automation", "codex_app")
Expand Down Expand Up @@ -532,13 +539,13 @@ def bind_scheduler_followup_cli_routes(
}
continue
bound_cli_args = list(cli_args)
if bound_cli_args[0] != "--registry":
bound_cli_args = [
"--registry",
str(registry_path.expanduser().resolve()),
"--runtime-root",
str(runtime_root.expanduser().resolve()),
*bound_cli_args,
# Standalone Core emits an interpreter route. Its launcher
# arguments must precede the CLI's global options.
option_index = len(launcher_args) if cli_args[:len(launcher_args)] == launcher_args else 0
if bound_cli_args[option_index:option_index + 1] != ["--registry"]:
bound_cli_args[option_index:option_index] = [
"--registry", str(registry_path.expanduser().resolve()),
"--runtime-root", str(runtime_root.expanduser().resolve()),
]
safe_turn_instance_id = str(turn_instance_id or "").strip()
if safe_turn_instance_id and "--turn-instance-id" not in bound_cli_args:
Expand Down Expand Up @@ -737,7 +744,7 @@ def build_live_quota_should_run_decision(
if remote_config is not None:
# Use the same hard-pause precedence and keep fresh local selection
# outside the remote route, even when a TLS read succeeds.
return _build_remote_authority_wait_decision(
payload = _build_remote_authority_wait_decision(
decision_status_payload, configured_goal=registered_goal or {},
goal_id=goal_id, agent_id=agent_id, remote_config=remote_config,
registry_path=registry_path, runtime_root=runtime_root, goal_ref=goal_ref,
Expand All @@ -747,6 +754,11 @@ def build_live_quota_should_run_decision(
turn_instance_id=turn_instance_id, receipt_bound_todo_id=receipt_bound_todo_id,
receipt_bound_replan_obligation_id=receipt_bound_replan_obligation_id,
)
bind_scheduler_followup_cli_routes(
payload, registry_path=registry_path, runtime_root=runtime_root,
turn_instance_id=turn_instance_id, source=route_source,
)
return payload
settlement_readback = read_heartbeat_settlement(
runtime_root,
goal_id=goal_id,
Expand Down Expand Up @@ -831,6 +843,13 @@ def build_live_quota_should_run_decision(
goal_id=goal_id,
agent_id=agent_id,
)
app_scheduler_state: dict[str, Any] | None = None

def observe_app_scheduler_state(readback: Mapping[str, Any]) -> None:
# A later recovery wait uses the same state this decision actually read.
nonlocal app_scheduler_state
app_scheduler_state = readback.get("state")

payload: dict[str, Any] = build_quota_should_run(
decision_status_payload,
goal_id=goal_id,
Expand Down Expand Up @@ -858,6 +877,7 @@ def build_live_quota_should_run_decision(
**receipt_bound_guard_fields,
turn_instance_id=turn_instance_id,
runtime_root=runtime_root,
app_scheduler_state_observer=observe_app_scheduler_state,
workspace_path=workspace_path,
goal_ref=goal_ref,
authoritative_fallback_todo_items=authoritative_fallback_todo_items,
Expand Down Expand Up @@ -928,6 +948,14 @@ def build_live_quota_should_run_decision(
agent_id=agent_id,
current_turn_instance_id=turn_instance_id,
available_capabilities=available_capabilities,
project_scheduler_hint=partial(
build_scheduler_hint, include_detail=include_scheduler_detail,
scheduler_execution_context=resolved_context,
available_capabilities=available_capabilities,
codex_app_scheduler_state=app_scheduler_state,
codex_app_current_rrule=observed_rrule,
codex_app_automation_id=observed_automation_id or None,
),
scheduler_execution_context=resolved_context,
goal_ref=goal_ref,
owner_action_id=goal_owner_resume_id(registered_goal),
Expand Down Expand Up @@ -1050,6 +1078,24 @@ def _build_remote_authority_wait_decision(
}
if goal_ref is not None:
payload["goal_ref"] = dict(goal_ref)

def project_recovery_scheduler_hint(
decision: dict[str, Any], *, arbitration: SchedulerArbitration,
) -> dict[str, Any]:
decision["agent_identity"] = build_quota_agent_identity(configured_goal, agent_id=agent_id)
host = scheduler_context.context
state = None
if scheduler_context.ok and host is not None and host.app_automation_applicable:
state = load_app_automation_scheduler_state(
runtime_root, goal_id=goal_id, agent_id=agent_id, surface=host.host_surface.value,
)
return build_scheduler_hint(
decision, arbitration=arbitration, include_detail=include_scheduler_detail,
available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context,
codex_app_scheduler_state=state, codex_app_current_rrule=codex_app_rrule,
codex_app_automation_id=codex_app_automation_id,
)

if scope_matches and not settled:
if (receipt_bound_todo_id and agent_id and turn_instance_id and readback is not None
and readback.replay_phase is ReceiptBoundReplayPhase.SETTLEMENT_PENDING):
Expand All @@ -1068,6 +1114,7 @@ def _build_remote_authority_wait_decision(
payload, registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id,
agent_id=agent_id, current_turn_instance_id=turn_instance_id,
available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context,
project_scheduler_hint=project_recovery_scheduler_hint,
goal_ref=goal_ref,
owner_action_id=goal_owner_resume_id(configured_goal),
):
Expand Down
6 changes: 5 additions & 1 deletion loopx/control_plane/quota/should_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ def _attach_quota_should_run_envelope(

def _app_scheduler_state_read_observer(
direct_sources: dict[str, dict[str, Any]],
observe_state: Callable[[Mapping[str, Any]], None] | None,
) -> Callable[[Mapping[str, Any]], None]:
def record(readback: Mapping[str, Any]) -> None:
previous = direct_sources.get("app_scheduler_state", {})
Expand All @@ -211,6 +212,8 @@ def record(readback: Mapping[str, Any]) -> None:
item_count=item_count,
unreadable_count=unreadable_count,
)
if observe_state is not None:
observe_state(readback)

return record

Expand Down Expand Up @@ -577,6 +580,7 @@ def build_quota_should_run(
goal_ref: Mapping[str, object] | None = None,
authoritative_fallback_todo_items: FallbackTodoSource = None,
replan_explore_reader: Callable[[str, str | None], Mapping[str, Any] | None] | None = None,
app_scheduler_state_observer: Callable[[Mapping[str, Any]], None] | None = None,
) -> dict[str, Any]:
safe_goal_id = str(goal_id or "").strip()
direct_sources: dict[str, dict[str, Any]] = {}
Expand Down Expand Up @@ -687,7 +691,7 @@ def build_quota_should_run(
runtime_root=runtime_root,
goal_ref=goal_ref,
scheduler_state_read_observer=_app_scheduler_state_read_observer(
direct_sources
direct_sources, app_scheduler_state_observer,
),
)
_apply_missing_state_health_guidance(
Expand Down
22 changes: 11 additions & 11 deletions loopx/control_plane/quota/unsettled_host_turn.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
EffectRuntimeResponseAmbiguous,
effect_runtime_result,
)
from ..scheduler.arbitration import SchedulerArbitration
from ..scheduler.execution_context import SchedulerExecutionContextResolution
from ..agents.agent_scope_frontier import AgentScopeFrontierAction
from ..todos.todo_semantics import todo_item_task_class
Expand Down Expand Up @@ -410,6 +411,7 @@ def apply_unsettled_host_turn_recovery_if_required(
agent_id: str | None,
current_turn_instance_id: str | None,
available_capabilities: list[str] | None,
project_scheduler_hint: Callable[..., dict[str, Any]],
scheduler_execution_context: (
Mapping[str, Any] | SchedulerExecutionContextResolution | None
),
Expand All @@ -432,9 +434,14 @@ def apply_unsettled_host_turn_recovery_if_required(
)
if verdict is None:
return False
return _apply_recovery_projection(payload, verdict=verdict, registry_path=registry_path, runtime_root=runtime_root,
arbitration = _apply_recovery_projection(
payload, verdict=verdict, registry_path=registry_path, runtime_root=runtime_root,
current_turn_instance_id=current_turn_instance_id, available_capabilities=available_capabilities,
scheduler_execution_context=scheduler_execution_context)
scheduler_execution_context=scheduler_execution_context,
)
if verdict["recovery"].get("state") == "attention_required":
payload["scheduler_hint"] = project_scheduler_hint(payload, arbitration=arbitration)
return True


def apply_receipt_bound_wait_recovery(
Expand Down Expand Up @@ -525,7 +532,7 @@ def _apply_recovery_projection(
payload: dict[str, Any], *, verdict: Mapping[str, Any], registry_path: Path, runtime_root: Path,
current_turn_instance_id: str | None, available_capabilities: list[str] | None,
scheduler_execution_context: Mapping[str, Any] | SchedulerExecutionContextResolution | None,
) -> bool:
) -> SchedulerArbitration:
recovery = verdict["recovery"]
obligation = verdict["obligation"]
attention = recovery.get("state") == "attention_required"
Expand Down Expand Up @@ -614,11 +621,4 @@ def _apply_recovery_projection(
cli_channel["spend_allowed_now"] = obligation.get("spend_allowed_now") is True
cli_channel["recovery_ref"] = "$.unsettled_host_turn_recovery"
payload["interaction_contract"] = interaction_contract
if attention:
from ..scheduler.scheduler_hint import build_scheduler_hint

payload["scheduler_hint"] = build_scheduler_hint(
payload, arbitration=arbitration, scheduler_execution_context=scheduler_execution_context,
available_capabilities=available_capabilities,
)
return True
return arbitration
2 changes: 2 additions & 0 deletions loopx/quota.py
Original file line number Diff line number Diff line change
Expand Up @@ -1088,6 +1088,7 @@ def build_quota_should_run(
workspace_path: Path | None = None,
goal_ref: Mapping[str, object] | None = None,
authoritative_fallback_todo_items: FallbackTodoSource = None,
app_scheduler_state_observer: Callable[[Mapping[str, Any]], None] | None = None,
) -> dict[str, Any]:
from .control_plane.quota.should_run import (
build_quota_should_run as _build_quota_should_run,
Expand Down Expand Up @@ -1134,6 +1135,7 @@ def read_explore_context(read_goal_id: str, read_agent_id: str | None) -> dict[s
goal_ref=goal_ref,
authoritative_fallback_todo_items=authoritative_fallback_todo_items,
replan_explore_reader=read_explore_context,
app_scheduler_state_observer=app_scheduler_state_observer,
)


Expand Down
30 changes: 23 additions & 7 deletions tests/control_plane/test_scheduler_fallback_hint.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
from __future__ import annotations

import sys
from datetime import datetime, timezone
from pathlib import Path

import pytest

from loopx.command_invocation import loopx_cli_route
from loopx.control_plane.heartbeat.rules import (
SCHEDULER_HINT_APPLICATION_RULE,
SCHEDULER_HINT_COMPACT_RULE,
Expand Down Expand Up @@ -345,12 +349,16 @@ def test_fallback_hint_keeps_a_concrete_projected_turn(tmp_path: Path) -> None:
assert fallback["route_binding"]["turn_instance_bound"] is False


def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None:
@pytest.mark.parametrize("launcher", [(), ("-I", "-m", "loopx.entrypoint")])
def test_scheduler_followup_routes_preserve_turn_lineage(
tmp_path: Path, launcher: tuple[str, ...],
) -> None:
payload = {
"scheduler_hint": {
"app_automation": {
"ack_hint": {
"cli_args": [
*launcher,
"quota",
"scheduler-ack-current",
"--execute",
Expand All @@ -359,6 +367,7 @@ def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None
},
"failure_hint": {
"cli_args": [
*launcher,
"quota",
"scheduler-fail-current",
"--execute",
Expand All @@ -369,16 +378,23 @@ def test_scheduler_followup_routes_preserve_turn_lineage(tmp_path: Path) -> None
}
turn_instance_id = "turn-scheduler-followup-001"

bind_scheduler_followup_cli_routes(
payload,
registry_path=tmp_path / "registry.json",
runtime_root=tmp_path / "runtime",
turn_instance_id=turn_instance_id,
)
with loopx_cli_route(None, default_argv=[sys.executable, *launcher]):
bind_scheduler_followup_cli_routes(
payload,
registry_path=tmp_path / "registry.json",
runtime_root=tmp_path / "runtime",
turn_instance_id=turn_instance_id,
)

app_automation = payload["scheduler_hint"]["app_automation"]
for hint_name in ("ack_hint", "failure_hint"):
hint = app_automation[hint_name]
assert hint["cli_args"][:len(launcher) + 5] == [
*launcher,
"--registry", str((tmp_path / "registry.json").resolve()),
"--runtime-root", str((tmp_path / "runtime").resolve()),
"quota",
]
assert hint["cli_args"][-3:] == [
"--turn-instance-id",
turn_instance_id,
Expand Down
Loading
Loading