Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions docs/quota-allocation.md
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,28 @@ instead sets `closeout_kind=typed_blocked_writeback_no_spend` and settles the
Turn without a quota debit. Todo completion and Goal acceptance retain their
separate checks in both cases.

Repeated recovery admissions for the same bound prior Turn are limited when
the missing receipts remain unchanged. The limit counts distinct committed
guard admissions. Replaying a guard or receiving a host-completed event leaves
that count as it was. Verified receipt progress or an explicit Owner resume
renews the allowance. The Goal activation record retains the last Owner
resume through later Controller Stop and Resume transitions. Resuming a
Goal that is already active leaves the recorded allowance in place.

Each new recovery grant checks its admission basis under the receipt append
lock. A contender with a stale basis reevaluates the same Turn. When an unbound
waiting Turn becomes eligible after resume, its first positive recovery
receipt is appended under the same lock and subsequent replays reuse it.

同一原始 Turn 的缺失回执没有减少时,恢复准入有次数上限,按已提交的不同 guard
准入计数。相同 guard 重放和宿主完成事件均保持原计数。验证到回执进展或 Owner
显式恢复 Goal 后重新计算。Goal 激活记录会保留最近一次 Owner 恢复,后续
Controller 停止或恢复均沿用该记录。对已启用的 Goal 再次恢复会保留既有额度。

新的恢复准入会在持有回执追加锁时复核准入依据。依据已变化的并发请求用同一 Turn
重新判定。未绑定的等待 Turn 在恢复后取得资格时,在同一把锁内追加首次准入回执,
后续重放复用该回执。

An admitted advancement Todo can also become persistently `blocked` through
its ordinary owner-authorized lifecycle. Its original Turn then offers the
same blocked writeback command. The settlement owner freezes the current
Expand Down
230 changes: 146 additions & 84 deletions loopx/cli_commands/quota.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
project_captured_context,
)
from ..usage_goal import observe_quota_result
from ..file_lock import LockAcquireTimeoutError
from ..rollout_event_log import RolloutAppendPreconditionChanged

from ..control_plane.coordination.local_authority import CanonicalAuthorityRequired
from ..control_plane.quota.effective_action import EffectiveAction
Expand All @@ -40,6 +42,7 @@
attach_uncommitted_heartbeat_receipt,
fail_heartbeat_receipt,
find_heartbeat_receipt,
heartbeat_receipt_request_subject,
heartbeat_receipt_view,
)
from .quota_registration import (
Expand Down Expand Up @@ -671,6 +674,123 @@ def execute() -> int:
return run_with_quota_failure_reader(args, limit, print_payload=print_payload, run=execute)


def _append_quota_heartbeat_receipt(
payload: dict[str, object],
args: argparse.Namespace,
*,
registry_path: Path,
runtime_root_arg: str | None,
runtime_root: Path,
goal_ref: dict[str, str] | None,
append_cli_rollout_event: RolloutEventAppender,
turn_instance_id: str,
stall_observation: str,
rollout_todo_id: str | None,
rollout_replan_obligation_id: str | None,
rollout_details: dict[str, object],
upgrade_recovery_wait: bool,
requested_todo_id: str | None,
) -> dict[str, object]:
from ..control_plane.quota.unsettled_host_turn import recovery_admission_append_precondition

settlement_identity = (
SettlementIdentity(
goal_id=args.goal_id,
agent_id=args.agent_id,
todo_id=rollout_todo_id,
turn_instance_id=turn_instance_id,
replan_obligation_id=rollout_replan_obligation_id,
)
if rollout_todo_id or rollout_replan_obligation_id
else None
)
rollout_details.update(
{
"turn_instance_id": turn_instance_id,
"stall_observation": stall_observation,
"settlement_effect_id": settlement_identity.effect_id if settlement_identity is not None else "",
}
)
try:
with recovery_admission_append_precondition(
payload, registry_path=registry_path, runtime_root=runtime_root,
goal_id=args.goal_id, agent_id=args.agent_id,
turn_instance_id=turn_instance_id, goal_ref=goal_ref,
) as precondition:
append_cli_rollout_event(
payload,
registry_path=registry_path,
runtime_root_arg=runtime_root_arg,
event_kind="quota_should_run",
goal_ref=goal_ref,
agent_id=args.agent_id,
run_id=turn_instance_id,
status=str(payload.get("effective_action") or payload.get("decision") or "should-run"),
summary=f"heartbeat quota receipt committed for turn={turn_instance_id} stall={stall_observation}",
details=rollout_details,
allow_failed=True,
idempotency_fields=[
"goal_id",
"event_kind",
"agent_id",
"run_id",
*(["goal_ref"] if goal_ref is not None else []),
*(["status"] if upgrade_recovery_wait else []),
],
**({"precondition": precondition} if precondition is not None else {}),
)
receipt = find_heartbeat_receipt(
runtime_root, goal_id=args.goal_id, agent_id=args.agent_id,
turn_instance_id=turn_instance_id, goal_ref=goal_ref,
)
except LockAcquireTimeoutError as exc:
failure = _export("quota_failure_payload")(
args, registry_path=registry_path, runtime_root_arg=runtime_root_arg, error=exc,
)
fail_heartbeat_receipt(
failure, turn_instance_id=turn_instance_id, stall_observation=stall_observation,
reason="recovery receipt append could not acquire the quota authority lock",
)
return failure
if receipt is None:
fail_heartbeat_receipt(
payload, turn_instance_id=turn_instance_id, stall_observation=stall_observation,
reason="heartbeat receipt append could not be read back; retry quota should-run with the same --turn-instance-id",
)
return payload
if precondition is not None and receipt.get("status") != EffectiveAction.UNSETTLED_HOST_TURN_RECOVERY.value:
raise RolloutAppendPreconditionChanged("A different guard was recorded for this Turn.")

rollout_event = payload.get("rollout_event")
status = "replayed"
if isinstance(rollout_event, Mapping) and rollout_event.get("appended"):
status = "upgraded" if upgrade_recovery_wait else "committed"
payload["heartbeat_receipt"] = heartbeat_receipt_view(receipt, turn_instance_id=turn_instance_id, status=status)
_export("commit_requested_action_selection")(payload, requested_todo_id=requested_todo_id)
return payload


def _can_upgrade_recovery_wait(
payload: Mapping[str, object], selection: RequestedQuotaActionSelection,
) -> bool:
receipt = selection.receipt
if receipt is None or payload.get("effective_action") != EffectiveAction.UNSETTLED_HOST_TURN_RECOVERY.value:
return False
details = receipt.get("details")
if payload.get("should_run") is not True or not isinstance(details, Mapping) or details.get("should_run") is not False:
return False
if (
receipt.get("status") != EffectiveAction.BLOCKED_WAIT.value
or selection.receipt_bound_todo_id
or selection.receipt_bound_replan_obligation_id
or heartbeat_receipt_request_subject(receipt) is not None
):
raise HeartbeatReceiptIdentityConflictError(
"The existing Turn has no recovery grant; use a fresh Turn for recovery."
)
return True


def _execute_quota_command(
args: argparse.Namespace,
*,
Expand All @@ -680,13 +800,15 @@ def _execute_quota_command(
append_cli_rollout_event: RolloutEventAppender,
capture_directory: Path | None = None,
retained_action_selection_todo_id: str | None = None,
recovery_refreshes_remaining: int = 3,
) -> int:
usage_quota_started = time.time_ns() // 1_000_000
heartbeat_turn_id: str | None = None
heartbeat_receipt_existing: dict[str, object] | None = None
heartbeat_receipt_existing_status = "replayed"
heartbeat_receipt_existing_appended = False
heartbeat_receipt_ready = False
upgrade_recovery_wait = False
action_selection_preflight_failed = False
closeout_query_unavailable = False
action_selection: RequestedQuotaActionSelection | None = None
Expand Down Expand Up @@ -818,7 +940,10 @@ def _execute_quota_command(
action_selection_preflight.receipt_appended
)
if heartbeat_turn_id and reentry is None and not action_selection_preflight_failed:
if heartbeat_receipt_existing:
# A wait receipt grants no recovery allowance. After a valid
# resume, bind its first grant under the ordinary append fence.
upgrade_recovery_wait = _can_upgrade_recovery_wait(payload, action_selection)
if heartbeat_receipt_existing and not upgrade_recovery_wait:
(
heartbeat_receipt_existing,
heartbeat_receipt_existing_status,
Expand Down Expand Up @@ -961,6 +1086,7 @@ def _execute_quota_command(
reentry_args, registry_path=registry_path, runtime_root_arg=runtime_root_arg,
print_payload=print_payload, append_cli_rollout_event=append_cli_rollout_event,
capture_directory=capture_directory, retained_action_selection_todo_id=retained_todo_id,
recovery_refreshes_remaining=recovery_refreshes_remaining,
)
preview = args.quota_command == "should-run" and getattr(args, "dry_run", False)
if not preview and _export("should_log_quota")(args.quota_command, payload):
Expand Down Expand Up @@ -1003,7 +1129,7 @@ def _execute_quota_command(
+ (f": {prior_reason}" if prior_reason else "")
),
)
elif heartbeat_receipt_existing:
elif heartbeat_receipt_existing and not upgrade_recovery_wait:
_export("render_existing_heartbeat_receipt_payload")(
payload,
receipt=heartbeat_receipt_existing,
Expand All @@ -1020,93 +1146,29 @@ def _execute_quota_command(
),
)
else:
settlement_identity = (
SettlementIdentity(
goal_id=args.goal_id,
agent_id=args.agent_id,
todo_id=rollout_todo_id,
turn_instance_id=heartbeat_turn_id,
replan_obligation_id=rollout_replan_obligation_id,
)
if rollout_todo_id or rollout_replan_obligation_id
else None
)
rollout_details.update(
{
"turn_instance_id": heartbeat_turn_id,
"stall_observation": heartbeat_stall_observation,
"settlement_effect_id": (
settlement_identity.effect_id
if settlement_identity is not None
else ""
),
}
)
append_cli_rollout_event(
payload,
registry_path=registry_path,
runtime_root_arg=runtime_root_arg,
event_kind="quota_should_run",
goal_ref=goal_ref,
agent_id=args.agent_id,
run_id=heartbeat_turn_id,
status=str(
payload.get("effective_action")
or payload.get("decision")
or "should-run"
),
summary=(
"heartbeat quota receipt committed for "
f"turn={heartbeat_turn_id} stall={heartbeat_stall_observation}"
),
details=rollout_details,
allow_failed=True,
idempotency_fields=[
"goal_id",
"event_kind",
"agent_id",
"run_id",
*(["goal_ref"] if goal_ref is not None else []),
],
)
receipt = find_heartbeat_receipt(
runtime_root,
goal_id=args.goal_id,
agent_id=args.agent_id,
turn_instance_id=heartbeat_turn_id,
goal_ref=goal_ref,
)
if receipt:
rollout_event_value = payload.get("rollout_event")
rollout_event: Mapping[str, object] = (
rollout_event_value
if isinstance(rollout_event_value, Mapping)
else {}
)
payload["heartbeat_receipt"] = heartbeat_receipt_view(
receipt,
turn_instance_id=heartbeat_turn_id,
status="committed"
if rollout_event.get("appended")
else "replayed",
try:
payload = _append_quota_heartbeat_receipt(
payload, args, registry_path=registry_path, runtime_root_arg=runtime_root_arg,
runtime_root=runtime_root, goal_ref=goal_ref, append_cli_rollout_event=append_cli_rollout_event,
turn_instance_id=heartbeat_turn_id, stall_observation=heartbeat_stall_observation,
rollout_todo_id=rollout_todo_id, rollout_replan_obligation_id=rollout_replan_obligation_id,
rollout_details=rollout_details, upgrade_recovery_wait=upgrade_recovery_wait,
requested_todo_id=action_selection.requested_todo_id if action_selection is not None else None,
)
_export("commit_requested_action_selection")(
payload,
requested_todo_id=(
action_selection.requested_todo_id
if action_selection is not None
else None
),
)
else:
except RolloutAppendPreconditionChanged:
if recovery_refreshes_remaining:
return _execute_quota_command(
args, registry_path=registry_path, runtime_root_arg=runtime_root_arg,
print_payload=print_payload, append_cli_rollout_event=append_cli_rollout_event,
capture_directory=capture_directory,
retained_action_selection_todo_id=retained_action_selection_todo_id,
recovery_refreshes_remaining=recovery_refreshes_remaining - 1,
)
fail_heartbeat_receipt(
payload,
turn_instance_id=heartbeat_turn_id,
stall_observation=heartbeat_stall_observation,
reason=(
"heartbeat receipt append could not be read back; retry "
"quota should-run with the same --turn-instance-id"
),
reason="recovery admission changed repeatedly before append; retry the same Turn",
)
else:
_record_non_guard_quota_event(
Expand Down
7 changes: 6 additions & 1 deletion loopx/cli_rollout.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
from __future__ import annotations

from collections.abc import Mapping
from collections.abc import Callable, Mapping
from pathlib import Path

from .control_plane.projects.registry_codec import load_registry
from .paths import resolve_runtime_root
from .rollout_event_log import (
RolloutAppendPreconditionChanged,
append_rollout_event,
append_rollout_event_once,
build_rollout_event,
Expand All @@ -31,6 +32,7 @@ def append_cli_rollout_event(
details: dict[str, object] | None = None,
allow_failed: bool = False,
idempotency_fields: list[str] | None = None,
precondition: Callable[[], None] | None = None,
) -> dict[str, object]:
"""Append a compact rollout event for core CLI lifecycle commands.

Expand Down Expand Up @@ -73,6 +75,7 @@ def append_cli_rollout_event(
log_path,
event,
identity_fields=idempotency_fields,
precondition=precondition,
)
else:
appended = append_rollout_event(log_path, event)
Expand All @@ -87,6 +90,8 @@ def append_cli_rollout_event(
if idempotency_fields:
rollout_event_view["appended"] = newly_appended
payload["rollout_event"] = rollout_event_view
except RolloutAppendPreconditionChanged:
raise
except Exception as exc:
payload["rollout_event_log_error"] = {
"recorded": False,
Expand Down
Loading
Loading