diff --git a/docs/quota-allocation.md b/docs/quota-allocation.md index 2a853eddd6..567b79cb26 100644 --- a/docs/quota-allocation.md +++ b/docs/quota-allocation.md @@ -1455,6 +1455,14 @@ markdown output and highlights `next_automatic_turn` when a goal is eligible. If that value is `none`, the automation should skip delivery compute and follow the displayed gate, evidence, or health reason. +`quota receipt --goal-id --agent-id --turn-instance-id ` +reads the committed guard for that exact Turn. `heartbeat_receipt` is null when +the scoped receipt is missing. A guard that admitted an explicit work choice +before it was made carries `selection_required` and `settlement_binding_owed`. +After selection, the same lookup returns the committed settlement identity. +Hosts use this read-only lookup to reconcile a finished attempt with its +original guard, including when the current queue has changed. + `quota should-run` is the per-goal guard for heartbeat jobs. It returns a small JSON or Markdown decision: diff --git a/loopx/cli_commands/quota.py b/loopx/cli_commands/quota.py index 053b0d5778..5daf6eba6c 100644 --- a/loopx/cli_commands/quota.py +++ b/loopx/cli_commands/quota.py @@ -171,6 +171,7 @@ def _quota_goal_ref( return exact_goal_ref(goal_id, goal_instance_id) if args.quota_command not in { "should-run", + "receipt", "monitor-poll", "scheduler-ack", "scheduler-ack-current", @@ -655,6 +656,15 @@ def handle_quota_command( capture_directory: Path | None = None, retained_action_selection_todo_id: str | None = None, ) -> int: + if args.quota_command == "receipt": + from .quota_receipt import read_quota_receipt, render_quota_receipt + + payload = read_quota_receipt( + args, registry_path=registry_path, runtime_root_arg=runtime_root_arg, + goal_ref_resolver=lambda: _quota_goal_ref(args, registry_path=registry_path), + ) + print_payload(payload, args.format, render_quota_receipt) + return 0 if payload["ok"] else 1 native_result = handle_native_context_command(args, print_payload=print_payload, registry_path=registry_path, runtime_root_arg=runtime_root_arg) if native_result is not None: diff --git a/loopx/cli_commands/quota_receipt.py b/loopx/cli_commands/quota_receipt.py new file mode 100644 index 0000000000..4a83b2a436 --- /dev/null +++ b/loopx/cli_commands/quota_receipt.py @@ -0,0 +1,52 @@ +"""Read the original persisted guard without collecting live quota or writing.""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable, Mapping +import json +from pathlib import Path + +from ..control_plane.quota.heartbeat_receipt import find_heartbeat_receipt, heartbeat_receipt_view +from ..history import load_registry +from ..paths import resolve_runtime_root, validate_goal_id_path_segment +from ..registry import find_registry_goal +from ..turn_identity import normalize_turn_instance_id + + +def read_quota_receipt( + args: argparse.Namespace, + *, + registry_path: Path, + runtime_root_arg: str | None, + goal_ref_resolver: Callable[[], Mapping[str, object] | None], +) -> dict[str, object]: + payload: dict[str, object] = {"ok": False, "schema_version": "quota_receipt_readback_v0"} + try: + if not (args.goal_id and args.agent_id and args.turn_instance_id): + raise ValueError("quota receipt requires --goal-id, --agent-id and --turn-instance-id") + goal_id = validate_goal_id_path_segment(args.goal_id) + turn_id = normalize_turn_instance_id(args.turn_instance_id) + assert turn_id is not None + registry = load_registry(registry_path) + if find_registry_goal(registry, goal_id) is None: + raise ValueError("quota receipt requires a registered Goal") + runtime_root = resolve_runtime_root(registry, runtime_root_arg, registry_path=registry_path) + event = find_heartbeat_receipt( + runtime_root, goal_id=goal_id, agent_id=args.agent_id, turn_instance_id=turn_id, + goal_ref=goal_ref_resolver(), + ) + payload.update( + ok=True, goal_id=goal_id, agent_id=args.agent_id, turn_instance_id=turn_id, + heartbeat_receipt=( + heartbeat_receipt_view(event, turn_instance_id=turn_id, status="committed") + if event is not None else None + ), + ) + except (OSError, RuntimeError, ValueError) as exc: + payload["error"] = str(exc) + return payload + + +def render_quota_receipt(payload: dict[str, object]) -> str: + return json.dumps(payload, ensure_ascii=False, indent=2) diff --git a/loopx/cli_commands/quota_registration.py b/loopx/cli_commands/quota_registration.py index ceec68a026..3ac914041f 100644 --- a/loopx/cli_commands/quota_registration.py +++ b/loopx/cli_commands/quota_registration.py @@ -49,6 +49,7 @@ def register_quota_command( "status", "plan", "should-run", + "receipt", "context-read", "context-project", "context-hook", @@ -62,7 +63,7 @@ def register_quota_command( "void-slot", ], default="status", - help="Use status for all groups, plan for next-turn groups, should-run for one goal, context-read for bounded original-capture recovery without a guard, context-project to present an existing digest-bound quota capture with native full/delta and a full-read fallback, context-hook for opt-in native Codex lifecycle IO, context-config for native hook/recovery configuration without installation or trust changes, monitor-poll for no-spend quiet poll evidence, scheduler-ack for successful Codex App RRULE state, scheduler-fail-current to suppress a repeated failed host update pair, spend-slot for accounting, or void-slot for a non-destructive accounting correction.", + help="Use status for all groups, plan for next-turn groups, should-run for one goal, receipt to read one exact committed guard without admission, context-read for bounded original-capture recovery without a guard, context-project to present an existing digest-bound quota capture with native full/delta and a full-read fallback, context-hook for opt-in native Codex lifecycle IO, context-config for native hook/recovery configuration without installation or trust changes, monitor-poll for no-spend quiet poll evidence, scheduler-ack for successful Codex App RRULE state, scheduler-fail-current to suppress a repeated failed host update pair, spend-slot for accounting, or void-slot for a non-destructive accounting correction.", ) quota_parser.add_argument("--decision-file", help="Original captured decision for read-only replan-view.") quota_parser.add_argument("--outcome-ref", help="Exact captured acceptance outcome reference for replan-view.") @@ -303,7 +304,7 @@ def register_quota_command( quota_parser.add_argument( "--turn-instance-id", help=( - "Stable heartbeat settlement id for `quota should-run`, " + "Stable heartbeat settlement id for `quota should-run`, `quota receipt`, " "`quota monitor-poll`, scheduler ACK/failure follow-ups, and " "`quota spend-slot`. The guard persists one idempotent receipt; " "reuse the same id through monitor writeback, scheduler handoff, " diff --git a/loopx/control_plane/quota/heartbeat_receipt.py b/loopx/control_plane/quota/heartbeat_receipt.py index b228abd519..da06b9652d 100644 --- a/loopx/control_plane/quota/heartbeat_receipt.py +++ b/loopx/control_plane/quota/heartbeat_receipt.py @@ -663,6 +663,8 @@ def heartbeat_receipt_view( # finished guard: the caller otherwise discovers the debt only when the # closeout cannot settle the turn and the work stays unaccounted. receipt["settlement_binding_owed"] = True + if details.get("selection_required") is True: + receipt["selection_required"] = True semantic_replan_obligation_id = ( heartbeat_receipt_semantic_replan_obligation_id(event) ) diff --git a/loopx/control_plane/quota/settlement_cli.py b/loopx/control_plane/quota/settlement_cli.py index 77f179f741..20f9fc2e2a 100644 --- a/loopx/control_plane/quota/settlement_cli.py +++ b/loopx/control_plane/quota/settlement_cli.py @@ -438,6 +438,14 @@ def quota_rollout_details( details["semantic_replan_gap_id"] = capability_guard["gap_id"] details["semantic_replan_frontier_revision"] = capability_guard["frontier_revision"] cli_channel = interaction.get("cli_channel") + if ( + isinstance(cli_channel, Mapping) + and cli_channel.get("selection_required") is True + and payload.get("ok") is True + and payload.get("should_run") is True + and not todo_id and not replan_obligation_id + ): + details["selection_required"] = True if isinstance(cli_channel, Mapping) and cli_channel.get("quota_spend_source"): details["quota_spend_source"] = cli_channel["quota_spend_source"] retained_todo_id = normalize_todo_id(retained_selection.get("retained_todo_id")) diff --git a/packages/dsh-loopx-plugin/README.md b/packages/dsh-loopx-plugin/README.md index fa051dc690..65843a52f1 100644 --- a/packages/dsh-loopx-plugin/README.md +++ b/packages/dsh-loopx-plugin/README.md @@ -561,6 +561,21 @@ cadence from `scheduler_hint.cold_path_detail.local_scheduler` with present, and enforces `unchanged_poll.limits.local_scheduler`. A changed `reset_policy.reset_token` resets the unchanged-poll streak. +When Core's recovery allowance is exhausted, the GoalBar for that exact +Session shows **Prior turn needs recovery**. An owner can repair the original +Turn or pause and start the Goal to renew the existing allowance. The label +reflects the Driver's last verified quota decision and updates with the existing +GoalBar watch. In-flight activity and an explicit Pause keep their usual labels. + +A normal DSH completion can also leave the original Turn awaiting required +Todo selection. For guarded Turns with that explicit requirement, the Driver +reads the exact receipt before scheduling another automatic Turn. If selection +is still owed, the Driver retries once with the original Turn identity. A second +completed attempt that still owes selection shows the recovery status and waits. +Pause and Start requests another attempt on the same Turn. A committed selection +lets the Driver follow the ordinary scheduling path. Interactive human messages +continue to use the native Session. + If DSH refuses the maintenance slot before admission starts, the Driver waits for `Agent.whenIdle()` and re-requests evaluation for that same Session. This wait uses no polling timer and is cancelled by Pause, human input, Session diff --git a/packages/dsh-loopx-plugin/src/client/LoopXGoalBar.tsx b/packages/dsh-loopx-plugin/src/client/LoopXGoalBar.tsx index 5379ddb22d..e05e5b5528 100644 --- a/packages/dsh-loopx-plugin/src/client/LoopXGoalBar.tsx +++ b/packages/dsh-loopx-plugin/src/client/LoopXGoalBar.tsx @@ -22,9 +22,10 @@ export interface LoopXGoalBarProps { function visibleStatus( snapshot: GoalBarSnapshotV1, -): 'active' | 'running' | 'paused' { +): 'active' | 'running' | 'paused' | 'recovery' { if (snapshot.goalActivation === 'stopped') return 'paused' - return snapshot.agentStatus === 'running' ? 'running' : 'active' + if (snapshot.agentStatus === 'running') return 'running' + return snapshot.recoveryAttention ? 'recovery' : 'active' } function classes(...values: Array): string { diff --git a/packages/dsh-loopx-plugin/src/client/goalbar.module.css b/packages/dsh-loopx-plugin/src/client/goalbar.module.css index 3811049a28..91f67984ae 100644 --- a/packages/dsh-loopx-plugin/src/client/goalbar.module.css +++ b/packages/dsh-loopx-plugin/src/client/goalbar.module.css @@ -139,6 +139,12 @@ border-radius: 1px; } +.status[data-status="recovery"] .statusMark { + background: transparent; + border-radius: 1px; + transform: rotate(45deg); +} + .status[data-status="running"] .statusMark { box-shadow: 0 0 0 2px var(--color-surface, var(--dsw-alias-bg-base, #fff)); outline: 1px solid currentColor; diff --git a/packages/dsh-loopx-plugin/src/client/locale.ts b/packages/dsh-loopx-plugin/src/client/locale.ts index 74b2c549ec..3cc2aa3b06 100644 --- a/packages/dsh-loopx-plugin/src/client/locale.ts +++ b/packages/dsh-loopx-plugin/src/client/locale.ts @@ -10,6 +10,7 @@ const en = { 'status.active': 'Active', 'status.running': 'Running', 'status.paused': 'Paused', + 'status.recovery': 'Prior turn needs recovery', 'action.start': 'Start', 'action.pause': 'Pause', 'action.refresh': 'Refresh', @@ -41,6 +42,7 @@ const zh: Record = { 'status.active': '已激活', 'status.running': '运行中', 'status.paused': '已暂停', + 'status.recovery': '先前轮次待恢复', 'action.start': '启动', 'action.pause': '暂停', 'action.refresh': '刷新', diff --git a/packages/dsh-loopx-plugin/src/driver.ts b/packages/dsh-loopx-plugin/src/driver.ts index 5f144538cf..37f6e89c13 100644 --- a/packages/dsh-loopx-plugin/src/driver.ts +++ b/packages/dsh-loopx-plugin/src/driver.ts @@ -21,6 +21,7 @@ import type { GoalBarDriverActionReceipt, GoalBarSessionCapture, } from './goalbar/events.ts' +import type { GoalBarExpectedBindingV1 } from './goalbar/protocol.ts' export const name = 'dsh-loopx-driver' export const inject = ['agents', 'loopxBootstrap'] @@ -29,8 +30,10 @@ const HOST_SURFACE = 'deepseek-harness-native' const RESOLUTION_SCHEMA = 'loopx_thread_agent_binding_resolution_v0' const HEARTBEAT_SCHEMA = 'heartbeat_agent_input_v1' const CONTINUATION_SCHEMA = 'loopx_dsh_continuation_v0' +const RECEIPT_READBACK_SCHEMA = 'quota_receipt_readback_v0' const DEFAULT_WAIT_MS = 5 * 60_000 const MAX_WAIT_MS = 24 * 60 * 60_000 +const SELECTION_RECOVERY_COMPLETION_LIMIT = 2 type TimerHandle = unknown type ReservationPhase = 'queued' | 'claimed' | 'admitted' | 'retired' @@ -45,12 +48,21 @@ interface Binding { readonly agentId: string } +interface SelectionTurn extends Binding { + readonly turnInstanceId: string +} + +interface SelectionRecovery extends SelectionTurn { + readonly completedAttempts: number +} + interface LoopXContinuationMessageSource { readonly kind: 'loopx-continuation' readonly schemaVersion: typeof CONTINUATION_SCHEMA readonly goalId: string readonly agentId: string readonly turnInstanceId: string + readonly selectionRequired?: true } declare module '@deepseek-ai/dsh-llm' { @@ -74,6 +86,8 @@ interface ActivationProjection { readonly session: Agent['session'] readonly pendingModelCalls: Set activated: boolean + hostTurn?: { turn: number; entered: boolean; selection?: SelectionTurn } | undefined + pendingSelection?: SelectionRecovery | undefined } interface EvaluationWaiter { @@ -96,6 +110,8 @@ interface DriverState { controller?: AbortController | undefined timer?: TimerHandle | undefined reservation?: Reservation | undefined + recoveryAttention?: Binding | undefined + retrySelection: boolean } interface QueuePlan extends Binding { @@ -103,6 +119,7 @@ interface QueuePlan extends Binding { readonly session: Agent['session'] readonly turnInstanceId: string readonly taskBody: string + readonly selectionRequired: boolean } interface WaitPlan { @@ -110,6 +127,7 @@ interface WaitPlan { readonly delayMs?: number | undefined readonly schedulerToken?: string | undefined readonly unchangedPolls?: number | undefined + readonly recoveryAttention?: Binding | undefined } type EvaluationPlan = QueuePlan | WaitPlan @@ -151,12 +169,14 @@ const systemClock: DriverClock = Object.freeze({ function continuationSource( message: UserMessage, ): LoopXContinuationMessageSource | undefined { - const source = exactRecord(message.source, [ + const observed = record(message.source) + const source = exactRecord(observed, [ 'kind', 'schemaVersion', 'goalId', 'agentId', 'turnInstanceId', + ...(observed?.selectionRequired === true ? ['selectionRequired'] : []), ]) return source?.kind === 'loopx-continuation' && source.schemaVersion === CONTINUATION_SCHEMA @@ -204,6 +224,7 @@ function automaticMessage( goalId: plan.goalId, agentId: plan.agentId, turnInstanceId: plan.turnInstanceId, + ...(plan.selectionRequired ? { selectionRequired: true as const } : {}), }), }) } @@ -301,6 +322,36 @@ function foldActivationEvent( } } +function foldSelectionEvent(projection: ActivationProjection, event: SessionEvent): void { + if (event.type === 'turn/start') { + projection.hostTurn = { turn: event.data.turn, entered: false } + return + } + const hostTurn = projection.hostTurn + if (hostTurn === undefined) return + if (event.type === 'user/message') { + const source = continuationSource(event.data) + if (source?.selectionRequired === true) { + hostTurn.selection = { goalId: source.goalId, agentId: source.agentId, + turnInstanceId: source.turnInstanceId } + } + } else if (event.type === 'step/start' && hostTurn.turn === event.data.turn) { + hostTurn.entered = true + } else if (event.type === 'turn/end' && hostTurn.turn === event.data.turn) { + projection.hostTurn = undefined + if (event.data.reason.kind === 'completed' && hostTurn.entered && hostTurn.selection !== undefined) { + const selection = hostTurn.selection + const previous = projection.pendingSelection + const sameOriginal = previous?.turnInstanceId === selection.turnInstanceId + && previous.goalId === selection.goalId && previous.agentId === selection.agentId + projection.pendingSelection = { + ...selection, + completedAttempts: sameOriginal ? previous.completedAttempts + 1 : 1, + } + } + } +} + function foldSessionActivation(session: Agent['session']): ActivationProjection { const projection: ActivationProjection = { session, @@ -308,10 +359,15 @@ function foldSessionActivation(session: Agent['session']): ActivationProjection activated: false, } try { - for (const event of session.snapshotEvents()) foldActivationEvent(projection, event) + for (const event of session.snapshotEvents()) { + foldActivationEvent(projection, event) + foldSelectionEvent(projection, event) + } } catch { projection.pendingModelCalls.clear() projection.activated = false + projection.hostTurn = undefined + projection.pendingSelection = undefined } return projection } @@ -354,7 +410,10 @@ const QUOTA_INDEX_SCHEMA = 'context_capture_index_v1' const QUOTA_SELECTION_SCHEMA = 'context_capture_selection_v1' const QUOTA_REQUIRED_POINTERS = ['/ok', '/mode', '/goal_id', '/should_run', '/agent_identity'] const QUOTA_RECEIPT_POINTER = '/heartbeat_receipt' -const QUOTA_OPTIONAL_POINTERS = [QUOTA_RECEIPT_POINTER, '/scheduler_hint', '/reward_memory_recall'] +const QUOTA_OPTIONAL_POINTERS = [ + QUOTA_RECEIPT_POINTER, '/scheduler_hint', '/reward_memory_recall', + '/unsettled_host_turn_recovery', +] type QuotaReadOptions = Pick @@ -676,6 +735,8 @@ export class LoopXContinuationDriver { this.retireReservation(state) state.competing = inboxHasPending(agent.inbox) state.pauseAfterTurnError = false + state.recoveryAttention = undefined + state.retrySelection = false state.schedulerToken = '' state.unchangedPolls = 0 state.commands.clear() @@ -723,6 +784,14 @@ export class LoopXContinuationDriver { this.cancelPending(state) } + hasRecoveryAttention(capture: GoalBarSessionCapture, expected: GoalBarExpectedBindingV1): boolean { + if (!this.captureIsLive(capture)) return false + const state = this.states.get(capture.agent) + return state?.activation.session === capture.session + && state.recoveryAttention?.goalId === expected.goalId + && state.recoveryAttention.agentId === expected.loopxAgentId + } + async evaluateActivatedSession( capture: GoalBarSessionCapture, ): Promise { @@ -738,6 +807,7 @@ export class LoopXContinuationDriver { if (!state.activation.activated) return { kind: 'applied' } state.pauseAfterTurnError = false + state.retrySelection = state.activation.pendingSelection !== undefined const evaluated = new Promise(resolve => { state.evaluationWaiters.add({ session: capture.session, resolve }) }) @@ -770,6 +840,7 @@ export class LoopXContinuationDriver { if (reservation?.phase === 'claimed' || reservation?.phase === 'admitted') { reservation.preserveAcrossPause = true } + state.retrySelection = false this.cancelPending(state) return this.captureIsLive(capture) && state.activation.session === capture.session @@ -783,6 +854,7 @@ export class LoopXContinuationDriver { const state = this.stateFor(agent) const wasActivated = state.activation.activated foldActivationEvent(state.activation, event) + foldSelectionEvent(state.activation, event) if (!wasActivated && state.activation.activated && agent.status === 'idle' && !state.pauseAfterTurnError) { this.requestEvaluation(state) @@ -922,6 +994,8 @@ export class LoopXContinuationDriver { this.retireReservation(existing) existing.competing = inboxHasPending(agent.inbox) existing.pauseAfterTurnError = false + existing.recoveryAttention = undefined + existing.retrySelection = false existing.schedulerToken = '' existing.unchangedPolls = 0 existing.commands.clear() @@ -936,6 +1010,7 @@ export class LoopXContinuationDriver { stopping: false, competing: inboxHasPending(agent.inbox), pauseAfterTurnError: false, + retrySelection: false, schedulerToken: '', unchangedPolls: 0, commands: new Set(), @@ -1011,6 +1086,7 @@ export class LoopXContinuationDriver { private async evaluate(state: DriverState): Promise { if (!this.ready(state)) return false + const session = state.agent.session const startedAt = performance.now() let stage: AdmissionStage = 'maintenance-slot' let admissionSignal: AbortSignal | undefined @@ -1056,6 +1132,10 @@ export class LoopXContinuationDriver { if (!waitingForIdle && state.controller === controller) state.controller = undefined } + // A cancelled maintenance result does not describe the current Session. + if (!admissionSignal?.aborted && this.ready(state) && state.agent.session === session) { + state.recoveryAttention = plan.kind === 'wait' ? plan.recoveryAttention : undefined + } if (plan.kind === 'wait') { if (plan.schedulerToken !== undefined) state.schedulerToken = plan.schedulerToken if (plan.unchangedPolls !== undefined) state.unchangedPolls = plan.unchangedPolls @@ -1122,8 +1202,26 @@ export class LoopXContinuationDriver { return { kind: 'wait', schedulerToken: '', unchangedPolls: 0 } } - const turnInstanceId = this.makeTurnInstanceId() setStage('quota') + const selection = state.activation.pendingSelection + let turnInstanceId: string | undefined + if (selection !== undefined + && selection.goalId === binding.goalId && selection.agentId === binding.agentId) { + const receipt = await this.readSelectionReceipt(command, state.agent, selection, signal) + this.ensureReady(state, signal, session) + if (receipt.selection_required === true && receipt.settlement_binding_owed === true) { + if (!state.retrySelection && selection.completedAttempts >= SELECTION_RECOVERY_COMPLETION_LIMIT) { + return { kind: 'wait', recoveryAttention: binding } + } + turnInstanceId = selection.turnInstanceId + } else { + state.activation.pendingSelection = undefined + } + } else { + state.activation.pendingSelection = undefined + } + state.retrySelection = false + turnInstanceId ??= this.makeTurnInstanceId() const quota = await this.readQuota(command, state.agent, binding, turnInstanceId, { signal }) this.ensureReady(state, signal, session) if (quota.ok !== true) { @@ -1142,7 +1240,13 @@ export class LoopXContinuationDriver { ) } if (quota.should_run !== true) { - return schedulerWait(quota, state.schedulerToken, state.unchangedPolls) + const wait = schedulerWait(quota, state.schedulerToken, state.unchangedPolls) + const recovery = record(quota.unsettled_host_turn_recovery) + return quota.should_run === false + && recovery?.schema_version === 'unsettled_host_turn_recovery_v0' + && recovery.state === 'attention_required' + ? { ...wait, recoveryAttention: binding } + : wait } setStage('heartbeat') const heartbeat = await this.readHeartbeat( @@ -1168,12 +1272,15 @@ export class LoopXContinuationDriver { false, ) } + const receipt = record(quota.heartbeat_receipt) return { kind: 'queue', ...binding, session, turnInstanceId, taskBody: taskBodyWithRewardMemory(taskBody, quota), + selectionRequired: (receipt?.status === 'committed' || receipt?.status === 'replayed') + && receipt.selection_required === true && receipt.settlement_binding_owed === true, } } @@ -1311,6 +1418,31 @@ export class LoopXContinuationDriver { return decision } + private async readSelectionReceipt( + command: LoopXCommand, + agent: Agent, + selection: SelectionTurn, + signal: AbortSignal, + ): Promise> { + const readback = await runJsonCommand(command, [ + '--registry', '.loopx/registry.json', '--format', 'json', 'quota', 'receipt', + '--goal-id', selection.goalId, '--agent-id', selection.agentId, + '--turn-instance-id', selection.turnInstanceId, + ], { runner: this.runner, cwd: agent.session.header.cwd, signal }) + const receipt = record(readback.heartbeat_receipt) + const identity = record(receipt?.settlement_identity) + if (readback.schema_version !== RECEIPT_READBACK_SCHEMA + || readback.ok !== true || readback.goal_id !== selection.goalId + || readback.agent_id !== selection.agentId || readback.turn_instance_id !== selection.turnInstanceId + || receipt?.status !== 'committed' || receipt.turn_instance_id !== selection.turnInstanceId + || !(receipt.selection_required === true && receipt.settlement_binding_owed === true + || identity?.goal_id === selection.goalId && identity.agent_id === selection.agentId + && identity.turn_instance_id === selection.turnInstanceId)) { + throw new LoopXCliError('invalid_schema', 'LoopX could not verify the original Turn selection', false) + } + return receipt + } + private readHeartbeat( command: LoopXCommand, agent: Agent, diff --git a/packages/dsh-loopx-plugin/src/goalbar/events.ts b/packages/dsh-loopx-plugin/src/goalbar/events.ts index f5f597e688..4cdd772b76 100644 --- a/packages/dsh-loopx-plugin/src/goalbar/events.ts +++ b/packages/dsh-loopx-plugin/src/goalbar/events.ts @@ -1,6 +1,6 @@ import type { Agent } from '@deepseek-ai/dsh-agent' import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' -import type { GoalBarAgentStatusV1 } from './protocol.ts' +import type { GoalBarAgentStatusV1, GoalBarExpectedBindingV1 } from './protocol.ts' export interface GoalBarSessionCapture { readonly agent: Agent; readonly session: Session } export type GoalBarSessionNotification = @@ -18,6 +18,7 @@ export type GoalBarDriverActionReceipt = { readonly kind: 'applied' } | { export interface GoalBarDriverBridge { evaluateActivatedSession(capture: GoalBarSessionCapture): Promise cancelQueued(capture: GoalBarSessionCapture): Promise + hasRecoveryAttention?(capture: GoalBarSessionCapture, expected: GoalBarExpectedBindingV1): boolean } export interface GoalBarSessionWaitOptions { readonly signal: AbortSignal @@ -149,6 +150,10 @@ export class GoalBarCoordinator { return this.invokeDriver('cancelQueued', capture) } + hasRecoveryAttention(capture: GoalBarSessionCapture, expected: GoalBarExpectedBindingV1): boolean { + return this.driverBridge?.hasRecoveryAttention?.(capture, expected) === true + } + private waitForSessionChange( service: WatchServiceState, session: Session, @@ -197,7 +202,7 @@ export class GoalBarCoordinator { waiter.resolve(receipt) } private async invokeDriver( - operation: keyof GoalBarDriverBridge, + operation: 'evaluateActivatedSession' | 'cancelQueued', capture: GoalBarSessionCapture, ): Promise { if (capture.agent.session !== capture.session) return { kind: 'unavailable', reason: 'session_unavailable' } diff --git a/packages/dsh-loopx-plugin/src/goalbar/protocol.ts b/packages/dsh-loopx-plugin/src/goalbar/protocol.ts index f35ea0260b..8246c717bd 100644 --- a/packages/dsh-loopx-plugin/src/goalbar/protocol.ts +++ b/packages/dsh-loopx-plugin/src/goalbar/protocol.ts @@ -51,6 +51,8 @@ export interface GoalBarSnapshotV1 { readonly goalActivation: GoalBarActivationV1 readonly agentStatus: GoalBarAgentStatusV1 readonly progress: GoalBarProgressV1 + /** Present only after the bound Driver observes a Core recovery hold. */ + readonly recoveryAttention?: true } export interface GoalBarExpectedBindingV1 { @@ -337,6 +339,7 @@ export function decodeGoalBarSnapshotV1( readonly binding?: GoalBarExpectedBindingV1 | undefined } = {}, ): GoalBarSnapshotV1 | undefined { + const recoveryAttention = record(value)?.recoveryAttention const input = exactRecord(value, [ 'sessionId', 'goalId', @@ -344,6 +347,7 @@ export function decodeGoalBarSnapshotV1( 'goalActivation', 'agentStatus', 'progress', + ...(recoveryAttention === true ? ['recoveryAttention'] : []), ]) const progress = exactRecord(input?.progress, ['processed', 'remaining', 'total']) if (!isGoalBarSessionId(input?.sessionId) @@ -368,6 +372,7 @@ export function decodeGoalBarSnapshotV1( loopxAgentId: input.loopxAgentId, goalActivation: input.goalActivation, agentStatus: input.agentStatus, + ...(recoveryAttention === true ? { recoveryAttention } : {}), progress: { processed: progress.processed, remaining: progress.remaining, diff --git a/packages/dsh-loopx-plugin/src/goalbar/service.ts b/packages/dsh-loopx-plugin/src/goalbar/service.ts index f6f44b8dcd..e4234e2025 100644 --- a/packages/dsh-loopx-plugin/src/goalbar/service.ts +++ b/packages/dsh-loopx-plugin/src/goalbar/service.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import type { Agent } from '@deepseek-ai/dsh-agent' import { LoopXCliError, @@ -29,6 +30,7 @@ import { } from './protocol.ts' import type { GoalBarActionResultV1, + GoalBarExpectedBindingV1, GoalBarReadFaultCode, GoalBarReadResultV1, GoalBarRequestV1, @@ -73,6 +75,7 @@ export interface GoalBarCoordinatorPort { cancelQueued( capture: GoalBarSessionCapture, ): Promise + hasRecoveryAttention?(capture: GoalBarSessionCapture, expected: GoalBarExpectedBindingV1): boolean } export interface GoalBarServiceOptions { @@ -143,6 +146,12 @@ function exactRecord( : undefined } +function recoverySourceRevision(revision: string, recoveryAttention: boolean): string { + // Retain the ordinary source revision shape for clients and unaffected sessions. + if (!recoveryAttention) return revision + return `sha256:${createHash('sha256').update('goalbar-recovery-attention\0').update(revision).digest('hex')}` +} + /** Strict verified relation for the existing lifecycle execution V1 receipt. */ export function decodeGoalBarLifecycleExecutionV1( value: unknown, @@ -421,14 +430,16 @@ export class GoalBarService implements GoalBarServiceHandle { if (model.kind === 'fault') { return { kind: 'fault', code: model.code, baseSessionEventSeq, sourceRevision } } + const recoveryAttention = this.coordinator.hasRecoveryAttention?.(capture, model.snapshot) === true return { kind: 'present', baseSessionEventSeq, - sourceRevision, + sourceRevision: recoverySourceRevision(sourceRevision, recoveryAttention), snapshot: { ...model.snapshot, // Status is process-local authority and is refreshed at publication. agentStatus: capture.agent.status, + ...(recoveryAttention ? { recoveryAttention: true } : {}), }, } } @@ -514,11 +525,7 @@ export class GoalBarService implements GoalBarServiceHandle { if (receipt.kind === 'runtime_changed' && receipt.sessionId === request.sessionId) { try { - const revision = await computeGoalBarSourceRevision({ - cwd: capture.cwd, - goalId: request.expected?.goalId, - loopxAgentId: request.expected?.loopxAgentId, - }) + const revision = await this.watchSourceRevision(capture, request.expected) if (!this.captureIsCurrent(capture)) { return { kind: 'fault', code: 'session_unavailable' } } @@ -545,11 +552,7 @@ export class GoalBarService implements GoalBarServiceHandle { return { kind: 'fault', code: 'session_unavailable' } } try { - const revision = await computeGoalBarSourceRevision({ - cwd: capture.cwd, - goalId: request.expected?.goalId, - loopxAgentId: request.expected?.loopxAgentId, - }) + const revision = await this.watchSourceRevision(capture, request.expected) if (!this.captureIsCurrent(capture)) { return { kind: 'fault', code: 'session_unavailable' } } @@ -566,6 +569,20 @@ export class GoalBarService implements GoalBarServiceHandle { return { kind: 'fault', code: 'session_unavailable' } } + private async watchSourceRevision( + capture: LiveCapture, + expected: GoalBarExpectedBindingV1 | null, + ): Promise { + const revision = await computeGoalBarSourceRevision({ + cwd: capture.cwd, + goalId: expected?.goalId, + loopxAgentId: expected?.loopxAgentId, + }) + const recoveryAttention = expected !== null + && this.coordinator.hasRecoveryAttention?.(capture, expected) === true + return recoverySourceRevision(revision, recoveryAttention) + } + private actionAdmission(): ActionAdmission { const mutation = deferred() const completion = deferred() diff --git a/packages/dsh-loopx-plugin/tests/admission-closeout.integration.spec.ts b/packages/dsh-loopx-plugin/tests/admission-closeout.integration.spec.ts index 97fb429ec8..aa1f7fb9c2 100644 --- a/packages/dsh-loopx-plugin/tests/admission-closeout.integration.spec.ts +++ b/packages/dsh-loopx-plugin/tests/admission-closeout.integration.spec.ts @@ -359,6 +359,7 @@ describe('DSH admission-to-closeout integration', () => { expect(source).toEqual({ kind: 'loopx-continuation', schemaVersion: 'loopx_dsh_continuation_v0', + selectionRequired: true, goalId, agentId, turnInstanceId, diff --git a/packages/dsh-loopx-plugin/tests/driver.spec.ts b/packages/dsh-loopx-plugin/tests/driver.spec.ts index 19e11c9443..e84def2d31 100644 --- a/packages/dsh-loopx-plugin/tests/driver.spec.ts +++ b/packages/dsh-loopx-plugin/tests/driver.spec.ts @@ -349,6 +349,7 @@ function runnerFixture(options: { readonly heartbeatTurnInstanceId?: string readonly heartbeatTaskBody?: string readonly rewardMemoryGuidance?: string + readonly recovery?: () => Record readonly typedQuotaFailure?: boolean readonly bindingStatus?: 'bound' | 'missing' | (() => 'bound' | 'missing') } = {}): RunnerFixture { @@ -455,6 +456,9 @@ function runnerFixture(options: { }, } : {}), scheduler_hint: schedulerHint, + ...(options.recovery === undefined ? {} : { + unsettled_host_turn_recovery: options.recovery(), + }), ...(options.rewardMemoryGuidance === undefined ? {} : { reward_memory_recall: { schema_version: 'agent_turn_recall_v0', @@ -2053,6 +2057,101 @@ describe('same-session LoopX driver', () => { }) describe('GoalBar exact-Session Driver bridge', () => { + it.each(['unfinished', 'bound', 'different-agent', 'no-step', 'one-completion'] as const)( + 'reconciles an original selection after Host completion on restore (%s)', async variant => { + const host = fakeAgent() + const original = 'dsh-prior-original-turn' + const source = { + kind: 'loopx-continuation', schemaVersion: 'loopx_dsh_continuation_v0', + goalId, agentId, turnInstanceId: original, selectionRequired: true, + } as const + host.appendEvent(userSkillInvocationEvent()) + const completed = variant === 'unfinished' ? 2 : 1 + for (let turn = 0; turn < completed; turn += 1) { + host.appendEvent(sessionEvent('turn/start', { turn })) + host.appendEvent(sessionEvent('user/message', { + ...userMessage('Continue the original Turn.'), source, + })) + if (variant !== 'no-step') host.appendEvent(sessionEvent('step/start', { turn, step: 0 })) + host.appendEvent(sessionEvent('turn/end', { turn, reason: { kind: 'completed' } })) + } + const fixture = runnerFixture() + const observed: string[] = [] + const runner: FileRunner = async (file, args, options) => { + if (!args.includes('receipt')) return fixture.runner(file, args, options) + observed.push(args[args.indexOf('--turn-instance-id') + 1] ?? '') + const identity = { goal_id: goalId, agent_id: agentId, turn_instance_id: original } + return { + exitCode: 0, stderr: '', stdout: JSON.stringify({ + ok: true, schema_version: 'quota_receipt_readback_v0', ...identity, + agent_id: variant === 'different-agent' ? 'agent-other' : agentId, + heartbeat_receipt: { status: 'committed', turn_instance_id: original, + ...(variant === 'bound' ? { settlement_identity: identity } : { + selection_required: true, settlement_binding_owed: true, + }), + }, + }), + } + } + const driver = new LoopXContinuationDriver({ + isLiveAgent: agent => agent === host.agent, runner, makeTurnInstanceId: () => 'fresh-turn', + }) + const capture = { agent: host.agent, session: host.agent.session } + try { + driver.observeAgent(host.agent) + if (variant === 'no-step' || variant === 'bound' || variant === 'one-completion') { + await waitFor(() => host.nextTurn.length === 1) + expect(observed).toEqual(variant === 'no-step' ? [] : [original]) + expect(fixture.quotaCalls[0]).toContain(variant === 'one-completion' ? original : 'fresh-turn') + } else { + await waitFor(() => observed.length === 1) + const exactBinding = { goalId, loopxAgentId: agentId } + if (variant === 'unfinished') { + await waitFor(() => driver.hasRecoveryAttention(capture, exactBinding)) + expect(fixture.quotaCalls).toHaveLength(0) + expect(host.nextTurn).toHaveLength(0) + await driver.evaluateActivatedSession(capture) + await waitFor(() => host.nextTurn.length === 1) + expect(fixture.quotaCalls[0]).toContain(original) + expect(observed).toEqual([original, original]) + } else { + await driver.dispose() + expect(fixture.quotaCalls).toHaveLength(0) + } + } + } finally { + await driver.dispose() + } + }, + ) + + it('projects only the observed Core recovery hold for the current binding', async () => { + const host = fakeAgent() + let recoveryState = 'attention_required' + const { runner } = runnerFixture({ + shouldRun: false, + schedulerMode: 'stop', + recovery: () => ({ + schema_version: 'unsettled_host_turn_recovery_v0', + state: recoveryState, + }), + }) + const driver = new LoopXContinuationDriver({ isLiveAgent: agent => agent === host.agent, runner }) + host.appendEvent(userSkillInvocationEvent()) + const capture = { agent: host.agent, session: host.agent.session } + const expected = { goalId, loopxAgentId: agentId } + await driver.evaluateActivatedSession(capture) + expect(driver.hasRecoveryAttention(capture, expected)).toBe(true) + expect(driver.hasRecoveryAttention(capture, { ...expected, loopxAgentId: 'different-agent' })) + .toBe(false) + expect(host.nextTurn).toHaveLength(0) + + recoveryState = 'recovery_required' + await driver.evaluateActivatedSession(capture) + expect(driver.hasRecoveryAttention(capture, expected)).toBe(false) + await driver.dispose() + }) + it.each(['agent/created', 'agent/session-start'] as const)( 'resets resumed Session state on %s without activating an uninvoked Session', async event => { diff --git a/packages/dsh-loopx-plugin/tests/goalbar-client.spec.tsx b/packages/dsh-loopx-plugin/tests/goalbar-client.spec.tsx index 1ff0eaaaac..6d4ef76008 100644 --- a/packages/dsh-loopx-plugin/tests/goalbar-client.spec.tsx +++ b/packages/dsh-loopx-plugin/tests/goalbar-client.spec.tsx @@ -243,6 +243,25 @@ describe('LoopXGoalBar presentation', () => { .toBe('No agent-lane todos') }) + it.each([ + ['active', 'idle', 'Prior turn needs recovery'], + ['active', 'running', 'Running'], + ['stopped', 'idle', 'Paused'], + ] as const)('shows the appropriate status on a recovery hold with %s and %s', async ( + goalActivation, agentStatus, text, + ) => { + const rpc = fakeRpc({ + read: sessionId => Promise.resolve(readOutcome(sessionId, { + kind: 'present', + snapshot: snapshot({ sessionId, goalActivation, agentStatus, recoveryAttention: true }), + baseSessionEventSeq: 0, + })), + }) + const view = mount({ rpcSessionId: 'session-one', rpc }) + await flush() + expect(view.container.querySelector('[data-status]')?.textContent).toBe(text) + }) + it.each(['binding_missing', 'binding_ambiguous'] as const)( 'renders no row for %s', async reason => { diff --git a/packages/dsh-loopx-plugin/tests/goalbar-service.spec.ts b/packages/dsh-loopx-plugin/tests/goalbar-service.spec.ts index 236a245158..0bd8102623 100644 --- a/packages/dsh-loopx-plugin/tests/goalbar-service.spec.ts +++ b/packages/dsh-loopx-plugin/tests/goalbar-service.spec.ts @@ -407,6 +407,40 @@ async function eventually(predicate: () => boolean): Promise { } describe('GoalBar Host read/watch', () => { + it('refreshes the bound snapshot after a recovery hold changes without a Session event', async () => { + const fixture = harness() + const signal = new AbortController().signal + const before = (await fixture.service.handle(readRequest(), signal)).result + if (before.kind !== 'present') throw new Error('fixture did not have a binding') + expect(before.snapshot).not.toHaveProperty('recoveryAttention') + + let attention = true + const unregister = fixture.coordinator.registerDriverBridge({ + evaluateActivatedSession: async () => ({ kind: 'applied' }), + cancelQueued: async () => ({ kind: 'applied' }), + hasRecoveryAttention: (capture, expected) => attention + && capture.session === fixture.host.agent.session + && expected.goalId === goalId && expected.loopxAgentId === loopxAgentId, + }) + try { + const watch = (sourceRevision: string) => fixture.service.handle( + watchRequest(null, { sourceRevision, expected: { goalId, loopxAgentId } }), signal, + ) + expect((await watch(before.sourceRevision)).result.kind).toBe('source_changed') + const held = (await fixture.service.handle(readRequest(), signal)).result + if (held.kind !== 'present') throw new Error('expected the same binding on hold') + expect(held.snapshot.recoveryAttention).toBe(true) + expect((await watch(held.sourceRevision)).result.kind).toBe('timeout') + + attention = false + expect((await watch(held.sourceRevision)).result.kind).toBe('source_changed') + expect((await fixture.service.handle(readRequest(), signal)).result).toEqual(before) + } finally { + unregister() + await fixture.service.dispose() + } + }) + it('captures the cursor before CLI, runs activation/Todo in parallel, and loses no wake', async () => { const fixture = harness() fixture.host.events.push(turnEnd(3)) diff --git a/tests/control_plane/test_quota_guard_receipt.py b/tests/control_plane/test_quota_guard_receipt.py new file mode 100644 index 0000000000..6db54a7d97 --- /dev/null +++ b/tests/control_plane/test_quota_guard_receipt.py @@ -0,0 +1,49 @@ +"""An exact admission read leaves quota and the original selection owner alone.""" +from pathlib import Path + +import pytest + +from loopx.rollout_event_log import rollout_event_log_path +from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from tests.control_plane.test_quota_settlement_cli import ( + AGENT_ID, GOAL_ID, TODO_ID, _run_cli, _spend_run_count, _write_fixture, +) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_read_original_quota_receipt_before_and_after_selection( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +) -> None: + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path, claimed=False, provider=provider) + log = rollout_event_log_path(runtime, GOAL_ID) + scope = ("--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--turn-instance-id", "host-original") + + rc, result = _run_cli(registry, runtime, "quota", "receipt", *scope, cwd=project) + assert rc == 0 and result["heartbeat_receipt"] is None and not log.exists(), result + + guard = ("quota", "should-run", "--runtime-profile", "generic_cli", *scope) + rc, original = _run_cli(registry, runtime, *guard, cwd=project) + assert rc == 0 and original["interaction_contract"]["cli_channel"]["selection_required"], original + before = log.read_bytes() + rc, result = _run_cli(registry, runtime, "quota", "receipt", *scope, cwd=project) + assert rc == 0 and result["heartbeat_receipt"] == original["heartbeat_receipt"], result + assert result["heartbeat_receipt"]["selection_required"] is True + assert result["heartbeat_receipt"]["settlement_binding_owed"] is True + assert log.read_bytes() == before + + rc, claim = _run_cli(registry, runtime, "todo", "claim", "--goal-id", GOAL_ID, + "--agent-id", AGENT_ID, "--todo-id", TODO_ID, "--claimed-by", AGENT_ID) + assert rc == 0 and claim["ok"], claim + rc, selected = _run_cli(registry, runtime, *guard, "--todo-id", TODO_ID, cwd=project) + assert rc == 0 and selected["heartbeat_receipt"]["settlement_identity"]["todo_id"] == TODO_ID, selected + before = log.read_bytes() + rc, result = _run_cli(registry, runtime, "quota", "receipt", *scope, cwd=project) + assert rc == 0 and result["heartbeat_receipt"]["settlement_identity"] == ( + selected["heartbeat_receipt"]["settlement_identity"] + ), result + assert "selection_required" not in result["heartbeat_receipt"] + rc, other = _run_cli(registry, runtime, "quota", "receipt", "--goal-id", GOAL_ID, + "--agent-id", "peer-agent", "--turn-instance-id", "host-original") + assert rc == 0 and other["heartbeat_receipt"] is None, other + assert log.read_bytes() == before and _spend_run_count(runtime) == 0