diff --git a/apps/presentation/dashboard/src/data/chat.ts b/apps/presentation/dashboard/src/data/chat.ts index c3db5a2c22..451aaebd1e 100644 --- a/apps/presentation/dashboard/src/data/chat.ts +++ b/apps/presentation/dashboard/src/data/chat.ts @@ -470,6 +470,7 @@ export const typedActionProposalSchema = z.object({ validation_evidence: z.array(z.string().refine((value) => value.trim().length > 0, "Validation evidence must be non-blank text")), available_transitions: z.array(z.enum(["apply", "cancel", "regenerate", "reject", "defer"])), status: z.enum(["preview_ready", "applying", "gated", "failed", "rejected", "deferred", "cancelled", "stale", "applied"]), + task_execution_dispatch_pending: z.literal(true).optional(), receipt: z.record(z.string(), z.unknown()).nullable(), stale: z.record(z.string(), z.unknown()).nullable(), gate: z.record(z.string(), z.unknown()).nullable().optional(), diff --git a/apps/presentation/dashboard/src/features/personal-workspace/context-drawer.tsx b/apps/presentation/dashboard/src/features/personal-workspace/context-drawer.tsx index 1ee6ce52e3..e4364606f6 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/context-drawer.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/context-drawer.tsx @@ -1231,7 +1231,9 @@ export function ContextDrawer({ agents, attentionHistory = [], onSelectAttention

{selection.item.title}

{selection.item.todoId && (selection.item.actionKind === "gate.resolve" || selection.item.actionKind === "todo.update") ?
: null} - {selection.item.impact ?

{selection.item.impact}

: null} + {selection.item.reviewPlan?.retryOriginal && selection.item.errorMessage + ?

{selection.item.errorMessage}

+ : selection.item.impact ?

{selection.item.impact}

: null} {selection.item.reviewPlan && !selection.item.reviewPlan.creationExecution && !selection.item.reviewPlan.retryOriginal && !taskExecution && selection.item.actionKind !== "team.plan" && !(decisionFrame && ["ready", "applied"].includes(selection.item.status)) ?

{operationUnknown ? t("actionReview.operation_reconcile_original") : selection.item.reviewPlan.operationFrame?.kind === "inactive" diff --git a/apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-page.tsx b/apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-page.tsx index 94e96ece80..66f5c99f70 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-page.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-page.tsx @@ -1832,10 +1832,21 @@ export function PersonalWorkspacePage({ // not have a Turn to revisit. .filter(proposal => proposal.actionKind !== "todo.create" || proposal.status !== "applied" || Boolean(proposal.reviewPlan?.taskExecution)); - const restored = Object.fromEntries(restoreable.map((proposal) => [proposal.previewId, proposal])); const previousIds = restoredProposalIdsRef.current; - restoredProposalIdsRef.current = new Set(Object.keys(restored)); - setProposals((current) => ({...Object.fromEntries(Object.entries(current).filter(([id]) => !previousIds.has(id))), ...restored})); + restoredProposalIdsRef.current = new Set(restoreable.map(proposal => proposal.previewId)); + setProposals((current) => { + const next = Object.fromEntries(Object.entries(current).filter(([id]) => !previousIds.has(id))); + for (const proposal of restoreable) { + const previous = current[proposal.previewId]; + // The stored confirmation outlives a failed dispatch attempt. + // Keep the local reason until readback observes a different confirmation. + const retryError = proposal.reviewPlan?.retryOriginal && previous?.reviewPlan?.retryOriginal + && previous.actionKind === proposal.actionKind && previous.goalId === proposal.goalId + && previous.updatedAt === proposal.updatedAt ? previous.errorMessage : undefined; + next[proposal.previewId] = retryError ? { ...proposal, errorMessage: retryError } : proposal; + } + return next; + }); // The manager conversation shows the cards this channel offered: a team // plan the steward proposed from here is confirmed here, instead of the // owner hunting for the Goal whose workspace happens to hold the card. @@ -2140,7 +2151,7 @@ export function PersonalWorkspacePage({ : null; const lifecycleChange = options.lifecycleProjection ?? inferredLifecycleChange; setActionFeedback(t("feedback.applying", { title: proposal.title })); - const applying = { ...proposal, reviewPlan: proposal.reviewPlan ? { ...proposal.reviewPlan, interaction: "pending" as const, reason: "apply_pending" as const, canApply: false as const } : undefined, status: "applying" as const }; + const applying = { ...proposal, errorMessage: undefined, reviewPlan: proposal.reviewPlan ? { ...proposal.reviewPlan, interaction: "pending" as const, reason: "apply_pending" as const, canApply: false as const } : undefined, status: "applying" as const }; setProposals((current) => ({ ...current, [proposal.previewId]: applying })); if (showDrawer) setSelection({ item: applying, kind: "proposal" }); let applyingSelection = selectionRevision.current; diff --git a/loopx/canary/module_metric_baseline.json b/loopx/canary/module_metric_baseline.json index 9e224aaf1d..8b9a42cbde 100644 --- a/loopx/canary/module_metric_baseline.json +++ b/loopx/canary/module_metric_baseline.json @@ -45,6 +45,11 @@ "loopx/capabilities/issue_fix/pr_lifecycle.py": { "lines": 2056 }, + "loopx/capabilities/native_chat/conversation_bindings.py": { + "any_count": 0, + "dict_any_count": 0, + "lines": 1108 + }, "loopx/capabilities/reward_memory/utility_reducer.py": { "any_count": 0, "dict_any_count": 0, diff --git a/loopx/capabilities/native_chat/conversation_bindings.py b/loopx/capabilities/native_chat/conversation_bindings.py index 76a3e2c34e..707ff02363 100644 --- a/loopx/capabilities/native_chat/conversation_bindings.py +++ b/loopx/capabilities/native_chat/conversation_bindings.py @@ -6,118 +6,230 @@ """ from __future__ import annotations -from collections.abc import Callable, Iterator, Mapping -from contextlib import contextmanager +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import AbstractContextManager, contextmanager +import hashlib +import json import logging from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Protocol, TypeAlias, TypedDict, TypeVar, cast import uuid -import hashlib from ...chat_store import TERMINAL_TURN_STATES, _atomic_write_json, _read_json -from ...control_plane.effect_runtime import ( - EffectRuntimeRejected, effect_runtime_request_scope, effect_runtime_result, -) +from ...control_plane.collaboration import inbox +from ...control_plane.collaboration.goal_instance_scope import collaboration_goal_scope +from ...control_plane.collaboration.source_policy_fence import source_policy_fence +from ...control_plane.effect_runtime import EffectRuntimeRejected, effect_runtime_request_scope, effect_runtime_result from ...file_lock import exclusive_file_lock +JsonDict: TypeAlias = dict[str, object] +JsonMap: TypeAlias = Mapping[str, object] +_T = TypeVar("_T") + if TYPE_CHECKING: from ...chat_store import ChatSessionStore + from ...control_plane.collaboration.goal_instance_scope import CollaborationGoalScope + class _ProjectContexts(Protocol): + workspace_grant: str + conversation_bindings: ChatConversationBindings | None + def available(self) -> Sequence[JsonMap]: ... + def session_context(self, session: JsonDict) -> JsonDict: ... + class _BindingController(Protocol): + store: ChatSessionStore + coordination_runtime_root: Path + registry_path: Path | None + @property + def project_contexts(self) -> _ProjectContexts: ... + def interrupt_turn(self, *, session_id: str, turn_id: str) -> JsonDict: ... + def resume_session_queue(self, *, session_id: str, work_dir: Path, objective: str) -> None: ... + class _NativePeerSession(Protocol): + def send(self, user_message: str, *, project_collaboration: bool = False, + turn_start_guard: Callable[[], AbstractContextManager[None]] | None = None, + on_event: Callable[[str, JsonDict], None] | None = None) -> JsonDict: ... + class _RequestLockFn(Protocol): + def __call__(self, root: Path, request_id: str, scope: object, legacy_path: Path, *, force_exact: bool = False) -> AbstractContextManager[None]: ... + class _RecordResultFn(Protocol): + def __call__(self, root: Path, row: JsonMap, phase: str, text: str, *, request_revision: int | None = None) -> JsonDict: ... + class _AgentObservation(TypedDict): + session: JsonDict + goal: JsonDict + host_binding_verified: bool + host_ref: str + host_audience_binding_ids: list[str] + _EventSink: TypeAlias = Callable[[str, JsonDict], None] + + +def _need(value: object, kind: type[_T], message: str = "unexpected value type") -> _T: + if not isinstance(value, kind): + raise ValueError(message) + return value + + +def _as_dict(value: object, message: str = "expected JSON object") -> JsonDict: + return cast("JsonDict", _need(value, dict, message)) + + +def _dict_or_empty(value: object) -> JsonDict: + return cast("JsonDict", value) if isinstance(value, dict) else {} + + +def _dict_list(value: object, message: str = "expected list of objects") -> list[JsonDict]: + return [_as_dict(item, message) for item in _need(value, list, message)] + + +def _find_binding(state: JsonMap, key: str, value: object, *, default_empty: bool = False) -> JsonDict | None: + rows = state.get("bindings", []) if default_empty else state["bindings"] + if not isinstance(rows, list): + raise ValueError("invalid conversation bindings state") + return next((cast("JsonDict", row) for row in rows if isinstance(row, dict) and row.get(key) == value), None) + + +def _require_binding(state: JsonMap, key: str, value: object) -> JsonDict: + if (row := _find_binding(state, key, value)) is None: + raise ValueError("conversation binding is no longer authorized") + return row + + +def _session_mentions_binding(session: JsonMap, binding_id: str) -> bool: + return any(isinstance(ctx := session.get(k), dict) and ctx.get("binding_id") == binding_id for k in ("project_context", "steward_context")) + + +def _iter_open_session_turns(store: ChatSessionStore) -> Iterator[tuple[JsonDict, JsonDict]]: + for row in store.list_sessions(): + if (session := store.load_session(_need(row["session_id"], str))) is None: + continue + session_id = _need(session["session_id"], str) + turns = store.queued_turns(session_id) + if isinstance(active_id := session.get("active_turn_id"), str) and active_id and (active := store.load_turn(session_id, active_id)) is not None: + turns.append(active) + for turn in turns: + if turn.get("status") not in TERMINAL_TURN_STATES: + yield session, turn + + +def _host_observation_fields(store: ChatSessionStore, session: JsonMap, goal: JsonMap) -> JsonDict: + return {"goal": {"goal_id": goal["id"], "workspace_path": str(Path(_need(goal["repo"], str)).resolve()), + "registered_agents": _dict_or_empty(goal.get("coordination")).get("registered_agents", [])}, + "host_binding_verified": True, "host_ref": _host_ref(session), "host_audience_binding_ids": _host_audiences(store, session)} class ChatConversationBindings: - def __init__(self, *, root: Path, project_contexts: Any, - observe: Callable[[str], dict[str, Any]], - observe_group: Callable[[str], dict[str, Any]] | None = None) -> None: - self.path = root / "conversation-bindings.json" - self.projects = project_contexts - self._observe_owner = observe - self._observe_group = observe_group - self.controller: Any | None = None - - def observe(self, transport_ref: str, *, audience: str | None = None) -> dict[str, Any]: - row: dict[str, Any] = next((item for item in self.read()["bindings"] if item["transport_ref"] == transport_ref), {}) + def __init__(self, *, root: Path, project_contexts: _ProjectContexts | None, + observe: Callable[[str], JsonDict], observe_group: Callable[[str], JsonDict] | None = None) -> None: + self.path, self.projects = root / "conversation-bindings.json", project_contexts + self._observe_owner, self._observe_group = observe, observe_group + self.controller: _BindingController | None = None + + def _require_projects(self) -> _ProjectContexts: + if self.projects is None: + raise ValueError("workspace grants unavailable") + return self.projects + + def _require_controller(self) -> _BindingController: + if self.controller is None: + raise ValueError("the bound conversation store is unavailable") + return self.controller + + def _publish_state(self, result: JsonMap, *, error_message: str, preserve_mode: bool = False) -> JsonDict: + state = _as_dict(result["state"]) + if result["changed"]: + _atomic_write_json(self.path, state, preserve_mode=True) if preserve_mode else _atomic_write_json(self.path, state) + if (readback := self.read()) != state: + raise OSError(error_message) + return readback + + def _verify_unchanged(self, key: str, value: object, expected: JsonMap, *, + message: str = "conversation binding changed during identity verification", default_empty: bool = False) -> JsonDict: + current = self.read() + if _find_binding(current, key, value, default_empty=default_empty) != expected: + raise ValueError(message) + return current + + def _resolve_source(self, binding_id: str, source: JsonMap, *, include_observation: bool = False, session: JsonMap | None = None) -> JsonDict: + session_context, group_ref, topic_ref, origin = source.get("session_context"), source.get("group_ref"), source.get("topic_ref"), source.get("origin") + return self.resolve( + binding_id=binding_id, source_ref=_need(source["source_ref"], str), sender_ref=_need(source["sender_ref"], str), + private_human_message=_need(source["private_human_message"], bool), + session_context=_as_dict(session_context) if session_context is not None else None, + group_ref=_need(group_ref, str) if group_ref is not None else None, topic_ref=_need(topic_ref, str) if topic_ref is not None else None, + group_human_message=_need(source.get("group_human_message", False), bool), include_observation=include_observation, + origin=_need(origin, str) if origin is not None else None, session=session, + ) + + def observe(self, transport_ref: str, *, audience: str | None = None) -> JsonDict: + row = _find_binding(self.read(), "transport_ref", transport_ref) or {} if audience is None: - audience = row.get("audience", "owner") + audience = raw if isinstance(raw := row.get("audience"), str) else "owner" if audience == "group" and self._observe_group is not None: observed = self._observe_group(transport_ref) # Earlier group grants used the verified user's principal. Keep # that exact identity/home until an explicit disconnect/reconnect; # changing it during a poll would invalidate resumed Sessions. - if (row.get("audience") == "group" and row.get("provider_ref") == observed["provider_ref"] - and row.get("operator_ref") != observed["operator_ref"]): + if row.get("audience") == "group" and row.get("provider_ref") == observed["provider_ref"] and row.get("operator_ref") != observed["operator_ref"]: return self._observe_owner(transport_ref) return observed return self._observe_owner(transport_ref) - def read(self) -> dict[str, Any]: + def read(self) -> JsonDict: if not self.path.exists(): return {"schema_version": "loopx_chat_conversation_bindings_v0", "revision": 0, "bindings": []} # An unreadable existing grant is not an empty configuration to overwrite. - return _read_json(self.path) + return cast("JsonDict", _read_json(self.path)) @staticmethod - def _core(operation: str, params: dict[str, Any], *, large_local_snapshot: bool = False) -> dict[str, Any]: + def _core(operation: str, params: JsonMap, *, large_local_snapshot: bool = False) -> JsonDict: try: result = effect_runtime_result(operation, params, large_local_snapshot=large_local_snapshot) except EffectRuntimeRejected as exc: raise ValueError(str(exc)) from exc if not isinstance(result, dict): raise ValueError("conversation authority returned an invalid result") - return result - - def configure(self, *, transport_ref: str, project_ref: str, - executor_endpoint_id: str, context_kind: str = "project", - project_grant: str | None = None, goal_scope: str | None = None, - audience: str | None = None, group_refs: list[str] | None = None, - available_group_refs: list[str] | None = None, - expected_provider_ref: str | None = None, - expected_operator_ref: str | None = None) -> dict[str, Any]: + return cast("JsonDict", result) + + def configure(self, *, transport_ref: str, project_ref: str, executor_endpoint_id: str, + context_kind: str = "project", project_grant: str | None = None, goal_scope: str | None = None, + audience: str | None = None, group_refs: list[str] | None = None, available_group_refs: list[str] | None = None, + expected_provider_ref: str | None = None, expected_operator_ref: str | None = None) -> JsonDict: if goal_scope is not None and (context_kind != "steward" or goal_scope not in {"selected", "all_registered"}): raise ValueError("select a supported steward Goal scope") + projects = self._require_projects() if project_grant is None: - project_grant = self.projects.workspace_grant if context_kind == "project" and executor_endpoint_id == "codex" else "workspace_read" + project_grant = projects.workspace_grant if context_kind == "project" and executor_endpoint_id == "codex" else "workspace_read" if project_grant not in {"workspace_read", "workspace_write"} or (context_kind != "project" and project_grant != "workspace_read"): raise ValueError("workspace write authorization is only available for project Chat") observation = self.observe(transport_ref, audience=audience or "owner") - if expected_provider_ref is not None and observation["provider_ref"] != expected_provider_ref: - raise ValueError("the selected App changed before binding publication; verify it again") - if expected_operator_ref is not None and observation["operator_ref"] != expected_operator_ref: - raise ValueError("the selected owner changed before binding publication; verify it again") - candidate = { - "schema_version": "loopx_chat_conversation_binding_v0", - "binding_id": uuid.uuid4().hex[:24], "transport_ref": transport_ref, - "provider_ref": observation["provider_ref"], "operator_ref": observation["operator_ref"], - "context_kind": context_kind, "project_ref": project_ref, - "executor_endpoint_id": executor_endpoint_id, + for key, expected, label in (("provider_ref", expected_provider_ref, "App"), ("operator_ref", expected_operator_ref, "owner")): + if expected is not None and observation[key] != expected: + raise ValueError(f"the selected {label} changed before binding publication; verify it again") + candidate: JsonDict = { + "schema_version": "loopx_chat_conversation_binding_v0", "binding_id": uuid.uuid4().hex[:24], + "transport_ref": transport_ref, "provider_ref": observation["provider_ref"], "operator_ref": observation["operator_ref"], + "context_kind": context_kind, "project_ref": project_ref, "executor_endpoint_id": executor_endpoint_id, "grant": project_grant if context_kind == "project" else "portfolio_read", "enabled": True, **({"goal_ids": [], "goal_scope": goal_scope or "all_registered"} if context_kind == "steward" else {}), - **({"audience": audience} if audience is not None else {}), - **({"group_refs": group_refs} if group_refs is not None else {}), + **({"audience": audience} if audience is not None else {}), **({"group_refs": group_refs} if group_refs is not None else {}), } with exclusive_file_lock(self.path, operation="configure_chat_conversation_binding"): current = self.read() - previous = next((row for row in current["bindings"] if row["transport_ref"] == transport_ref), None) - if previous and all(previous.get(key) == candidate.get(key) for key in ["context_kind", "project_ref", "executor_endpoint_id", "provider_ref", "operator_ref", "grant", "audience", "group_refs"]): - if previous.get("agent_targets"): + previous = _find_binding(current, "transport_ref", transport_ref) + if previous is not None and all(previous.get(k) == candidate.get(k) for k in ("context_kind", "project_ref", "executor_endpoint_id", "provider_ref", "operator_ref")): + if all(previous.get(k) == candidate.get(k) for k in ("grant", "audience", "group_refs")) and previous.get("agent_targets"): candidate["agent_targets"] = previous["agent_targets"] - if (context_kind == "steward" and previous and all(previous.get(key) == candidate.get(key) - for key in ["context_kind", "project_ref", "executor_endpoint_id", "provider_ref", "operator_ref"])): - candidate["goal_ids"] = previous["goal_ids"] - # Existing installations retain their exact audience and scope - # until the local operator explicitly changes it. - if goal_scope is None: - candidate.pop("goal_scope", None) - if "goal_scope" in previous: - candidate["goal_scope"] = previous["goal_scope"] - candidate["binding_id"] = previous["binding_id"] - if previous.get("managed_actor") is not None: - candidate["managed_actor"] = previous["managed_actor"] + if context_kind == "steward": + candidate.update(goal_ids=previous["goal_ids"], binding_id=previous["binding_id"]) + # Existing installations retain their exact audience and scope + # until the local operator explicitly changes it. + if goal_scope is None: + candidate.pop("goal_scope", None) + if "goal_scope" in previous: + candidate["goal_scope"] = previous["goal_scope"] + if previous.get("managed_actor") is not None: + candidate["managed_actor"] = previous["managed_actor"] result = self._core("collaboration.conversation.binding", { - "current": current, "expected_revision": current.get("revision"), "operation": "configure", - "binding": candidate, "observation": observation, "available_projects": self.projects.available(), - **({"available_group_refs": available_group_refs} if audience == "group" else {}), + "current": current, "expected_revision": current.get("revision"), "operation": "configure", "binding": candidate, + "observation": observation, "available_projects": projects.available(), **({"available_group_refs": available_group_refs} if audience == "group" else {}), }) - proposed = next(row for row in result["state"]["bindings"] if row["transport_ref"] == transport_ref) + proposed = _require_binding(_as_dict(result["state"]), "transport_ref", transport_ref) channels = self.delivery_channels(proposed) if goal_scope is not None else [] for channel in channels: self._ensure_delivery_scope({"binding": proposed, "channel_id": channel}, execute=False) @@ -128,72 +240,50 @@ def configure(self, *, transport_ref: str, project_ref: str, for channel in channels: self._ensure_delivery_scope({"binding": proposed, "channel_id": channel}, if_absent=False) if previous and previous["binding_id"] != proposed["binding_id"]: - self._revoke_delivery_sources(previous["binding_id"]) - if result["changed"]: - _atomic_write_json(self.path, result["state"]) - readback = self.read() - if readback != result["state"]: - raise OSError("conversation binding publication did not verify") - binding: dict[str, Any] = next(row for row in readback["bindings"] if row["transport_ref"] == transport_ref) + self._revoke_delivery_sources(_need(previous["binding_id"], str)) + binding = _require_binding(self._publish_state(result, error_message="conversation binding publication did not verify"), "transport_ref", transport_ref) if previous and previous["binding_id"] != binding["binding_id"]: - self._stop_revoked_binding(previous["binding_id"]) + self._stop_revoked_binding(_need(previous["binding_id"], str)) return binding - def disconnect(self, binding_id: str, *, expected_revision: int) -> dict[str, Any]: + def disconnect(self, binding_id: str, *, expected_revision: int) -> JsonDict: with exclusive_file_lock(self.path, operation="disconnect_chat_conversation_binding"): - result = self._core("collaboration.conversation.binding", { - "current": self.read(), "expected_revision": expected_revision, - "operation": "disconnect", "binding_id": binding_id, - }) + result = self._core("collaboration.conversation.binding", {"current": self.read(), "expected_revision": expected_revision, "operation": "disconnect", "binding_id": binding_id}) self._revoke_delivery_sources(binding_id) - if result["changed"]: - _atomic_write_json(self.path, result["state"]) - if self.read() != result["state"]: - raise OSError("conversation binding revocation did not verify") + state = self._publish_state(result, error_message="conversation binding revocation did not verify") self._stop_revoked_binding(binding_id) - return dict(result["state"]) + return dict(state) - def select_app_steward(self, binding_id: str | None, *, expected_revision: int) -> dict[str, Any]: + def select_app_steward(self, binding_id: str | None, *, expected_revision: int) -> JsonDict: """Select one existing source for the App entry without changing grants.""" - row = next((item for item in self.read()["bindings"] if item["binding_id"] == binding_id), None) - observation = self.observe(row["transport_ref"]) if row is not None else None + row = _find_binding(self.read(), "binding_id", binding_id) if binding_id is not None else None + observation = self.observe(_need(row["transport_ref"], str)) if row is not None else None with exclusive_file_lock(self.path, operation="select_app_steward"): result = self._core("collaboration.conversation.binding", { - "current": self.read(), "expected_revision": expected_revision, - "operation": "select_app_steward", "binding_id": binding_id, - "observation": observation, "available_projects": self.projects.available() if binding_id is not None else [], + "current": self.read(), "expected_revision": expected_revision, "operation": "select_app_steward", + "binding_id": binding_id, "observation": observation, "available_projects": self._require_projects().available() if binding_id is not None else [], }) - if result["changed"]: - _atomic_write_json(self.path, result["state"]) - if self.read() != result["state"]: - raise OSError("App steward selection did not verify") - return dict(result["state"]) + return dict(self._publish_state(result, error_message="App steward selection did not verify")) def reconcile_steward_executors(self) -> None: """Publish a previously adopted machine executor at Chat startup.""" from ...chat_manager import steward_machine_defaults - controller = self.controller - if controller is None: + if (controller := self.controller) is None: return defaults = steward_machine_defaults(controller) - if (not defaults or defaults.get("status") != "ready" - or defaults.get("source") != "machine_configuration" - or defaults.get("selection_policy") not in {"preferred", "pinned"} - or defaults.get("executor_endpoint") not in {"antigravity", "codex"}): + if (not defaults or defaults.get("status") != "ready" or defaults.get("source") != "machine_configuration" + or defaults.get("selection_policy") not in {"preferred", "pinned"} or defaults.get("executor_endpoint") not in {"antigravity", "codex"}): return try: - rows = self.read()["bindings"] - if not isinstance(rows, list): + if not isinstance(rows := self.read()["bindings"], list): raise ValueError("invalid conversation bindings state") except (OSError, ValueError, KeyError) as exc: logging.getLogger(__name__).warning("Steward bindings unavailable for reconciliation (%s)", type(exc).__name__) return for row in rows: - if (not isinstance(row, dict) or row.get("context_kind") != "steward" - or row.get("audience", "owner") != "owner" - or row.get("enabled") is not True - or row.get("executor_endpoint_id") not in {"antigravity", "codex"} + if (not isinstance(row, dict) or row.get("context_kind") != "steward" or row.get("audience", "owner") != "owner" + or row.get("enabled") is not True or row.get("executor_endpoint_id") not in {"antigravity", "codex"} or row.get("executor_endpoint_id") == defaults["executor_endpoint"]): continue try: @@ -201,103 +291,84 @@ def reconcile_steward_executors(self) -> None: except (OSError, ValueError, TypeError, KeyError, RuntimeError) as exc: logging.getLogger(__name__).warning("Steward executor reconciliation held (%s)", type(exc).__name__) - def _reconcile_steward_executor(self, row: dict[str, Any], defaults: Mapping[str, Any]) -> None: + def _reconcile_steward_executor(self, row: JsonDict, defaults: JsonMap) -> None: from ...chat_manager import manager_executor_allocation, steward_machine_defaults - from ..machine_configuration.builtins import build_builtin_machine_configuration_registry - from ..machine_configuration.store import machine_configuration_revision, read_machine_configuration - from ..steward_executor.allocation import normalize_manager_executor_allocation - from ..steward_executor.machine_defaults import effective_steward_executor_defaults + from ..machine_configuration import builtins as _mc_builtins, contract as _mc_contract, store as _mc_store + from ..steward_executor import allocation as _alloc, machine_defaults as _md - store = self.controller.store - binding_id = row["binding_id"] - endpoint = defaults["executor_endpoint"] - channel_prefix = f"manager.external.native.{binding_id}." - candidates = [s for s in store.list_sessions() - if s.get("channel_id", "").startswith(channel_prefix) and s.get("status") != "closed"] + controller = self._require_controller() + store, binding_id, endpoint = controller.store, _need(row["binding_id"], str), defaults["executor_endpoint"] + prefix = f"manager.external.native.{binding_id}." + candidates = [s for s in store.list_sessions() if str(s.get("channel_id") or "").startswith(prefix) and s.get("status") != "closed"] if len(candidates) != 1 or candidates[0].get("agent_id") != endpoint or candidates[0].get("status") != "ready": return # Defaults alone do not retarget an existing session. - observation = self.observe(row["transport_ref"]) + observation, session_id = self.observe(_need(row["transport_ref"], str)), _need(candidates[0]["session_id"], str) with exclusive_file_lock(self.path, operation="reconcile_steward_executor"): current = self.read() - if row not in current["bindings"] or steward_machine_defaults(self.controller) != defaults: + if row not in _dict_list(current["bindings"]) or steward_machine_defaults(controller) != defaults or not (session := store.load_session(session_id)): return - session = store.load_session(candidates[0]["session_id"]) - saved = (session or {}).get("steward_context") - stored = (session or {}).get("manager_executor_allocation") - if (not session or not isinstance(saved, dict) or not isinstance(stored, dict) - or session.get("status") != "ready" or session.get("active_turn_id") is not None - or session.get("agent_id") != endpoint or session.get("executor_endpoint_id") != endpoint - or stored.get("selection_policy") != defaults["selection_policy"] - or store.queued_turns(session["session_id"], strict=True, include_reserved=True)): + saved, stored = session.get("steward_context"), session.get("manager_executor_allocation") + if (not isinstance(saved, dict) or not isinstance(stored, dict) or session.get("status") != "ready" + or session.get("active_turn_id") is not None or session.get("agent_id") != endpoint or session.get("executor_endpoint_id") != endpoint + or stored.get("selection_policy") != defaults["selection_policy"] or store.queued_turns(session_id, strict=True, include_reserved=True)): return - allocation = None + allocation: JsonDict | None = None if stored.get("configuration_revision") == defaults.get("configuration_revision"): try: - allocation = normalize_manager_executor_allocation(stored) + allocation = _alloc.normalize_manager_executor_allocation(stored) except (TypeError, ValueError): pass else: # A machine configuration transaction records the revision of # the whole document. Match that exact basis before projecting # it to the narrower revision used by steward allocations. - configuration = read_machine_configuration(store.root.parent, - registry=build_builtin_machine_configuration_registry()) - if (not configuration or machine_configuration_revision(configuration) != stored.get("configuration_revision") - or effective_steward_executor_defaults(configuration) != defaults): + configuration = _mc_store.read_machine_configuration(store.root.parent, registry=_mc_builtins.build_builtin_machine_configuration_registry()) + if (not configuration or _mc_contract.machine_configuration_revision(configuration) != stored.get("configuration_revision") + or _md.effective_steward_executor_defaults(configuration) != defaults): return if allocation is None: - allocation = manager_executor_allocation(self.controller, machine_defaults=defaults) - expected = {**self._session_observation(session), "manager_executor_allocation": allocation, - "active_turn_id": session.get("active_turn_id"), "agent_model_id": session.get("agent_model_id"), - "agent_reasoning_effort": session.get("agent_reasoning_effort")} + allocation = manager_executor_allocation(controller, machine_defaults=defaults) + expected: JsonDict = {**self._session_observation(session), "manager_executor_allocation": allocation, + "active_turn_id": session.get("active_turn_id"), "agent_model_id": session.get("agent_model_id"), + "agent_reasoning_effort": session.get("agent_reasoning_effort")} result = self._core("collaboration.conversation.binding", { - "current": current, "expected_revision": current["revision"], "operation": "reconcile_steward_executor", - "binding_id": binding_id, "observation": observation, "available_projects": self.projects.available(), - "available_goal_ids": self.goal_scope_ids(row), "machine_defaults": dict(defaults), - "session": expected, "queue_empty": True, + "current": current, "expected_revision": current["revision"], "operation": "reconcile_steward_executor", "binding_id": binding_id, + "observation": observation, "available_projects": self._require_projects().available(), "available_goal_ids": self.goal_scope_ids(row), + "machine_defaults": dict(defaults), "session": expected, "queue_empty": True, }) if stored != allocation: - store.update_session(session["session_id"], expected_statuses={"ready"}, - expected_upstream_thread_id=session.get("upstream_thread_id"), manager_executor_allocation=allocation) - fresh = store.load_session(session["session_id"]) - if (not fresh or any(fresh.get(key) != value for key, value in expected.items()) - or store.queued_turns(session["session_id"], strict=True, include_reserved=True) - or steward_machine_defaults(self.controller) != defaults): + upstream = session.get("upstream_thread_id") + store.update_session(session_id, expected_statuses={"ready"}, expected_upstream_thread_id=upstream if isinstance(upstream, str) else None, + manager_executor_allocation=allocation) + fresh = store.load_session(session_id) + if (not fresh or any(fresh.get(k) != v for k, v in expected.items()) + or store.queued_turns(session_id, strict=True, include_reserved=True) or steward_machine_defaults(controller) != defaults): return - if result["changed"]: - _atomic_write_json(self.path, result["state"], preserve_mode=True) - if self.read() != result["state"]: - raise OSError("steward executor binding publication did not verify") - - def app_steward_target(self) -> dict[str, Any]: - session = self.app_steward_session() - if session is None: + self._publish_state(result, error_message="steward executor binding publication did not verify", preserve_mode=True) + + def app_steward_target(self) -> dict[str, str | None]: + if (session := self.app_steward_session()) is None: return {"binding_id": None, "session_id": None} - return {"binding_id": session["steward_context"]["binding_id"], "session_id": session["session_id"]} + return {"binding_id": _need(_as_dict(session["steward_context"])["binding_id"], str), "session_id": _need(session["session_id"], str)} - def app_steward_session(self) -> dict[str, Any] | None: + def app_steward_session(self) -> JsonDict | None: """Read the Session within the selected source without starting work.""" current = self.read() - selected = current.get("app_steward_binding_id") - if selected is None: + if (selected := current.get("app_steward_binding_id")) is None: # Validate legacy state through the same pure owner plan. An # unreadable grant must not masquerade as an unselected entry. - self._core("collaboration.conversation.binding", {"current": current, - "expected_revision": current.get("revision"), "operation": "select_app_steward", "binding_id": None}) + self._core("collaboration.conversation.binding", {"current": current, "expected_revision": current.get("revision"), "operation": "select_app_steward", "binding_id": None}) return None - current, sessions, _ = self._observe_local_sessions(selected, app_steward_entry=True) + current, sessions, _ = self._observe_local_sessions(_need(selected, str), app_steward_entry=True) if not sessions: raise ValueError("the original steward Session is unavailable") - session = max(sessions, key=lambda item: (item.get("last_activity_at") or "", item["session_id"])) + session = max(sessions, key=lambda item: (str(item.get("last_activity_at") or ""), _need(item["session_id"], str))) # Close uses the same Session fences. Keep it from committing between # this read and the final grant decision; grant revocation still races # freely and is detected by the version check. All provider/Core IO has # already completed, and background grant inspection remains nonblocking. - with self.controller.store.read_session_fenced(session["session_id"]) as fresh: - if fresh is None or fresh.get("status") == "closed": - raise ValueError("the original steward Session is unavailable") - expected = {**self._session_observation(session), "status": fresh.get("status")} - if self._session_observation(fresh) != expected: + with self._require_controller().store.read_session_fenced(_need(session["session_id"], str)) as fresh: + if fresh is None or fresh.get("status") == "closed" or self._session_observation(fresh) != {**self._session_observation(session), "status": fresh.get("status")}: raise ValueError("the original steward Session is unavailable") self._recheck_local_bindings(current) return fresh @@ -315,129 +386,93 @@ def _stop_revoked_binding(self, binding_id: str) -> None: local Turns in a shared attached Session retain their own authority. Retrying disconnect with the current revision retries an unconfirmed stop. """ - if self.controller is None: + if (controller := self.controller) is None: return - store = self.controller.store - failures = [] - for row in store.list_sessions(): - session = store.load_session(row["session_id"]) - if session is None: + failures: list[Exception] = [] + for session, turn in _iter_open_session_turns(controller.store): + if not _session_mentions_binding(session, binding_id) and _dict_or_empty(_dict_or_empty(turn.get("external_agent_target")).get("context")).get("binding_id") != binding_id: continue - bound = any(isinstance(session.get(key), dict) - and session[key].get("binding_id") == binding_id - for key in ("project_context", "steward_context")) - turns = store.queued_turns(session["session_id"]) - active = store.load_turn(session["session_id"], session["active_turn_id"]) if session.get("active_turn_id") else None - if active: - turns.append(active) - for turn in turns: - target = turn.get("external_agent_target") or {} - if (not bound and (target.get("context") or {}).get("binding_id") != binding_id - or turn.get("status") in TERMINAL_TURN_STATES): - continue - try: - stopped = self.controller.interrupt_turn(session_id=session["session_id"], turn_id=turn["turn_id"]) - if stopped.get("status") not in TERMINAL_TURN_STATES: - raise RuntimeError("Turn stop did not settle") - except Exception as exc: - # Revocation remains published even when a host lacks stop - # control. Continue stopping other exact targets, then report - # failure without exposing private transport diagnostics. - failures.append(exc) + try: + if controller.interrupt_turn(session_id=_need(session["session_id"], str), turn_id=_need(turn["turn_id"], str)).get("status") not in TERMINAL_TURN_STATES: + raise RuntimeError("Turn stop did not settle") + except Exception as exc: + # Revocation remains published even when a host lacks stop + # control. Continue stopping other exact targets, then report + # failure without exposing private transport diagnostics. + failures.append(exc) if failures: raise OSError("Conversation grant revoked; execution stop is not confirmed. Retry stop for this binding.") from failures[0] - def resolve(self, *, binding_id: str, source_ref: str, sender_ref: str, - private_human_message: bool, session_context: dict[str, Any] | None = None, - group_ref: str | None = None, topic_ref: str | None = None, - group_human_message: bool = False, include_observation: bool = False, - origin: str | None = None, session: dict[str, Any] | None = None) -> dict[str, Any]: - current = self.read() - row = next((item for item in current.get("bindings", []) if item.get("binding_id") == binding_id), None) - if row is None: + def resolve(self, *, binding_id: str, source_ref: str, sender_ref: str, private_human_message: bool, + session_context: JsonMap | None = None, group_ref: str | None = None, topic_ref: str | None = None, + group_human_message: bool = False, include_observation: bool = False, origin: str | None = None, session: JsonMap | None = None) -> JsonDict: + if (row := _find_binding(self.read(), "binding_id", binding_id, default_empty=True)) is None: raise ValueError("conversation binding is no longer authorized") - observation = dict(self.observe(row["transport_ref"])) + observation = dict(self.observe(_need(row["transport_ref"], str))) # Provider verification can wait on IO. It cannot authorize a binding # that was revoked or replaced while that observation was in flight. # Unrelated binding changes do not invalidate this exact source. - current = self.read() - if next((item for item in current.get("bindings", []) - if item.get("binding_id") == binding_id), None) != row: - raise ValueError("conversation binding changed during identity verification") + current = self._verify_unchanged("binding_id", binding_id, row, default_empty=True) selected = self._resolve_observed(current, row, observation, { "source_ref": source_ref, "sender_ref": sender_ref, "private_human_message": private_human_message, - **({"origin": origin} if origin is not None else {}), - **({"session_context": session_context} if session_context is not None else {}), + **({"origin": origin} if origin is not None else {}), **({"session_context": session_context} if session_context is not None else {}), **({"session": self._session_observation(session)} if session is not None else {}), - **({"group_ref": group_ref, "topic_ref": topic_ref, - "group_human_message": group_human_message} if group_ref is not None or topic_ref is not None or group_human_message else {}), + **({"group_ref": group_ref, "topic_ref": topic_ref, "group_human_message": group_human_message} if group_ref is not None or topic_ref is not None or group_human_message else {}), }) # Presentation may need the name from this exact verified read. Keep it # outside the persisted context and opt-in; it is never a cached grant # or a substitute for the next admission/delivery owner's fresh check. return {**selected, "observation": observation} if include_observation else selected - def _resolve_observed(self, current: dict[str, Any], row: dict[str, Any], - observation: dict[str, Any], source: dict[str, Any]) -> dict[str, Any]: + def _resolve_observed(self, current: JsonMap, row: JsonMap, observation: JsonMap, source: JsonMap) -> JsonDict: return self._core("collaboration.conversation.bound_context", { - "current": current, "binding_id": row["binding_id"], "observation": observation, - "available_projects": self.projects.available(), **source, + "current": current, "binding_id": row["binding_id"], "observation": observation, "available_projects": self._require_projects().available(), **source, **({"available_goal_ids": self.goal_scope_ids(row)} if row.get("goal_scope") == "all_registered" else {}), }) @contextmanager - def input_context(self, saved: dict[str, Any], *, origin: str, session: dict[str, Any], - store: ChatSessionStore) -> Iterator[dict[str, Any]]: + def input_context(self, saved: JsonMap, *, origin: str, session: JsonMap, store: ChatSessionStore) -> Iterator[JsonDict]: """Observe outside the publication fence, then retain it through enqueue. Saved fields select an identity; fresh Core checks grant it. Local Web continuation keeps its actual origin and never becomes a provider event. """ if origin == "web": - source = {"source_ref": saved["source_ref"], "sender_ref": saved["operator_ref"], - "private_human_message": False, "origin": "web", "session_context": saved} + source: JsonDict = {"source_ref": saved["source_ref"], "sender_ref": saved["operator_ref"], "private_human_message": False, "origin": "web", "session_context": saved} elif origin in {"lark", "external"}: source = self._session_source(saved) else: raise ValueError("unsupported bound conversation input origin") - observed = self.resolve(binding_id=saved["binding_id"], include_observation=True, session=session, **source) + binding_id = _need(saved["binding_id"], str) + observed = self._resolve_source(binding_id, source, include_observation=True, session=session) with exclusive_file_lock(self.path, operation="admit_bound_input"): - current = self.read() - if next((row for row in current.get("bindings", []) - if row.get("binding_id") == saved["binding_id"]), None) != observed["binding"]: - raise ValueError("conversation binding changed before input admission") + current = self._verify_unchanged("binding_id", binding_id, _as_dict(observed["binding"]), message="conversation binding changed before input admission", default_empty=True) # The provider observation may have waited. Re-read canonical # identity under the binding fence before Core authorizes input. - fresh = store.load_session(session["session_id"]) - if fresh is None: + if (fresh := store.load_session(_need(session["session_id"], str))) is None: raise KeyError("chat session was not found") - yield self._resolve_observed(current, observed["binding"], observed["observation"], - {**source, "session": self._session_observation(fresh)}) + yield self._resolve_observed(current, _as_dict(observed["binding"]), _as_dict(observed["observation"]), {**source, "session": self._session_observation(fresh)}) @staticmethod - def _session_observation(session: dict[str, Any]) -> dict[str, Any]: - return {key: session.get(key) for key in ( - "session_id", "goal_id", "agent_id", "executor_endpoint_id", "channel_id", - "session_mode", "status", "project_context", "steward_context", - "manager_executor_allocation", "upstream_context_history")} + def _session_observation(session: JsonMap) -> JsonDict: + return {key: session.get(key) for key in ("session_id", "goal_id", "agent_id", "executor_endpoint_id", "channel_id", + "session_mode", "status", "project_context", "steward_context", "manager_executor_allocation", "upstream_context_history")} - def local_sessions(self, binding_id: str, *, app_steward_entry: bool = False) -> list[dict[str, Any]]: - return self.local_session_snapshot(binding_id, app_steward_entry=app_steward_entry)["sessions"] + def local_sessions(self, binding_id: str, *, app_steward_entry: bool = False) -> list[JsonDict]: + return cast("list[JsonDict]", self.local_session_snapshot(binding_id, app_steward_entry=app_steward_entry)["sessions"]) - def local_session_snapshot(self, binding_id: str, *, app_steward_entry: bool = False) -> dict[str, Any]: + def local_session_snapshot(self, binding_id: str, *, app_steward_entry: bool = False) -> JsonDict: current, sessions, source = self._observe_local_sessions(binding_id, app_steward_entry=app_steward_entry) - payload = {"sessions": [self.controller.store.public_session(session) for session in sessions], - "conversation_source": source} + store = self._require_controller().store + payload: JsonDict = {"sessions": [store.public_session(dict(s)) for s in sessions], "conversation_source": source} self._recheck_local_bindings(current) return payload - def _recheck_local_bindings(self, observed: dict[str, Any]) -> None: + def _recheck_local_bindings(self, observed: JsonMap) -> None: if self.read() != observed: raise ValueError("conversation binding changed during discovery") - def _observe_local_sessions( - self, binding_id: str, *, app_steward_entry: bool, - ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: + def _observe_local_sessions(self, binding_id: str, *, app_steward_entry: bool) -> tuple[JsonDict, list[JsonDict], JsonDict]: """Read original targets; metadata hints never replace Core's grant. This read-only snapshot does not wait for a mutation fence. Core checks @@ -445,177 +480,139 @@ def _observe_local_sessions( changes. actual submissions retain their fresh identity and effect-time grant fences. """ - if self.controller is None: - raise ValueError("the bound conversation store is unavailable") - row = next((item for item in self.read()["bindings"] if item["binding_id"] == binding_id), None) - if row is None: - raise ValueError("conversation binding is no longer authorized") - observation = dict(self.observe(row["transport_ref"])) - store = self.controller.store - - def mentions_binding(session: dict[str, Any]) -> bool: - return any(isinstance(session.get(key), dict) and session[key].get("binding_id") == binding_id - for key in ("project_context", "steward_context")) - + store = self._require_controller().store + row = _require_binding(self.read(), "binding_id", binding_id) + observation = dict(self.observe(_need(row["transport_ref"], str))) # Keep aliases in the candidate set even when their own metadata is # unrelated. Core must still see duplicate canonical Session identities. - hints = (session for path, session in store.iter_session_metadata() - if path.parent.name != session["session_id"] or mentions_binding(session)) - sessions: list[dict[str, Any]] = [] - for hint in sorted(hints, key=lambda item: str(item.get("updated_at") or ""), reverse=True): - session = store.load_session(hint["session_id"]) - if session and mentions_binding(session): - sessions.append(session) - current = self.read() - if next((item for item in current["bindings"] if item["binding_id"] == binding_id), None) != row: - raise ValueError("conversation binding changed during identity verification") - projects = self.projects.available() + hints = (s for path, s in store.iter_session_metadata() if path.parent.name != s["session_id"] or _session_mentions_binding(s, binding_id)) + sessions = [s for h in sorted(hints, key=lambda item: str(item.get("updated_at") or ""), reverse=True) + if (s := store.load_session(_need(h["session_id"], str))) and _session_mentions_binding(s, binding_id)] + current, projects = self._verify_unchanged("binding_id", binding_id, row), self._require_projects().available() result = self._core("collaboration.conversation.local_sessions", { - "current": current, "binding_id": binding_id, "observation": observation, - "available_projects": projects, - "sessions": [self._session_observation(session) for session in sessions], - **({"app_steward_entry": True} if app_steward_entry else {}), + "current": current, "binding_id": binding_id, "observation": observation, "available_projects": projects, + "sessions": [self._session_observation(s) for s in sessions], **({"app_steward_entry": True} if app_steward_entry else {}), **({"available_goal_ids": self.goal_scope_ids(row)} if row.get("goal_scope") == "all_registered" else {}), }) # Build the lookup only after Core has rejected duplicate identity; # an IO dictionary must not silently turn ambiguity into one target. - by_id = {session["session_id"]: session for session in sessions} - project = next(project for project in projects if project["project_ref"] == row["project_ref"]) - source = {"binding_id": binding_id, "app_ref": row["transport_ref"], - "context_kind": row["context_kind"], "audience": row.get("audience", "owner"), - "project_title": Path(project["workspace_path"]).name, "context_available": True, - "executor_endpoint_id": row["executor_endpoint_id"]} + by_id = {_need(s["session_id"], str): s for s in sessions} + project = next(p for p in projects if p["project_ref"] == row["project_ref"]) + source: JsonDict = {"binding_id": binding_id, "app_ref": row["transport_ref"], "context_kind": row["context_kind"], "audience": row.get("audience", "owner"), + "project_title": Path(_need(project["workspace_path"], str)).name, "context_available": True, "executor_endpoint_id": row["executor_endpoint_id"]} # Bind the projection to the exact atomic configuration Core checked. # This is not an execution grant or a cached provider observation. - return current, [by_id[sid] for sid in result["session_ids"]], source + return current, [by_id[_need(sid, str)] for sid in _need(result["session_ids"], list)], source - def goal_scope_ids(self, binding: dict[str, Any]) -> list[str]: + def goal_scope_ids(self, binding: JsonMap) -> list[str]: if binding.get("goal_scope") != "all_registered": - return list(binding.get("goal_ids", [])) + return [_need(item, str) for item in raw_ids] if isinstance(raw_ids := binding.get("goal_ids", []), list) else [] from ...control_plane.projects.registry_codec import load_registry if self.controller is None or self.controller.registry_path is None: raise ValueError("the steward registry is unavailable") - return [goal["id"] for goal in load_registry(self.controller.registry_path).get("goals", [])] + return [_need(goal["id"], str) for goal in load_registry(self.controller.registry_path).get("goals", []) if isinstance(goal, dict)] - def delivery_channels(self, binding: dict[str, Any]) -> list[str]: + def delivery_channels(self, binding: JsonMap) -> list[str]: if self.controller is None: return [] from ..manager_context import POLICY_SCHEMA, _root - path = _root(self.controller.coordination_runtime_root) / "policy.json" - if not path.exists(): + if not (path := _root(self.controller.coordination_runtime_root) / "policy.json").exists(): return [] policy = _read_json(path) - if policy.get("schema_version") != POLICY_SCHEMA or not isinstance(policy.get("sources"), dict): + if policy.get("schema_version") != POLICY_SCHEMA or not isinstance(sources := policy.get("sources"), dict): raise ValueError("invalid manager policy") prefix = f"manager.external.native.{binding['binding_id']}." - return [channel for channel, source in policy["sources"].items() - if channel.startswith(prefix) and isinstance(source, dict) - and binding["operator_ref"] in source.get("sender_ids", [])] + return [str(ch) for ch, src in sources.items() if str(ch).startswith(prefix) and isinstance(src, dict) and binding["operator_ref"] in (src.get("sender_ids") or [])] - def ensure_delivery_scope(self, selected: dict[str, Any], *, if_absent: bool = True, execute: bool = True) -> None: - if selected["binding"].get("goal_scope") not in {"all_registered", "selected"} or self.controller is None: + def ensure_delivery_scope(self, selected: JsonMap, *, if_absent: bool = True, execute: bool = True) -> None: + if (binding := _as_dict(selected["binding"])).get("goal_scope") not in {"all_registered", "selected"} or self.controller is None: return # Match configuration's binding -> source-policy order. An observation # from before disconnect must never recreate its retired source grant. with exclusive_file_lock(self.path, operation="configure_bound_delivery_scope"): - if selected["binding"] not in self.read()["bindings"]: + if binding not in _dict_list(self.read()["bindings"]): raise ValueError("conversation binding is no longer authorized") self._ensure_delivery_scope(selected, if_absent=if_absent, execute=execute) - def _ensure_delivery_scope(self, selected: dict[str, Any], *, if_absent: bool = True, execute: bool = True) -> None: + def _ensure_delivery_scope(self, selected: JsonMap, *, if_absent: bool = True, execute: bool = True) -> None: """The caller retains the binding publication fence, including configure.""" - binding = selected["binding"] - if binding.get("goal_scope") not in {"all_registered", "selected"} or self.controller is None: + binding = _as_dict(selected["binding"]) + if (goal_scope := binding.get("goal_scope")) not in {"all_registered", "selected"} or self.controller is None: return from ..manager_context import configure_delivery_scope - configure_delivery_scope(self.controller.coordination_runtime_root, - channel=selected["channel_id"], local_delivery_scope=binding["goal_scope"], - sender_id=binding["operator_ref"], execute=execute, if_absent=if_absent) + configure_delivery_scope(self.controller.coordination_runtime_root, channel=_need(selected["channel_id"], str), + local_delivery_scope=_need(goal_scope, str), sender_id=_need(binding["operator_ref"], str), execute=execute, if_absent=if_absent) @staticmethod - def _session_source(saved: dict[str, Any]) -> dict[str, Any]: + def _session_source(saved: JsonMap) -> JsonDict: + base: JsonDict = {"source_ref": saved["source_ref"], "sender_ref": saved["operator_ref"], "private_human_message": True, "session_context": saved} if saved.get("audience") == "bound_group": - return {"source_ref": saved["source_ref"], "sender_ref": saved["operator_ref"], - "private_human_message": False, "group_ref": saved["group_ref"], - "topic_ref": saved["topic_ref"], "group_human_message": True, "session_context": saved} - return {"source_ref": saved["source_ref"], "sender_ref": saved["operator_ref"], - "private_human_message": True, "session_context": saved} + return {**base, "private_human_message": False, "group_ref": saved["group_ref"], "topic_ref": saved["topic_ref"], "group_human_message": True} + return base - def session_context(self, saved: dict[str, Any], *, session: dict[str, Any] | None = None) -> dict[str, Any]: - return self.resolve(binding_id=saved["binding_id"], session=session, **self._session_source(saved)) + def session_context(self, saved: JsonMap, *, session: JsonMap | None = None) -> JsonDict: + return self._resolve_source(_need(saved["binding_id"], str), self._session_source(saved), session=session) - def manager_runtime_owner(self, saved: dict[str, Any], channel_id: str) -> bool: + def manager_runtime_owner(self, saved: JsonMap, channel_id: str) -> bool: """Fresh Core audience proof; saved Session text is never authority.""" - selected = self.session_context(saved) - if selected["channel_id"] != channel_id: + if (selected := self.session_context(saved))["channel_id"] != channel_id: raise ValueError("bound manager audience changed") return selected["owner_manager_audience"] is True - def steward_scope(self, session: dict[str, Any]) -> list[str] | None: - saved = session.get("steward_context") - if not isinstance(saved, dict): + def steward_scope(self, session: JsonMap) -> list[str] | None: + if not isinstance(saved := session.get("steward_context"), dict): return None selected = self.session_context(saved) if session.get("goal_id") != "loopx-manager" or session.get("channel_id") != selected["channel_id"]: raise ValueError("bound steward audience changed") - return list(selected["context"]["goal_ids"]) + goal_ids = _as_dict(selected["context"])["goal_ids"] + return [_need(item, str) for item in goal_ids] if isinstance(goal_ids, list) else [] - def adopt_created_goal(self, *, binding_id: str, source: dict[str, Any], proposal: dict[str, Any], - goal: dict[str, Any]) -> None: - selected = self.resolve(binding_id=binding_id, **source) + def adopt_created_goal(self, *, binding_id: str, source: JsonMap, proposal: JsonMap, goal: JsonMap) -> None: + selected = self._resolve_source(binding_id, source) with exclusive_file_lock(self.path, operation="adopt_steward_created_goal"): current = self.read() result = self._core("collaboration.conversation.binding", { "current": current, "expected_revision": current["revision"], "operation": "adopt_created_goal", "binding_id": binding_id, "context": selected["context"], "proposal": proposal, - "goal": {"goal_id": goal["id"], "workspace_path": str(Path(goal["repo"]).resolve()), - "creation_operation_id": goal.get("creation_operation_id")}, + "goal": {"goal_id": goal["id"], "workspace_path": str(Path(_need(goal["repo"], str)).resolve()), "creation_operation_id": goal.get("creation_operation_id")}, }) - if result["changed"]: - _atomic_write_json(self.path, result["state"]) - if self.read() != result["state"]: - raise OSError("steward scope publication did not verify") + self._publish_state(result, error_message="steward scope publication did not verify") - def agent_observation(self, session_id: str) -> dict[str, Any]: + def agent_observation(self, session_id: str) -> _AgentObservation: """Read only the explicitly selected canonical attached Session/Goal. No host history, remote thread database or prior identity is discovered. """ - from ...attached_session import require_current_attached_session, _require_bound_host + from ... import attached_session as _attached, registry as _registry from ...control_plane.projects.registry_codec import load_registry - from ...registry import find_registry_goal + if self.controller is None or self.controller.registry_path is None: raise ValueError("the attached Session registry is unavailable") - session = require_current_attached_session(store=self.controller.store, - registry_path=self.controller.registry_path, session_id=session_id) - registry = load_registry(self.controller.registry_path) - goal = find_registry_goal(registry, str(session.get("goal_id") or "")) + store = self.controller.store + session = _attached.require_current_attached_session(store=store, registry_path=self.controller.registry_path, session_id=session_id) + goal = _registry.find_registry_goal(load_registry(self.controller.registry_path), str(session.get("goal_id") or "")) if goal is None or not goal.get("repo"): raise ValueError("the exact Agent workspace is unavailable") - _require_bound_host(goal=goal, agent_id=session["agent_id"], host_surface=session["host_surface"], - host_session_id=session["upstream_thread_id"]) - return {"session": {key: session.get(key) for key in ["session_id", "goal_id", "goal_instance_id", - "agent_id", "executor_endpoint_id", "session_mode", "status", "external_conversation_binding_id"]}, - "goal": {"goal_id": goal["id"], "workspace_path": str(Path(goal["repo"]).resolve()), - "registered_agents": (goal.get("coordination") or {}).get("registered_agents", [])}, - "host_binding_verified": True, "host_ref": _host_ref(session), - "host_audience_binding_ids": _host_audiences(self.controller.store, session)} - - def agent_candidates(self, binding_id: str) -> list[dict[str, Any]]: - row = next(item for item in self.read()["bindings"] if item["binding_id"] == binding_id) + _attached._require_bound_host(goal=goal, agent_id=session["agent_id"], host_surface=session["host_surface"], host_session_id=session["upstream_thread_id"]) + host = _host_observation_fields(store, session, goal) + keys = ("session_id", "goal_id", "goal_instance_id", "agent_id", "executor_endpoint_id", "session_mode", "status", "external_conversation_binding_id") + return {"session": {k: session.get(k) for k in keys}, "goal": _as_dict(host["goal"]), "host_binding_verified": True, + "host_ref": _need(host["host_ref"], str), "host_audience_binding_ids": [_need(i, str) for i in cast("list[object]", host["host_audience_binding_ids"])]} + + def agent_candidates(self, binding_id: str) -> list[JsonDict]: + row = _require_binding(self.read(), "binding_id", binding_id) if row["context_kind"] != "project" or row.get("audience") == "group" or self.controller is None: return [] - project = next((item for item in self.projects.available() if item["project_ref"] == row["project_ref"]), None) - if project is None: + if (project := next((item for item in self._require_projects().available() if item["project_ref"] == row["project_ref"]), None)) is None: return [] - candidates = [] + candidates: list[JsonDict] = [] for session in self.controller.store.list_sessions(): if session.get("session_mode") != "attached_host" or session.get("status") == "closed": continue try: - observed = self.agent_observation(session["session_id"]) - if (observed["session"]["status"] in {"ready", "busy"} - and observed["goal"]["workspace_path"] == project["workspace_path"] + observed = self.agent_observation(_need(session["session_id"], str)) + if (observed["session"]["status"] in {"ready", "busy"} and observed["goal"]["workspace_path"] == project["workspace_path"] and observed["session"].get("external_conversation_binding_id") in {None, binding_id} and all(owner == binding_id for owner in observed["host_audience_binding_ids"])): candidates.append(observed["session"]) @@ -623,48 +620,35 @@ def agent_candidates(self, binding_id: str) -> list[dict[str, Any]]: continue return candidates[:16] - def change_agent_target(self, *, binding_id: str, expected_revision: int, - session_id: str | None = None, target_ref: str | None = None) -> dict[str, Any]: - row = next((item for item in self.read()["bindings"] if item["binding_id"] == binding_id), None) - if row is None: - raise ValueError("conversation binding is no longer authorized") + def change_agent_target(self, *, binding_id: str, expected_revision: int, session_id: str | None = None, target_ref: str | None = None) -> JsonDict: + row = _require_binding(self.read(), "binding_id", binding_id) # Provider IO must not prevent disconnect from publishing revocation. # Recheck the exact binding under its fence before using this observation. - observation = self.observe(row["transport_ref"]) + observation = self.observe(_need(row["transport_ref"], str)) with exclusive_file_lock(self.path, operation="configure_conversation_agent_target"): - current = self.read() - if next((item for item in current["bindings"] if item["binding_id"] == binding_id), None) != row: - raise ValueError("conversation binding changed during identity verification") - params = {"current": current, "expected_revision": expected_revision, "binding_id": binding_id, - "observation": observation, "available_projects": self.projects.available()} + params: JsonDict = {"current": self._verify_unchanged("binding_id", binding_id, row), "expected_revision": expected_revision, + "binding_id": binding_id, "observation": observation, "available_projects": self._require_projects().available()} if session_id is not None: observed = self.agent_observation(session_id) - target = {key: observed["session"].get(key) for key in ["session_id", "goal_id", "goal_instance_id", - "agent_id", "executor_endpoint_id"]} - params.update(operation="grant_agent_target", target={"target_ref": uuid.uuid4().hex[:24], "host_ref": observed["host_ref"], **target}, - target_observation=observed) + target = {k: observed["session"].get(k) for k in ("session_id", "goal_id", "goal_instance_id", "agent_id", "executor_endpoint_id")} + params.update(operation="grant_agent_target", target_observation=observed, target={"target_ref": uuid.uuid4().hex[:24], "host_ref": observed["host_ref"], **target}) else: params.update(operation="revoke_agent_target", target_ref=target_ref) result = self._core("collaboration.conversation.binding", params) - if result["changed"]: - if session_id is not None: - # agent_observation already requires this exact local controller. - assert self.controller is not None - self.controller.store.update_session(session_id, external_conversation_binding_id=binding_id) - _atomic_write_json(self.path, result["state"]) - if self.read() != result["state"]: - raise OSError("Agent target grant publication did not verify") - state: dict[str, Any] = result["state"] - return state - - def resolve_agent_target(self, selected: dict[str, Any], target: dict[str, Any]) -> dict[str, Any]: - return self._core("collaboration.conversation.agent_target", {"binding": selected["binding"], - "context": selected["context"], "target": target, - "target_observation": self.agent_observation(target["session_id"])}) - + if result["changed"] and session_id is not None: + # agent_observation already requires this exact local controller. + assert self.controller is not None + self.controller.store.update_session(session_id, external_conversation_binding_id=binding_id) + return self._publish_state(result, error_message="Agent target grant publication did not verify") + + def resolve_agent_target(self, selected: JsonMap, target: JsonMap) -> JsonDict: + return self._core("collaboration.conversation.agent_target", { + "binding": selected["binding"], "context": selected["context"], "target": target, + "target_observation": self.agent_observation(_need(target["session_id"], str)), + }) @effect_runtime_request_scope() - def stop_recoveries(self, current: dict[str, Any]) -> list[dict[str, Any]]: + def stop_recoveries(self, current: JsonMap) -> list[JsonDict]: """Project revoked execution from canonical Turns, never recreate a grant. Session and external-target context retain the exact binding identity @@ -674,75 +658,56 @@ def stop_recoveries(self, current: dict[str, Any]) -> list[dict[str, Any]]: """ if self.controller is None: return [] - authorized = {row["binding_id"] for row in current["bindings"]} - store = self.controller.store - pending: dict[str, dict[str, Any]] = {} - for row in store.list_sessions(): - session = store.load_session(row["session_id"]) - if session is None: + authorized = {row["binding_id"] for row in _dict_list(current["bindings"])} + counts: dict[str, int] = {} + project_refs: dict[str, str] = {} + for session, turn in _iter_open_session_turns(self.controller.store): + context = _dict_or_empty(_dict_or_empty(turn.get("external_agent_target")).get("context")) or _dict_or_empty(session.get("project_context")) or _dict_or_empty(session.get("steward_context")) + if not isinstance(binding_id := context.get("binding_id"), str) or not binding_id or binding_id in authorized: continue - turns = store.queued_turns(session["session_id"]) - active = store.load_turn(session["session_id"], session["active_turn_id"]) if session.get("active_turn_id") else None - if active: - turns.append(active) - for turn in turns: - if turn.get("status") in TERMINAL_TURN_STATES: - continue - target = turn.get("external_agent_target") or {} - context = target.get("context") or session.get("project_context") or session.get("steward_context") or {} - binding_id = context.get("binding_id") - if not binding_id or binding_id in authorized: - continue - recovery = pending.setdefault(binding_id, {"binding_id": binding_id, - "project_ref": context.get("project_ref", ""), "pending_count": 0}) - recovery["pending_count"] += 1 - return [pending[key] for key in sorted(pending)] + project_refs.setdefault(binding_id, str(context.get("project_ref") or "")) + counts[binding_id] = counts.get(binding_id, 0) + 1 + return [{"binding_id": bid, "project_ref": project_refs[bid], "pending_count": counts[bid]} for bid in sorted(counts)] -def validate_external_agent_turn(*, store: Any, session: dict[str, Any], - goal: dict[str, Any] | None, turn: dict[str, Any]) -> None: +def validate_external_agent_turn(*, store: ChatSessionStore, session: JsonMap, goal: JsonMap | None, turn: JsonMap) -> None: """Called under the grant and canonical queue fences before host claim. The owner-local grant persists independently of a live provider connection. Admission and result delivery also verify the App/owner with the provider. """ - saved = turn.get("external_agent_target") - if saved is None: + if (saved := turn.get("external_agent_target")) is None: return from ...attached_session import _require_bound_host if not isinstance(saved, dict) or turn.get("origin") not in {"lark", "external"} or goal is None: raise ValueError("the attached external audience is unavailable") - context, target = saved["context"], saved["target"] - if not Path(goal["repo"]).is_dir(): + context, target = _as_dict(saved["context"]), saved["target"] + if not Path(_need(goal["repo"], str)).is_dir(): raise ValueError("the attached Agent workspace is unavailable") - state = _read_json(store.root / "conversation-bindings.json") - selected = next((row for row in state["bindings"] if row["binding_id"] == context["binding_id"]), None) - if selected is None: + if (selected := _find_binding(_read_json(store.root / "conversation-bindings.json"), "binding_id", context["binding_id"])) is None: raise ValueError("the Agent audience grant was revoked") - _require_bound_host(goal=goal, agent_id=session["agent_id"], host_surface=session["host_surface"], - host_session_id=session["upstream_thread_id"]) - ChatConversationBindings._core("collaboration.conversation.agent_target", {"binding": selected, - "context": context, "target": target, "target_observation": {"session": session, - "goal": {"goal_id": goal["id"], "workspace_path": str(Path(goal["repo"]).resolve()), - "registered_agents": (goal.get("coordination") or {}).get("registered_agents", [])}, - "host_binding_verified": True, "host_ref": _host_ref(session), - "host_audience_binding_ids": _host_audiences(store, session)}}) + _require_bound_host(goal=dict(goal), agent_id=_need(session["agent_id"], str), + host_surface=_need(session["host_surface"], str), host_session_id=_need(session["upstream_thread_id"], str)) + ChatConversationBindings._core("collaboration.conversation.agent_target", { + "binding": selected, "context": context, "target": target, + "target_observation": {"session": session, **_host_observation_fields(store, session, goal)}, + }) -def _host_ref(session: dict[str, Any]) -> str: +def _host_ref(session: JsonMap) -> str: return hashlib.sha256(f"{session['host_surface']}\0{session['upstream_thread_id']}".encode()).hexdigest()[:24] -def _host_audiences(store: Any, session: dict[str, Any]) -> list[str]: +def _host_audiences(store: ChatSessionStore, session: JsonMap) -> list[str]: """Content-free stamps in this canonical store, including closed Sessions.""" - owners = set() - for row in store.list_sessions(): - candidate = store.load_session(row["session_id"]) - if (candidate and candidate.get("host_surface") == session["host_surface"] - and candidate.get("upstream_thread_id") == session["upstream_thread_id"] - and candidate.get("external_conversation_binding_id")): - owners.add(candidate["external_conversation_binding_id"]) - return sorted(owners) + return sorted({ + candidate["external_conversation_binding_id"] + for row in store.list_sessions() + if (candidate := store.load_session(_need(row["session_id"], str))) + and candidate.get("host_surface") == session["host_surface"] + and candidate.get("upstream_thread_id") == session["upstream_thread_id"] + and isinstance(candidate.get("external_conversation_binding_id"), str) and candidate["external_conversation_binding_id"] + }) class OriginalConversationRequests: @@ -752,121 +717,109 @@ class OriginalConversationRequests: Session FIFO remain their respective owners. No shadow queue is created. """ - def __init__(self, controller): - self.controller = controller - self.store = controller.store - self.root = controller.coordination_runtime_root - self.bindings = controller.project_contexts.conversation_bindings - if self.bindings is None: + def __init__(self, controller: _BindingController) -> None: + self.controller, self.store, self.root = controller, controller.store, controller.coordination_runtime_root + if (bindings := controller.project_contexts.conversation_bindings) is None: raise ValueError("original conversation bindings are unavailable") + self.bindings = bindings _core = staticmethod(ChatConversationBindings._core) - def _policy(self): - from ...control_plane.collaboration.inbox import _read, _root - path = _root(self.root) / "policy.json" - return _read(path) if path.exists() else {"schema_version": "loopx_manager_context_policy_v1", "sources": {}} + def _path(self, *parts: str) -> Path: + return inbox._root(self.root).joinpath(*parts) + + def _entry_path(self, target: JsonMap, request_id: str) -> Path: + return self._path("entries", inbox._hash(dict(target)), f"{request_id}.json") + + def _lock_request(self, request_id: str, legacy_path: Path | None = None) -> AbstractContextManager[None]: + return cast("_RequestLockFn", inbox._request_lock)(self.root, request_id, None, legacy_path or Path(), force_exact=True) + + @staticmethod + def _write_verified(path: Path, payload: JsonMap, error_message: str, *, write: bool = True) -> None: + if write: + inbox._write(path, dict(payload)) + if inbox._read(path) != payload: + raise OSError(error_message) - def _observation(self, session_id): - session = self.store.load_session(session_id) - if session is None: + def _policy(self) -> JsonDict: + return cast("JsonDict", inbox._read(path)) if (path := self._path("policy.json")).exists() else {"schema_version": "loopx_manager_context_policy_v1", "sources": {}} + + def _observation(self, session_id: str) -> tuple[JsonDict, JsonDict]: + if (session := self.store.load_session(session_id)) is None: raise ValueError("original project Session is unavailable") - saved = session.get("project_context") or {} - current = self.bindings.read() - binding = next((row for row in current["bindings"] if row["binding_id"] == saved.get("binding_id")), None) - if binding is None: + if (binding := _find_binding(self.bindings.read(), "binding_id", _dict_or_empty(session.get("project_context")).get("binding_id"))) is None: raise ValueError("original conversation grant was revoked") - observed = dict(self.bindings.observe(binding["transport_ref"])) - return binding, observed - - def _recipient(self, session_id, expected_binding, observation, policy): - current = self.bindings.read() - if next((row for row in current["bindings"] if row["binding_id"] == expected_binding["binding_id"]), None) != expected_binding: - raise ValueError("original conversation binding changed during verification") - return self._core("collaboration.project.recipient", {"current": current, - "binding_id": expected_binding["binding_id"], "observation": observation, - "available_projects": self.bindings.projects.available(), - "session": self.store.load_session(session_id), "policy": policy, - "sessions": [self.store.load_session(row["session_id"]) for row in self.store.list_sessions()]}) + return binding, dict(self.bindings.observe(_need(binding["transport_ref"], str))) + + def _recipient(self, session_id: str, expected_binding: JsonMap, observation: JsonMap, policy: JsonMap) -> JsonDict: + current = self.bindings._verify_unchanged("binding_id", expected_binding["binding_id"], expected_binding, message="original conversation binding changed during verification") + return self._core("collaboration.project.recipient", { + "current": current, "binding_id": expected_binding["binding_id"], "observation": observation, + "available_projects": self.bindings._require_projects().available(), "session": self.store.load_session(session_id), "policy": policy, + "sessions": [self.store.load_session(_need(row["session_id"], str)) for row in self.store.list_sessions()], + }) @contextmanager - def _source(self, requester, parent_request_id, *, require_active=True): - from ...control_plane.collaboration.goal_instance_scope import collaboration_goal_scope - from ...control_plane.collaboration.inbox import _entry + def _source(self, requester: JsonMap, parent_request_id: str, *, require_active: bool = True) -> Iterator[tuple[JsonDict, CollaborationGoalScope]]: from ...control_plane.collaboration.peer_context_observation import require_parent_context_access - ref = requester.get("goal_ref") or {} - with collaboration_goal_scope(self.controller.registry_path, goal_id=ref.get("goal_id"), - agents=(requester.get("agent_id"),), caller_goal_ref=ref, require_active=require_active) as scope: - if not scope.exact or requester != {"goal_ref": scope.caller_goal_ref, "agent_id": requester.get("agent_id")}: + + if (registry_path := self.controller.registry_path) is None: + raise ValueError("the steward registry is unavailable") + ref, agent_id = requester.get("goal_ref"), requester.get("agent_id") + if not isinstance(ref, Mapping) or not isinstance(ref.get("goal_id"), str) or not isinstance(agent_id, str) or not all(isinstance(k, str) and isinstance(v, str) for k, v in ref.items()): + raise ValueError("an exact original requester is required") + with collaboration_goal_scope(registry_path, goal_id=ref["goal_id"], agents=(agent_id,), caller_goal_ref={str(k): str(v) for k, v in ref.items()}, require_active=require_active) as scope: + if not scope.exact or requester != {"goal_ref": scope.caller_goal_ref, "agent_id": agent_id}: raise ValueError("an exact original requester is required") - parent = _entry(self.root, scope.goal_id, requester["agent_id"], parent_request_id, scope=scope) + parent = inbox._entry(self.root, scope.goal_id, agent_id, parent_request_id, scope=scope) if "request_control" in parent and self._view(parent)["control_state"] in {"cancelled", "cancellation_pending"}: raise ValueError("the original source request was cancelled") - access = require_parent_context_access(self.root, self.controller.registry_path, parent, - requester["agent_id"], scope=scope) - yield {"requester": requester, "parent": parent, "parent_request_id": parent_request_id, - "original_source_id": access["source_id"], "parent_access_verified": True}, scope + access = require_parent_context_access(self.root, registry_path, parent, agent_id, scope=scope) + yield {"requester": requester, "parent": parent, "parent_request_id": parent_request_id, "original_source_id": access["source_id"], "parent_access_verified": True}, scope @contextmanager - def _fenced(self, session_id, requester, parent_request_id): - from ...control_plane.collaboration.source_policy_fence import source_policy_fence - from ...control_plane.collaboration.inbox import _root + def _fenced(self, session_id: str, requester: JsonMap, parent_request_id: str) -> Iterator[tuple[JsonDict, CollaborationGoalScope]]: binding, observed = self._observation(session_id) # Every writer uses source -> Goal -> binding -> request -> Session. # Provider IO precedes these fences; canonical identity is reread inside. - with source_policy_fence(_root(self.root) / "policy.json"), self._source(requester, parent_request_id) as (source, scope): - with exclusive_file_lock(self.bindings.path, operation="original_conversation_peer"): - policy = self._policy() - selected = self._recipient(session_id, binding, observed, policy) - yield {**source, "policy": policy, "recipient": selected["recipient"], - "current_recipient": selected["recipient"], "policy_sha256": selected["policy_sha256"]}, scope - - def configure(self, session_id, body): - from ...control_plane.collaboration.inbox import _root, _write, _read - from ...control_plane.collaboration.source_policy_fence import source_policy_fence - enable = body.get("grant") - if not isinstance(enable, bool): + with source_policy_fence(self._path("policy.json")), self._source(requester, parent_request_id) as (source, scope), exclusive_file_lock(self.bindings.path, operation="original_conversation_peer"): + policy = self._policy() + selected = self._recipient(session_id, binding, observed, policy) + yield {**source, "policy": policy, "recipient": selected["recipient"], "current_recipient": selected["recipient"], "policy_sha256": selected["policy_sha256"]}, scope + + def configure(self, session_id: str, body: JsonMap) -> JsonDict: + if not isinstance(enable := body.get("grant"), bool): raise ValueError("grant must be an explicit boolean") + execute = body.get("execute") is True - def apply(facts): - plan = self._core("collaboration.source.original_conversation_grant", { - **facts, "grant": enable, "expected_policy_sha256": body.get("expected_policy_sha256")}) - path = _root(self.root) / "policy.json" - if body.get("execute") is True and plan["would_change"]: - _write(path, plan["policy"]) - if _read(path) != plan["policy"]: - raise OSError("original conversation source grant did not verify") - return {"ok": True, "executed": body.get("execute") is True, - **{key: value for key, value in plan.items() if key != "policy"}} + def apply(facts: JsonMap) -> JsonDict: + plan = self._core("collaboration.source.original_conversation_grant", {**facts, "grant": enable, "expected_policy_sha256": body.get("expected_policy_sha256")}) + if execute and plan["would_change"]: + self._write_verified(self._path("policy.json"), _as_dict(plan["policy"]), "original conversation source grant did not verify") + return {"ok": True, "executed": execute, **{k: v for k, v in plan.items() if k != "policy"}} + requester, parent_request_id = _as_dict(body["requester"]), _need(body["parent_request_id"], str) if enable: - with self._fenced(session_id, body["requester"], body["parent_request_id"]) as (facts, _): + with self._fenced(session_id, requester, parent_request_id) as (facts, _): return apply(facts) # An owner may revoke a retained grant even while its provider or source # is unavailable. - with source_policy_fence(_root(self.root) / "policy.json"): - recipient = body["recipient"] - if recipient.get("session_id") != session_id: + with source_policy_fence(self._path("policy.json")): + if (recipient := _as_dict(body["recipient"])).get("session_id") != session_id: raise ValueError("original conversation revocation scope mismatch") - return apply({"policy": self._policy(), "recipient": recipient, - "requester": body["requester"], "parent_request_id": body["parent_request_id"], - "original_source_id": body["source_id"]}) - - def management_view(self, session_id, *, search="", cursor=None): - from ...control_plane.collaboration.inbox import _root, _read - from ...control_plane.collaboration.source_policy_fence import source_policy_fence - with source_policy_fence(_root(self.root) / "policy.json"): + return apply({"policy": self._policy(), "recipient": recipient, "requester": requester, "parent_request_id": parent_request_id, "original_source_id": body["source_id"]}) + + def management_view(self, session_id: str, *, search: str = "", cursor: str | None = None) -> JsonDict: + with source_policy_fence(self._path("policy.json")): policy = self._policy() view = self._core("collaboration.source.original_conversation_grants", {"policy": policy, "session_id": session_id}) - for grant in view["grants"]: - requester = grant["requester"] - address = {"goal_ref": requester["goal_ref"], "agent_id": requester["agent_id"]} + for grant in _dict_list(view["grants"]): + requester = _as_dict(grant["requester"]) try: - from ...control_plane.collaboration.inbox import _hash - parent = _read(_root(self.root) / "entries" / _hash(address) / (grant["parent_request_id"] + ".json")) + parent = inbox._read(self._entry_path({"goal_ref": requester["goal_ref"], "agent_id": requester["agent_id"]}, _need(grant["parent_request_id"], str))) if parent.get("goal_ref") != requester["goal_ref"] or parent.get("agent_id") != requester["agent_id"]: raise ValueError("source grant parent identity changed") - grant["source_purpose"] = (parent.get("brief") or {}).get("purpose", parent.get("message", "")) + grant["source_purpose"] = _dict_or_empty(parent.get("brief")).get("purpose", parent.get("message", "")) except (ValueError, OSError): grant["source_purpose"] = None try: @@ -878,278 +831,264 @@ def management_view(self, session_id, *, search="", cursor=None): candidates, unreadable, overflow = [], 0, False # Content is projected only to the trusted local owner. A later grant # still rechecks the exact registration, parent lineage and source. - for path in sorted((_root(self.root) / "entries").glob("*/*.json")): + for path in sorted(self._path("entries").glob("*/*.json")): try: - row = _read(path) + row = inbox._read(path) if row.get("schema_version") != "loopx_manager_context_entry_v2" or not row.get("goal_ref"): continue - purpose = (row.get("brief") or {}).get("purpose", row.get("message", "")) - if search.strip().casefold() not in " ".join((row["goal_ref"]["goal_id"], row["agent_id"], purpose)).casefold(): + purpose = str(_dict_or_empty(row.get("brief")).get("purpose", row.get("message", ""))) + if search.strip().casefold() not in " ".join((str(row["goal_ref"]["goal_id"]), str(row["agent_id"]), purpose)).casefold(): continue - candidates.append({"requester": {"goal_ref": row["goal_ref"], "agent_id": row["agent_id"]}, - "parent_request_id": row["request_id"], "purpose": (row.get("brief") or {}).get("purpose", row.get("message", ""))}) + candidates.append({"requester": {"goal_ref": row["goal_ref"], "agent_id": row["agent_id"]}, "parent_request_id": row["request_id"], "purpose": purpose}) if len(candidates) == 128: overflow = True break - except (ValueError, OSError, KeyError): + except (ValueError, OSError, KeyError, TypeError): unreadable += 1 view.update(candidates=candidates, coverage={"unreadable": unreadable, "overflow": overflow}) - page = self._inbox(view["recipient"], mark_read=False, cursor=cursor) - view["requests"], view["requests_has_more"] = page["items"], page["has_more"] - view["requests_next_cursor"] = page["next_cursor"] - from ...control_plane.collaboration.inbox import result_paths, inspect_result_identity, reply_status - for item in view["requests"]: - results = [] - statuses = {row.get("result_key", row["phase"]): row["status"] for row in reply_status(self.root, item)} - queue = self._core("collaboration.project.continuation", {"entry": self._entry(item["request_id"], item["recipient"])}) - turn = self.store.turn_for_client(session_id, queue["client_turn_id"]) + page = self._inbox(_as_dict(view["recipient"]), mark_read=False, cursor=cursor) + requests = _dict_list(page["items"]) + view["requests"], view["requests_has_more"], view["requests_next_cursor"] = requests, page["has_more"], page["next_cursor"] + for item in requests: + request_id = _need(item["request_id"], str) + entry = self._entry(request_id, _as_dict(item["recipient"])) + statuses = {str(r.get("result_key", r["phase"])): r["status"] for r in cast("Callable[[Path, JsonMap], list[JsonDict]]", inbox.reply_status)(self.root, item)} + turn = self.store.turn_for_client(session_id, _need(self._core("collaboration.project.continuation", {"entry": entry})["client_turn_id"], str)) item["queue_status"] = turn.get("status") if turn else None - for path in result_paths(_root(self.root) / "replies" / item["request_id"]): - reply = _read(path) - if inspect_result_identity(reply, self._entry(item["request_id"], item["recipient"]), path): - results.append({"result_key": path.stem, "text": reply["text"], "decision": reply["decision"], - "request_revision": reply.get("request_revision", 1), "delivery_status": statuses.get(path.stem, "queued")}) - item["returns"] = results + item["returns"] = [ + {"result_key": path.stem, "text": reply["text"], "decision": reply["decision"], + "request_revision": reply.get("request_revision", 1), "delivery_status": statuses.get(path.stem, "queued")} + for path in cast("Callable[[Path], list[Path]]", inbox.result_paths)(self._path("replies", request_id)) + if cast("Callable[[JsonMap, JsonMap, Path], object]", inbox.inspect_result_identity)(reply := inbox._read(path), entry, path) + ] return {"ok": True, **view} - def discover(self, requester, parent_request_id): - from ...control_plane.collaboration.inbox import _root - from ...control_plane.collaboration.source_policy_fence import source_policy_fence - with source_policy_fence(_root(self.root) / "policy.json"), self._source(requester, parent_request_id) as (source, _): - grants = self._core("collaboration.source.original_conversation_grants", {"policy": self._policy()})["grants"] + def discover(self, requester: JsonMap, parent_request_id: str) -> JsonDict: + with source_policy_fence(self._path("policy.json")), self._source(requester, parent_request_id): + grants = _dict_list(self._core("collaboration.source.original_conversation_grants", {"policy": self._policy()})["grants"]) recipients, unavailable = [], 0 for grant in grants: if grant["requester"] != requester or grant["parent_request_id"] != parent_request_id: continue try: - with self._fenced(grant["recipient"]["session_id"], requester, parent_request_id) as (facts, _): + grant_recipient = _as_dict(grant["recipient"]) + with self._fenced(_need(grant_recipient["session_id"], str), requester, parent_request_id) as (facts, _): self._core("collaboration.source.original_conversation_access", facts) - if facts["recipient"] != grant["recipient"]: + if facts["recipient"] != grant_recipient: raise ValueError("original conversation recipient changed") - session = self.store.load_session(facts["recipient"]["session_id"]) - recipients.append({"recipient": facts["recipient"], "title": session.get("title") or "Project conversation"}) + facts_recipient = _as_dict(facts["recipient"]) + session = self.store.load_session(_need(facts_recipient["session_id"], str)) + recipients.append({"recipient": facts_recipient, "title": (session or {}).get("title") or "Project conversation"}) except (ValueError, KeyError, OSError): unavailable += 1 return {"ok": True, "recipients": recipients, "unavailable": unavailable} - def publish(self, body): - from ...control_plane.collaboration.inbox import _hash, _now, _read, _request_lock, _root, _write + def publish(self, body: JsonMap) -> JsonDict: from ...control_plane.collaboration.peers import _operation_address - recipient = body["recipient"] - session_id = recipient["session_id"] - with self._fenced(session_id, body["requester"], body["parent_request_id"]) as (facts, scope): + + recipient, requester = _as_dict(body["recipient"]), _as_dict(body["requester"]) + session_id = _need(recipient["session_id"], str) + with self._fenced(session_id, requester, _need(body["parent_request_id"], str)) as (facts, scope): if recipient != facts["recipient"]: raise ValueError("original conversation recipient identity changed") - source_id, op_path = _operation_address(self.root, scope, body["requester"]["agent_id"], body["operation_id"]) + source_id, op_path = _operation_address(self.root, scope, _need(requester["agent_id"], str), _need(body["operation_id"], str)) # Native planning owns identity and replay. The old operation is # consulted before any queue input or return route can be written. - prior = _read(op_path) if op_path.exists() else None - preview = self._core("collaboration.project.request", {**facts, "source_id": source_id, - "request": {"recipient": recipient, "brief": body["brief"]}, "observed_at": _now()}) - request_id = preview["entry"]["request_id"] - path = _root(self.root) / "entries" / _hash({"recipient": recipient}) / (request_id + ".json") - with _request_lock(self.root, request_id, None, op_path, force_exact=True): - prior = _read(op_path) if op_path.exists() else prior - plan = self._core("collaboration.project.request", {**facts, "source_id": source_id, - "request": {"recipient": recipient, "brief": body["brief"]}, "observed_at": _now(), - "operation": prior, "entry": _read(path) if path.exists() else None}) - row = plan["entry"] - route = plan["return_route"] - route_path = _root(self.root) / "roundtrips" / (request_id + ".json") - if route_path.exists() and _read(route_path) != route: + prior = inbox._read(op_path) if op_path.exists() else None + req_payload: JsonDict = {**facts, "source_id": source_id, "request": {"recipient": recipient, "brief": body["brief"]}} + request_id = _need(_as_dict(self._core("collaboration.project.request", {**req_payload, "observed_at": inbox._now()})["entry"])["request_id"], str) + path = self._entry_path({"recipient": recipient}, request_id) + with self._lock_request(request_id, op_path): + prior = inbox._read(op_path) if op_path.exists() else prior + plan = self._core("collaboration.project.request", {**req_payload, "observed_at": inbox._now(), "operation": prior, "entry": inbox._read(path) if path.exists() else None}) + row, route, route_path = _as_dict(plan["entry"]), _as_dict(plan["return_route"]), self._path("roundtrips", f"{request_id}.json") + if route_path.exists() and inbox._read(route_path) != route: raise ValueError("original requester return route identity conflict") if prior is None: - _write(op_path, plan["operation"]) - if not route_path.exists(): - _write(route_path, route) - if _read(route_path) != route: - raise OSError("original requester return route did not verify") - if not plan["replayed"]: - _write(path, row) - if _read(path) != row: - raise OSError("original conversation request did not verify") - view = self._view(row) - if view["control_state"] == "cancelled": + inbox._write(op_path, _as_dict(plan["operation"])) + self._write_verified(route_path, route, "original requester return route did not verify", write=not route_path.exists()) + self._write_verified(path, row, "original conversation request did not verify", write=not plan["replayed"]) + if self._view(row)["control_state"] == "cancelled": raise ValueError("original conversation request is cancelled") turn, created = self._enqueue(row) - recovered = self._resume(session_id, turn, created) - return {"ok": True, "request_id": request_id, "recipient": recipient, "turn_id": turn["turn_id"], - "status": turn["status"], "replayed": not created, "goal_created": False, "recovered_result": recovered} + return {"ok": True, "request_id": request_id, "recipient": recipient, "turn_id": turn["turn_id"], "status": turn["status"], + "replayed": not created, "goal_created": False, "recovered_result": self._resume(session_id, turn, created)} - def _enqueue(self, row): + def _enqueue(self, row: JsonMap) -> tuple[JsonDict, bool]: queue = self._core("collaboration.project.continuation", {"entry": row}) - turn, created = self.store.create_queued_turn(row["recipient"]["session_id"], - client_turn_id=queue["client_turn_id"], message=queue["message"], origin="peer", - collaboration_request=queue["collaboration_request"]) - return turn, created + return self.store.create_queued_turn(_need(_as_dict(row["recipient"])["session_id"], str), client_turn_id=_need(queue["client_turn_id"], str), + message=_need(queue["message"], str), origin="peer", collaboration_request=_as_dict(queue["collaboration_request"])) - def _stop_turns(self, session_id, client_ids): - stopped = None + def _stop_turns(self, session_id: str, client_ids: Sequence[str]) -> JsonDict | None: + stopped: JsonDict | None = None for client_id in client_ids: - turn = self.store.turn_for_client(session_id, client_id) - if turn and turn["status"] not in TERMINAL_TURN_STATES: - stopped = self.controller.interrupt_turn(session_id=session_id, turn_id=turn["turn_id"]) + if (turn := self.store.turn_for_client(session_id, client_id)) and turn["status"] not in TERMINAL_TURN_STATES: + stopped = self.controller.interrupt_turn(session_id=session_id, turn_id=_need(turn["turn_id"], str)) return stopped - def _resume(self, session_id, turn, created): - recovered = None - if not created and turn.get("collaboration_answer") is not None: - recovered = self._complete_answer(session_id, turn, turn["collaboration_answer"])["collaboration_result"] + def _resume(self, session_id: str, turn: JsonMap, created: bool) -> JsonDict | None: + recovered = _as_dict(self._complete_answer(session_id, turn, _as_dict(answer))["collaboration_result"]) if not created and (answer := turn.get("collaboration_answer")) is not None else None if turn["status"] not in TERMINAL_TURN_STATES: - context = self.controller.project_contexts.session_context(self.store.load_session(session_id)) - self.controller.resume_session_queue(session_id=session_id, work_dir=Path(context["project"]), objective=str(context.get("objective") or context["title"])) + if (session := self.store.load_session(session_id)) is None: + raise ValueError("original project Session is unavailable") + context = self.controller.project_contexts.session_context(session) + project = context["project"] + self.controller.resume_session_queue(session_id=session_id, work_dir=project if isinstance(project, Path) else Path(_need(project, str)), objective=str(context.get("objective") or context["title"])) return recovered - def _entry(self, request_id, recipient): - from ...control_plane.collaboration.inbox import _entry - return _entry(self.root, None, None, request_id, recipient=recipient) - - def _view(self, row): - return self._core("collaboration.request.control", {"entry": row, "input": {"operation": "inspect"}, "source": {}})["view"] - - def _inbox(self, recipient, *, mark_read, request_id=None, cursor=None): - from ...control_plane.collaboration.inbox import _now - result = self._core("collaboration.inbox.read", {"runtime_root": str(self.root.resolve()), - "scope": {"profile_id": "original_conversation_v0", "recipient": recipient}, - "operation": "receiver_read" if mark_read else "pending", "request_id": request_id, - "include_settled": not mark_read, - "cursor": cursor, - "observed_at": _now(), "read_command": "original project Turn supplied context"}, large_local_snapshot=True) - return result["result"] - - def control(self, body): - from ...control_plane.collaboration.goal_instance_scope import collaboration_goal_scope - from ...control_plane.collaboration.source_policy_fence import source_policy_fence - from ...control_plane.collaboration.inbox import _now, _root, _request_lock, _hash, _write, _read - requester, recipient = body["requester"], body["recipient"] + def _entry(self, request_id: str, recipient: JsonMap) -> JsonDict: + return inbox._entry(self.root, None, None, request_id, recipient=dict(recipient)) + + def _view(self, row: JsonMap) -> JsonDict: + return _as_dict(self._core("collaboration.request.control", {"entry": row, "input": {"operation": "inspect"}, "source": {}})["view"]) + + def _inbox(self, recipient: JsonMap, *, mark_read: bool, request_id: str | None = None, cursor: str | None = None) -> JsonDict: + return _as_dict(self._core("collaboration.inbox.read", { + "runtime_root": str(self.root.resolve()), "scope": {"profile_id": "original_conversation_v0", "recipient": recipient}, + "operation": "receiver_read" if mark_read else "pending", "request_id": request_id, "include_settled": not mark_read, + "cursor": cursor, "observed_at": inbox._now(), "read_command": "original project Turn supplied context", + }, large_local_snapshot=True)["result"]) + + def control(self, body: JsonMap) -> JsonDict: + requester, recipient = _as_dict(body["requester"]), _as_dict(body["recipient"]) operation, parameters = body["control"], body["parameters"] if operation not in {"inspect", "revise", "cancel"} or not isinstance(parameters, dict): raise ValueError("unsupported original conversation source control") - row = self._entry(body["request_id"], recipient) + row = self._entry(_need(body["request_id"], str), recipient) if row["requester"] != requester: raise ValueError("original requester identity changed") - if operation == "revise": - fence = self._fenced(recipient["session_id"], requester, row["parent_request_id"]) - else: - # Revocation or a closed recipient must not remove the original - # source's ability to inspect/cancel its retained request. - @contextmanager - def retained_source(): - with source_policy_fence(_root(self.root) / "policy.json"), collaboration_goal_scope( - self.controller.registry_path, goal_id=requester["goal_ref"]["goal_id"], - agents=(requester["agent_id"],), caller_goal_ref=requester["goal_ref"]) as scope: - if not scope.exact: - raise ValueError("original requester exact instance is unavailable") - yield {}, scope - fence = retained_source() - with fence as (facts, _), _request_lock(self.root, row["request_id"], None, Path(), force_exact=True): - current = self._entry(row["request_id"], recipient) - route = _read(_root(self.root) / "roundtrips" / (row["request_id"] + ".json")) - self._core("collaboration.project.return_route", {"entry": current, "route": route}) + request_id, session_id = _need(row["request_id"], str), _need(recipient["session_id"], str) + + @contextmanager + def retained_source() -> Iterator[tuple[JsonDict, CollaborationGoalScope]]: + if (registry_path := self.controller.registry_path) is None: + raise ValueError("the steward registry is unavailable") + goal_ref = _as_dict(requester["goal_ref"]) + with source_policy_fence(self._path("policy.json")), collaboration_goal_scope( + registry_path, goal_id=_need(goal_ref["goal_id"], str), agents=(_need(requester["agent_id"], str),), caller_goal_ref={k: _need(v, str) for k, v in goal_ref.items()}, + ) as scope: + if not scope.exact: + raise ValueError("original requester exact instance is unavailable") + yield {}, scope + + # Revocation or a closed recipient must not remove the original + # source's ability to inspect/cancel its retained request. + fence = self._fenced(session_id, requester, _need(row["parent_request_id"], str)) if operation == "revise" else retained_source() + with fence as (facts, _), self._lock_request(request_id): + current = self._entry(request_id, recipient) + self._core("collaboration.project.return_route", {"entry": current, "route": inbox._read(self._path("roundtrips", f"{request_id}.json"))}) if operation == "revise": if recipient != facts["recipient"]: raise ValueError("original conversation recipient changed") self._core("collaboration.source.original_conversation_access", facts) - plan = self._core("collaboration.request.control", {"entry": current, - "input": {**parameters, "operation": operation}, - "source": {"actor": "source", "actor_agent_id": requester["agent_id"], - "actor_requester": requester, - "recipient_active": operation == "revise", "observed_at": _now()}}) + plan = self._core("collaboration.request.control", { + "entry": current, "input": {**_as_dict(parameters), "operation": operation}, + "source": {"actor": "source", "actor_agent_id": requester["agent_id"], "actor_requester": requester, "recipient_active": operation == "revise", "observed_at": inbox._now()}, + }) if plan["changed"]: - _write(_root(self.root) / "entries" / _hash({"recipient": recipient}) / (row["request_id"] + ".json"), plan["entry"]) + inbox._write(self._entry_path({"recipient": recipient}, request_id), _as_dict(plan["entry"])) if operation == "revise": - turn, created = self._enqueue(plan["entry"]) - stop = None + turn, created = self._enqueue(_as_dict(plan["entry"])) + stop: JsonDict | None = None if operation == "revise": - self._resume(recipient["session_id"], turn, created) + self._resume(session_id, turn, created) elif operation == "cancel": queue = self._core("collaboration.project.continuation", {"entry": plan["entry"]}) - stop = self._stop_turns(recipient["session_id"], [*queue["superseded_client_turn_ids"], queue["client_turn_id"]]) + superseded = queue["superseded_client_turn_ids"] + client_ids = [*(_need(cid, str) for cid in superseded)] if isinstance(superseded, list) else [] + stop = self._stop_turns(session_id, [*client_ids, _need(queue["client_turn_id"], str)]) return {"ok": True, "request_control": plan["view"], "replayed": plan.get("replayed", False), "turn_stop": stop} - def validate_turn(self, session_id, turn): + def validate_turn(self, session_id: str, turn: JsonMap) -> None: if turn.get("origin") != "peer" and "collaboration_request" not in turn: return - reference = turn.get("collaboration_request") - if turn.get("origin") != "peer" or not isinstance(reference, dict): + if turn.get("origin") != "peer" or not isinstance(reference := turn.get("collaboration_request"), dict): raise ValueError("peer Turn provenance is unavailable") - row = self._entry(reference["request_id"], reference["recipient"]) - if row["recipient"]["session_id"] != session_id: + row = self._entry(_need(reference["request_id"], str), _as_dict(reference["recipient"])) + if _as_dict(row["recipient"])["session_id"] != session_id: raise ValueError("peer Turn does not belong to this original Session") self._core("collaboration.project.continuation", {"entry": row, "turn": turn}) - with self._fenced(session_id, row["requester"], row["parent_request_id"]) as (facts, _): + with self._fenced(session_id, _as_dict(row["requester"]), _need(row["parent_request_id"], str)) as (facts, _): self._validate_current(row, facts) if self._view(row)["control_state"] == "cancelled": raise ValueError("original conversation request was cancelled") - def _validate_current(self, row, facts): - from ...control_plane.collaboration.inbox import _read, _root - self._core("collaboration.project.request", {**facts, "source_id": row["source_id"], - "request": {"recipient": row["recipient"], "brief": row["brief"]}, "entry": row, - "operation": None, "observed_at": row["delivered_at"], - "route": _read(_root(self.root) / "roundtrips" / (row["request_id"] + ".json"))}) + def _validate_current(self, row: JsonMap, facts: JsonMap, *, check_access: bool = False) -> None: + self._core("collaboration.project.request", { + **facts, "source_id": row["source_id"], "request": {"recipient": row["recipient"], "brief": row["brief"]}, + "entry": row, "operation": None, "observed_at": row["delivered_at"], "route": inbox._read(self._path("roundtrips", f"{_need(row['request_id'], str)}.json")), + }) + if check_access: + self._core("collaboration.source.original_conversation_access", facts) - def _complete_answer(self, session_id, turn, answer): - from ...control_plane.collaboration.inbox import _now, _request_lock, _root, _hash, _write, record_result + def _complete_answer(self, session_id: str, turn: JsonMap, answer: JsonMap) -> JsonDict: from ...chat import CHAT_AGENT_RESPONSE_SCHEMA_VERSION - if set(answer) != set(self._core("collaboration.project.response_schema", {})["required"]): + + if not isinstance(required := self._core("collaboration.project.response_schema", {})["required"], list) or set(answer) != set(required): raise ValueError("original conversation answer has unsupported fields") - ref = turn["collaboration_request"] - row = self._entry(ref["request_id"], ref["recipient"]) - with self._fenced(session_id, row["requester"], row["parent_request_id"]) as (facts, _): - self._validate_current(row, facts) - self._core("collaboration.source.original_conversation_access", facts) + ref = _as_dict(turn["collaboration_request"]) + request_id = _need(ref["request_id"], str) + row = self._entry(request_id, _as_dict(ref["recipient"])) + with self._fenced(session_id, _as_dict(row["requester"]), _need(row["parent_request_id"], str)) as (facts, _): + self._validate_current(row, facts, check_access=True) if facts["recipient"] != row["recipient"]: raise ValueError("original conversation recipient changed") - with _request_lock(self.root, row["request_id"], None, Path(), force_exact=True): - row = self._entry(row["request_id"], row["recipient"]) + with self._lock_request(request_id): + row = self._entry(request_id, _as_dict(row["recipient"])) self._core("collaboration.project.continuation", {"entry": row, "turn": turn}) view = self._view(row) - if (view["control_state"] == "cancelled" or answer["request_revision"] != view["revision"] - or answer["request_digest"] != view["revision_digest"]): + if view["control_state"] == "cancelled" or answer["request_revision"] != view["revision"] or answer["request_digest"] != view["revision_digest"]: raise ValueError("original conversation answer belongs to a stale or cancelled request") - plan = self._core("collaboration.request.assess", {"entry": row, - "actor_recipient": row["recipient"], "recipient_active": True, "observed_at": _now(), - "revision": answer["request_revision"], "revision_digest": answer["request_digest"], - "decision": answer["decision"], "reason": answer["reason"]}) + plan = self._core("collaboration.request.assess", { + "entry": row, "actor_recipient": row["recipient"], "recipient_active": True, "observed_at": inbox._now(), + "revision": answer["request_revision"], "revision_digest": answer["request_digest"], "decision": answer["decision"], "reason": answer["reason"], + }) if plan["changed"]: - _write(_root(self.root) / "entries" / _hash({"recipient": row["recipient"]}) / (row["request_id"] + ".json"), plan["entry"]) - receipt = record_result(self.root, row, "conclusion", answer["text"], request_revision=answer["request_revision"]) - return {"schema_version": CHAT_AGENT_RESPONSE_SCHEMA_VERSION, "message": answer["text"], - "proposals": [], "protected_action": None, "goal_draft": None, "context_handoff": None, - "gate": None, "unfinished_work": None, "collaboration_result": receipt} - - def execute(self, session_id, turn_id, native_session, event_sink): - import json - turn = self.store.load_turn(session_id, turn_id) + inbox._write(self._entry_path({"recipient": row["recipient"]}, request_id), _as_dict(plan["entry"])) + receipt = cast("_RecordResultFn", inbox.record_result)( + self.root, row, "conclusion", _need(answer["text"], str), + request_revision=raw if isinstance(raw := answer["request_revision"], int) and not isinstance(raw, bool) else None, + ) + return {"schema_version": CHAT_AGENT_RESPONSE_SCHEMA_VERSION, "message": answer["text"], "proposals": [], + "protected_action": None, "goal_draft": None, "context_handoff": None, "gate": None, "unfinished_work": None, "collaboration_result": receipt} + + def execute(self, session_id: str, turn_id: str, native_session: _NativePeerSession, event_sink: _EventSink) -> JsonDict: + if (turn := self.store.load_turn(session_id, turn_id)) is None: + raise KeyError("chat turn was not found") self.validate_turn(session_id, turn) - ref = turn["collaboration_request"] - row = self._entry(ref["request_id"], ref["recipient"]) - answer = turn.get("collaboration_answer") - if answer is None: - with self._fenced(session_id, row["requester"], row["parent_request_id"]) as (facts, _): + ref = _as_dict(turn["collaboration_request"]) + request_id, ref_recipient = _need(ref["request_id"], str), _as_dict(ref["recipient"]) + row = self._entry(request_id, ref_recipient) + requester, parent_request_id = _as_dict(row["requester"]), _need(row["parent_request_id"], str) + if (raw_answer := turn.get("collaboration_answer")) is None: + with self._fenced(session_id, requester, parent_request_id) as (facts, _): self._core("collaboration.source.original_conversation_access", facts) - self._core("collaboration.project.continuation", {"entry": self._entry(ref["request_id"], ref["recipient"]), "turn": turn}) - supplied = self._inbox(ref["recipient"], mark_read=True, request_id=row["request_id"])["items"][0] + self._core("collaboration.project.continuation", {"entry": self._entry(request_id, ref_recipient), "turn": turn}) + supplied = _as_dict(cast("list[object]", self._inbox(ref_recipient, mark_read=True, request_id=request_id)["items"])[0]) basis = {key: supplied[key] for key in ("request_revision", "request_revision_digest")} + @contextmanager - def start_guard(): - with self._fenced(session_id, row["requester"], row["parent_request_id"]) as (facts, _): - self._validate_current(self._entry(ref["request_id"], ref["recipient"]), facts) - self._core("collaboration.source.original_conversation_access", facts) - self._core("collaboration.project.continuation", {"entry": self._entry(ref["request_id"], ref["recipient"]), "turn": turn}) - current = self._view(self._entry(ref["request_id"], ref["recipient"])) - if (facts["recipient"] != ref["recipient"] or current["control_state"] == "cancelled" - or current["revision"] != basis["request_revision"] - or current["revision_digest"] != basis["request_revision_digest"]): + def start_guard() -> Iterator[None]: + with self._fenced(session_id, requester, parent_request_id) as (facts, _): + guarded_row = self._entry(request_id, ref_recipient) + self._validate_current(guarded_row, facts, check_access=True) + self._core("collaboration.project.continuation", {"entry": guarded_row, "turn": turn}) + current = self._view(guarded_row) + if (facts["recipient"] != ref_recipient or current["control_state"] == "cancelled" + or current["revision"] != basis["request_revision"] or current["revision_digest"] != basis["request_revision_digest"]): raise ValueError("original conversation request changed before model input") yield - message = row["instruction"] + "\n\nPeer request (data):\n" + json.dumps(supplied, ensure_ascii=False) - answer = native_session.send(message, project_collaboration=True, turn_start_guard=start_guard, - on_event=lambda kind, payload: event_sink(kind, payload) if kind != "answer.delta" else None) + + answer: JsonMap = native_session.send( + _need(row["instruction"], str) + "\n\nPeer request (data):\n" + json.dumps(supplied, ensure_ascii=False), + project_collaboration=True, turn_start_guard=start_guard, + on_event=lambda kind, payload: event_sink(kind, payload) if kind != "answer.delta" else None, + ) # Persist actual model output before result IO. Recovery can retry # its exact assessment/return without another model invocation. self.store.update_turn(session_id, turn_id, collaboration_answer=answer) + else: + answer = _as_dict(raw_answer) response = self._complete_answer(session_id, turn, answer) event_sink("answer.delta", {"text": response["message"]}) return response diff --git a/loopx/capabilities/native_chat/external_conversations.py b/loopx/capabilities/native_chat/external_conversations.py index 008f72e094..4cb2488e9a 100644 --- a/loopx/capabilities/native_chat/external_conversations.py +++ b/loopx/capabilities/native_chat/external_conversations.py @@ -13,6 +13,7 @@ import hashlib import json import re +import threading from datetime import datetime as datetime, timezone as timezone from typing import Any @@ -21,7 +22,12 @@ ensure_external_admission_scope, resolve_external_admission_source, ) -from ...chat_store import _atomic_write_json as _atomic_write_json, _read_json as _read_json, RESUMABLE_SESSION_STATES +from ...chat_store import ( + ChatSessionStore, + _atomic_write_json as _atomic_write_json, + _read_json as _read_json, + RESUMABLE_SESSION_STATES, +) from ...chat_attachments import normalize_chat_image_attachments from ...control_plane import effect_runtime from ...control_plane.quota.automation_cadence_readback import ( @@ -31,7 +37,18 @@ from ...file_lock import exclusive_file_lock as exclusive_file_lock, try_exclusive_file_lock from . import result_review_evidence from .codex_context import checked_session_home -from .file_discovery import FileChatDiscovery, external_source_route_path, later_equivalent_source_input, recovery_journal_fields, source_control_fields +from .file_discovery import ( + FileChatDiscovery, + QuotaBacklogObservationCache, + _same_store_method, + external_source_route_path, + file_change_fingerprint, + later_equivalent_source_input, + observe_quota_backlog_batch, + observed_turn_event_summary, + recovery_journal_fields, + source_control_fields, +) from .steward_commands import ExternalStewardCommissionMixin # Request journals are named by their 24-hex request_ref; lock metadata and @@ -41,6 +58,23 @@ _QUOTA_BACKLOG_BATCH_BYTES = 1024 * 1024 +class _DeferredBacklogRequest(dict[str, Any]): + """Bounded backlog request view carrying its observed file fingerprint.""" + + __slots__ = ("req_fp", "needs_full_read") + + def __init__( + self, + data: Mapping[str, Any], + *, + req_fp: tuple[int, ...] | None, + needs_full_read: bool, + ) -> None: + super().__init__(data) + self.req_fp = req_fp + self.needs_full_read = needs_full_read + + def _preparation_image_identity(value: Any) -> dict[str, Any] | None: """Fingerprint already normalized Core images without exporting their bytes.""" if not isinstance(value, list) or not value: @@ -55,6 +89,24 @@ def _preparation_image_identity(value: Any) -> dict[str, Any] | None: return None +def _project_source_request_digest(_path: Path, item: dict[str, Any]) -> dict[str, Any]: + canonical = json.dumps(item, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return { + "request_ref": item.get("request_ref"), + "binding_id": item.get("binding_id"), + "source": item.get("source"), + "sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), + } + + +def _project_user_message_hint(message: Mapping[str, Any]) -> dict[str, Any]: + return { + key: message[key] + for key in ("role", "turn_id", "text", "origin", "created_at") + if key in message + } + + class _RecoveryDiscoveryReads: """Lazy facts for one queue-discovery pass, never recovery admission. @@ -77,8 +129,7 @@ def queued(self, session_id: str) -> bool: def user_messages(self, session_id: str) -> list[dict[str, Any]]: if session_id not in self.histories: - self.histories[session_id] = [message for message in self.core.controller.store.messages(session_id) - if message.get("role") == "user"] + self.histories[session_id] = list(self.core._cached_user_messages(session_id)) return self.histories[session_id] def source_controls(self) -> list[dict[str, Any]]: @@ -102,10 +153,21 @@ def __init__(self, controller: Any) -> None: self.bindings.controller = controller self.root = controller.store.root / "external-requests" self._source_control_discovery = FileChatDiscovery() + self._source_request_discovery = FileChatDiscovery() self._backlog_request_discovery = FileChatDiscovery() self._recovery_discovery = FileChatDiscovery() + self._user_messages_lock = threading.Lock() + self._session_user_messages_cache: dict[str, tuple[tuple[int, ...], tuple[dict[str, Any], ...]]] = {} + self._quota_backlog_cache = QuotaBacklogObservationCache() self.actions: Any | None = None + def _event_summary_cache(self) -> QuotaBacklogObservationCache: + cache_store = getattr(self, "_quota_backlog_cache", None) + if cache_store is None: + cache_store = QuotaBacklogObservationCache() + self._quota_backlog_cache = cache_store + return cache_store + def _retained_result_review(self, row: dict[str, Any]) -> dict[str, Any] | None: """Read correlation only; the typed owner still checks continuation eligibility.""" reservation = row.get("context_recovery") or {} @@ -162,7 +224,19 @@ def _context_recovery_plan(self, row: dict[str, Any], *, and reservation.get("failed_turn_id") == turn["turn_id"] and (retained_review is None or reservation.get("work_attempt") == 1 or reservation.get("legacy_result_review")) else None) - events, anchors = self._started_native_inputs(session["session_id"], turn["turn_id"]) + image_failure = turn.get("error_code") == "runtime_error" and bool(row.get("attachments")) + runtime_failure = turn.get("error_code") in {"model_auth_unavailable", "resume_failed"} + needs_full_events = (not discovery_only or retained_review is not None + or turn.get("error_code") == "manager_authorization_unavailable" + or image_failure or runtime_failure) + if needs_full_events: + events, anchors = self._started_native_inputs(session["session_id"], turn["turn_id"]) + steering_observed = interrupt_observed = False + else: + single_anchor, steering_observed, interrupt_observed = observed_turn_event_summary( + store, session["session_id"], turn["turn_id"], self._event_summary_cache(), + real_events=_same_store_method(store, "events_after", ChatSessionStore)) + events, anchors = [], ([single_anchor] if single_anchor is not None else []) session = self._original_native_session(session, anchors) unfinished = turn.get("status") == "completed" and bool((turn.get("response") or {}).get("unfinished_work")) related_review = (result_review_evidence.collect_related_result_review(self, row, turn, discovery_reads=discovery_reads) @@ -185,8 +259,6 @@ def _context_recovery_plan(self, row: dict[str, Any], *, later_equivalent = None read_source_controls = (discovery_reads.source_controls if discovery_reads is not None else cache(self.source_controls)) - image_failure = turn.get("error_code") == "runtime_error" and bool(row.get("attachments")) - runtime_failure = turn.get("error_code") in {"model_auth_unavailable", "resume_failed"} if turn.get("error_code") == "manager_authorization_unavailable" or image_failure or runtime_failure: # Collect canonical facts only. The typed planner owns the exact # pre-dispatch sequence, retry budget and authority decision. @@ -239,9 +311,11 @@ def _context_recovery_plan(self, row: dict[str, Any], *, "queued_input_present": (discovery_reads.queued(session["session_id"]) if discovery_reads is not None else bool(store.queued_turns(session["session_id"]))), "attachments_present": bool(row.get("attachments")), - "steering_observed": any(event.get("kind") == "turn.steered" for event in events), - "stop_observed": self._source_stop_observed(row, turn, events, - source_controls=read_source_controls), + "steering_observed": (any(event.get("kind") == "turn.steered" for event in events) + if needs_full_events else steering_observed), + "stop_observed": (self._source_stop_observed(row, turn, events, source_controls=read_source_controls) + if needs_full_events else (interrupt_observed + or self._source_stop_observed(row, turn, (), source_controls=read_source_controls))), "preparation_observation": preparation, "later_equivalent_input_present": later_equivalent, "preparation_scope_available": preparation_scope_available, @@ -270,6 +344,73 @@ def _source_requests(self, row: dict[str, Any]) -> list[dict[str, Any]]: return [item for item in self.pending() if item.get("binding_id") == row["binding_id"] and item.get("source") == row["source"]] + def _source_request_digests(self, row: dict[str, Any]) -> list[dict[str, Any]]: + discovery = getattr(self, "_source_request_discovery", None) + if discovery is None: + discovery = FileChatDiscovery() + self._source_request_discovery = discovery + binding_id, source = row["binding_id"], row["source"] + return [ + item for _, item in discovery.rows( + self.root, self._request_paths(), read=_read_json, project=_project_source_request_digest, + ) + if item.get("binding_id") == binding_id and item.get("source") == source + ] + + def _cached_user_messages(self, session_id: str) -> tuple[dict[str, Any], ...]: + store = self.controller.store + is_real_store = ( + isinstance(store, ChatSessionStore) + and getattr(store.messages, "__func__", None) is ChatSessionStore.messages + ) + if not is_real_store: + return tuple( + _project_user_message_hint(message) + for message in store.messages(session_id) + if message.get("role") == "user" + ) + try: + msg_path = store._session_dir(session_id) / "messages.jsonl" + before = file_change_fingerprint(msg_path) + except (OSError, ValueError): + before = None + msg_path = None + lock = getattr(self, "_user_messages_lock", None) + if lock is None: + lock = threading.Lock() + self._user_messages_lock = lock + cache_map = getattr(self, "_session_user_messages_cache", None) + if cache_map is None: + cache_map = {} + self._session_user_messages_cache = cache_map + if before is None: + with lock: + cache_map.pop(session_id, None) + if msg_path is None: + return () + return tuple( + _project_user_message_hint(message) + for message in store.messages(session_id) + if message.get("role") == "user" + ) + with lock: + saved = cache_map.get(session_id) + if saved is not None and saved[0] == before: + return saved[1] + rows = tuple( + _project_user_message_hint(message) + for message in store.messages(session_id) + if message.get("role") == "user" + ) + try: + unchanged = msg_path is not None and file_change_fingerprint(msg_path) == before + except OSError: + unchanged = False + if unchanged: + with lock: + cache_map[session_id] = (before, rows) + return rows + def _codex_quota_route_capacity(self, session: dict[str, Any]) -> dict[str, Any] | None: from ...chat_manager import manager_session_quota_route route = manager_session_quota_route(self.controller, session) @@ -295,7 +436,7 @@ def _original_native_session(session: dict[str, Any], anchors: list[dict[str, An "upstream_thread_id": matches[0]["thread_id"]} def _source_stop_observed(self, row: dict[str, Any], turn: dict[str, Any], - events: list[dict[str, Any]], *, + events: Sequence[dict[str, Any]], *, source_controls: Callable[[], list[dict[str, Any]]] | None = None) -> bool: """Read the same durable source controls for recovery and display.""" return (any(event.get("kind") in {"turn.interrupt_requested", "turn.interrupted"} for event in events) @@ -306,36 +447,31 @@ def _source_stop_observed(self, row: dict[str, Any], turn: dict[str, Any], for item in (source_controls or self.source_controls)()))) def quota_backlog_observer( - self, *, source_controls: Callable[[], list[dict[str, Any]]] | None = None, + self, *, source_controls: Callable[[], list[dict[str, Any]]] | None = None, use_cache: bool = False, + observed_fps: Mapping[Path, tuple[int, ...]] | None = None, ) -> Callable[[Sequence[dict[str, Any]]], list[bool]]: """Read Session state after each batch's Turns and share source controls across batches.""" - read_controls = cache(source_controls or self.source_controls) + read_controls = cache(source_controls or (self.source_controls if observed_fps is None + else (lambda: self.source_controls(known_fingerprints=observed_fps)))) + + @cache + def stop_controls() -> tuple[tuple[Any, Any, datetime], ...]: + return tuple( + (item.get("binding_id"), item.get("source"), datetime.fromisoformat(item["created_at"])) + for item in read_controls() + if item.get("command") in {"stop", "new"} + ) def observe(rows: Sequence[dict[str, Any]]) -> list[bool]: if len(rows) > QUOTA_BACKLOG_BATCH_SIZE: raise ValueError("select at most 64 pending requests per observation") - params = [self._quota_pending_inputs(row) for row in rows] - load_session = self.controller.store.load_session - sessions = {} - for session_id in dict.fromkeys(row["session_id"] for row in rows): - # A transient read failure must not mask a closed Session. - session = load_session(session_id) or load_session(session_id) - sessions[session_id] = ({key: session[key] for key in ("session_id", "status") if key in session} - if session is not None else None) - for row, values in zip(rows, params, strict=True): - values["current_session"] = sessions[row["session_id"]] - flags = self._quota_pending_flags(params) - stopped = [] - for index, (row, values, pending) in enumerate(zip(rows, params, flags, strict=True)): - if pending: - events = self.controller.store.events_after(row["session_id"], row["turn_id"], None) - if self._source_stop_observed(row, values["pending_observation"]["turn"], events, - source_controls=read_controls): - values["pending_observation"]["stop_observed"] = True - stopped.append(index) - for index, pending in zip(stopped, self._quota_pending_flags([params[i] for i in stopped]), strict=True): - flags[index] = pending - return flags + cache_store = self._event_summary_cache() if use_cache else None + return observe_quota_backlog_batch( + root=self.root, store=self.controller.store, rows=rows, + build_inputs=self._quota_pending_inputs, eval_flags=self._quota_pending_flags, + stop_controls=stop_controls, evaluator_id=id(effect_runtime.effect_runtime_result), + cache_store=cache_store, + ) return observe @@ -376,6 +512,8 @@ def flush() -> None: return flags def _quota_pending_inputs(self, row: dict[str, Any]) -> dict[str, Any]: + if getattr(row, "needs_full_read", False): + row = self.read_request(row["request_ref"]) turn = self.controller.store.load_turn(row["session_id"], row["turn_id"]) return { "request": {key: value for key, value in row.items() if key != "attachments"}, @@ -462,7 +600,7 @@ def recover_failed_request(self, request_ref: str, *, else checked_session_home(self.controller.codex_home, session)) context = self.controller.project_contexts.session_context(session) - requests = self._source_requests(row) + requests = self._source_request_digests(row) # The read-only native client can consume its entire RPC deadline. # Retain this request's fence, but let new source input/control in. # Its observation is only usable if the original local state still @@ -564,7 +702,7 @@ def recover_failed_request(self, request_ref: str, *, or self.controller.store.load_turn(row["session_id"], current_turn["turn_id"]) != current_turn or (plan.get("completed_review_turn_id") or plan.get("original_result_turn_id")) and self.controller.store.load_turn(row["session_id"], turn["turn_id"]) != turn - or self._source_requests(row) != requests): + or self._source_request_digests(row) != requests): return row plan = self._context_recovery_plan(row, observation=proof, source_verified=source is not None, selected_binding=selected["binding"], preparation_scope_available=scope_available, @@ -725,7 +863,7 @@ def _recover_native_result(self, path: Path, row: dict[str, Any], session: dict[ turn: dict[str, Any], source_observation: Callable) -> dict[str, Any]: """Adopt an original provider result, without reopening/dispatching a Turn.""" store = self.controller.store - requests = self._source_requests(row) + requests = self._source_request_digests(row) context = self.controller.project_contexts.session_context(session) recovered = row.get("native_result_recovery") proof = self._original_native_result(row, session, turn, context) if recovered is None else recovered.get("native_observation") @@ -741,7 +879,7 @@ def _recover_native_result(self, path: Path, row: dict[str, Any], session: dict[ selected = self.bindings._resolve_observed(current, selected["binding"], selected["observation"], row["source"]) if (store.load_session(session["session_id"]) != session or store.load_turn(session["session_id"], turn["turn_id"]) != turn - or self._source_requests(row) != requests): + or self._source_request_digests(row) != requests): return row plan = self._context_recovery_plan(row, observation=proof, source_verified=source is not None, selected_binding=selected["binding"]) @@ -973,7 +1111,7 @@ def _checked_metadata_review(self, session: dict[str, Any], turn: dict[str, Any] error_message="The original source, grant or queue changed; metadata review is held.", require_empty_queue=True, ) - refs = sorted(item["request_ref"] for item in self._source_requests(row)) + refs = sorted(item["request_ref"] for item in self._source_request_digests(row)) native_session = self._original_native_session(current, [continuation["native_observation"]]) if (native_session.get("agent_id") != "antigravity" or native_session.get("upstream_thread_id") != continuation["native_observation"]["thread_id"] @@ -1747,28 +1885,52 @@ def _recovery_rows(self) -> list[dict[str, Any]]: return [r for _, r in self._recovery_discovery.rows( self.root, self._request_paths(), read=_read_json, project=recovery_journal_fields)] - def source_controls(self) -> list[dict[str, Any]]: + def source_controls(self, *, known_fingerprints: Mapping[Path, tuple[int, ...]] | None = None) -> list[dict[str, Any]]: """Observe source controls and private text digests without reparsing unchanged bodies.""" - return [r for _, r in self._source_control_discovery.rows( - self.root, self._request_paths(), read=_read_json, project=source_control_fields)] + return [r for _, r, _ in self._source_control_discovery.rows_with_fingerprints( + self.root, self._request_paths(), read=_read_json, project=source_control_fields, + known_fingerprints=known_fingerprints)] def read_request(self, request_ref: str) -> dict[str, Any]: return _read_json(self.root / f"{request_ref}.json") - def read_backlog_requests(self, request_refs: Iterable[str]) -> Iterator[dict[str, Any]]: + def read_backlog_requests( + self, request_refs: Iterable[str], *, defer_recovery: bool = False, + observed_fps: dict[Path, tuple[int, ...]] | None = None, + ) -> Iterator[dict[str, Any]]: """Observe settings correlation, without retaining ordinary request bodies.""" fields = ("request_ref", "binding_id", "source", "session_id", "turn_id", "status", "command", "created_at") + fresh: dict[Path, dict[str, Any]] = {} + + def read_current(path: Path) -> dict[str, Any]: + payload = _read_json(path) + if not defer_recovery: + fresh[path] = payload + return payload def project(_path: Path, row: dict[str, Any]) -> dict[str, Any] | None: - # Recovery evidence is consumed by the typed pending rule. - if any(row.get(k) is not None for k in ("context_recovery", "native_result_recovery")): + if not all(key in row for key in fields): return None - return {key: row[key] for key in fields} if all(key in row for key in fields) else None + return { + **{key: row[key] for key in fields}, + "_has_recovery": any(row.get(k) is not None for k in ("context_recovery", "native_result_recovery")), + } paths = (self.root / f"{ref}.json" for ref in request_refs) - for _, row in self._backlog_request_discovery.rows(self.root, paths, read=_read_json, project=project): - yield row + for path, row, fp in self._backlog_request_discovery.rows_with_fingerprints( + self.root, paths, read=read_current, project=project, + ): + payload = fresh.pop(path, None) + is_recovery = bool(row.pop("_has_recovery", False)) + if observed_fps is not None and fp is not None: + observed_fps[path] = fp + if is_recovery and not defer_recovery: + yield payload if payload is not None else _read_json(path) + elif defer_recovery: + yield _DeferredBacklogRequest(row, req_fp=fp, needs_full_read=is_recovery) + else: + yield row def record_delivery(self, request_ref: str, *, session_id: str | None, turn_id: str | None, message_text: str | None = None) -> None: diff --git a/loopx/capabilities/native_chat/file_discovery.py b/loopx/capabilities/native_chat/file_discovery.py index fe0958ad43..39bc49ae3f 100644 --- a/loopx/capabilities/native_chat/file_discovery.py +++ b/loopx/capabilities/native_chat/file_discovery.py @@ -1,7 +1,7 @@ """File-backed Chat journal discovery and bounded reply image artifacts.""" from __future__ import annotations -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from copy import deepcopy from datetime import datetime import base64 @@ -108,6 +108,9 @@ def later_equivalent_source_input( return False +_MISSING_FILE_FP: tuple[int, ...] = (-1,) + + def file_change_fingerprint(path: Path) -> tuple[int, ...] | None: """A local mutation token, or no reuse when the platform lacks one.""" # Windows stat ctime can be creation time, not a mutation stamp. Keep @@ -118,6 +121,300 @@ def file_change_fingerprint(path: Path) -> tuple[int, ...] | None: return stat.st_dev, stat.st_ino, stat.st_mtime_ns, stat.st_ctime_ns, stat.st_size +def optional_file_fingerprint(path: Path) -> tuple[int, ...] | None: + """Return a file fingerprint or a sentinel when absent; None when unavailable.""" + try: + return file_change_fingerprint(path) + except FileNotFoundError: + return None if sys.platform == "win32" else _MISSING_FILE_FP + except OSError: + return None + + +def _set_bounded_cache(cache_map: dict[Any, Any], key: Any, value: Any, max_entries: int) -> None: + if key in cache_map: + del cache_map[key] + elif len(cache_map) >= max_entries: + del cache_map[next(iter(cache_map))] + cache_map[key] = value + + +class QuotaBacklogObservationCache: + """Revision-bound cache for read-only quota backlog health classification. + + Stores only pure boolean verdicts and Turn status labels keyed by file + fingerprints, Session status, and evaluator identity. Never retains Turn + bodies or request attachments. + """ + + def __init__(self, *, max_entries: int = 8192) -> None: + self._lock = Lock() + self._max_entries = max(1, max_entries) + self._base: dict[str, tuple[tuple[Any, ...], tuple[bool, str | None]]] = {} + self._stopped: dict[str, tuple[tuple[Any, ...], bool]] = {} + self._event_summaries: dict[tuple[str, str], tuple[tuple[int, ...], tuple[dict[str, Any] | None, bool, bool]]] = {} + + def get_base(self, request_ref: str, key: tuple[Any, ...]) -> tuple[bool, str | None] | None: + with self._lock: + entry = self._base.get(request_ref) + return entry[1] if entry is not None and entry[0] == key else None + + def has_base_prefix(self, request_ref: str, key_prefix: tuple[Any, ...]) -> bool: + with self._lock: + entry = self._base.get(request_ref) + return entry is not None and entry[0][:-1] == key_prefix + + def put_base(self, request_ref: str, key: tuple[Any, ...], pending: bool, turn_status: str | None) -> None: + with self._lock: + _set_bounded_cache(self._base, request_ref, (key, (pending, turn_status)), self._max_entries) + stopped = self._stopped.get(request_ref) + if stopped is not None and stopped[0] != key: + self._stopped.pop(request_ref, None) + + def get_stopped(self, request_ref: str, key: tuple[Any, ...]) -> bool | None: + with self._lock: + entry = self._stopped.get(request_ref) + return entry[1] if entry is not None and entry[0] == key else None + + def put_stopped(self, request_ref: str, key: tuple[Any, ...], pending: bool) -> None: + with self._lock: + _set_bounded_cache(self._stopped, request_ref, (key, pending), self._max_entries) + + def drop_request(self, request_ref: str) -> None: + with self._lock: + self._base.pop(request_ref, None) + self._stopped.pop(request_ref, None) + + @staticmethod + def build_key_prefix( + root: Path, store: Any, row: dict[str, Any], evaluator_id: int, + *, req_fp_override: tuple[int, ...] | None = None, + ) -> tuple[Any, Path | None, Path | None, tuple[int, ...] | None, tuple[int, ...] | None]: + try: + req_path = root / f"{row['request_ref']}.json" + turn_path = store._turn_path(row["session_id"], row["turn_id"]) + req_fp = req_fp_override if isinstance(req_fp_override, tuple) else optional_file_fingerprint(req_path) + turn_fp = optional_file_fingerprint(turn_path) + except (OSError, ValueError): + return None, None, None, None, None + if req_fp is None or turn_fp is None: + return None, req_path, turn_path, req_fp, turn_fp + prefix = (req_fp, turn_fp, row["session_id"], row["turn_id"], + row.get("status"), row.get("command"), evaluator_id) + return prefix, req_path, turn_path, req_fp, turn_fp + + def commit(self, request_ref: str, meta_entry: tuple[Any, ...], pending: bool, *, + turn_status: str | None = None, stopped: bool = False) -> None: + key, req_path, turn_path, req_fp, turn_fp = meta_entry + if key is None or req_path is None or turn_path is None: + return + if (optional_file_fingerprint(req_path), optional_file_fingerprint(turn_path)) != (req_fp, turn_fp): + self.drop_request(request_ref) + return + if stopped: + self.put_stopped(request_ref, key, pending) + else: + self.put_base(request_ref, key, pending, turn_status) + + def get_event_summary( + self, session_id: str, turn_id: str, events_fp: tuple[int, ...], + ) -> tuple[dict[str, Any] | None, bool, bool] | None: + with self._lock: + entry = self._event_summaries.get((session_id, turn_id)) + if entry is None or entry[0] != events_fp: + return None + anchor, steering, interrupted = entry[1] + return (deepcopy(anchor) if anchor is not None else None), steering, interrupted + + def put_event_summary( + self, session_id: str, turn_id: str, events_fp: tuple[int, ...], + summary: tuple[dict[str, Any] | None, bool, bool], + ) -> None: + anchor, steering, interrupted = summary + stored = (deepcopy(anchor) if anchor is not None else None), steering, interrupted + with self._lock: + _set_bounded_cache(self._event_summaries, (session_id, turn_id), (events_fp, stored), self._max_entries) + + def drop_event_summary(self, session_id: str, turn_id: str) -> None: + with self._lock: + self._event_summaries.pop((session_id, turn_id), None) + + def get_interrupt(self, session_id: str, turn_id: str, events_fp: tuple[int, ...]) -> bool | None: + summary = self.get_event_summary(session_id, turn_id, events_fp) + return summary[2] if summary is not None else None + + def put_interrupt(self, session_id: str, turn_id: str, events_fp: tuple[int, ...], observed: bool) -> None: + self.put_event_summary(session_id, turn_id, events_fp, (None, False, observed)) + + def drop_interrupt(self, session_id: str, turn_id: str) -> None: + self.drop_event_summary(session_id, turn_id) + + +def _summarize_turn_events(events: Sequence[dict[str, Any]]) -> tuple[dict[str, Any] | None, bool, bool]: + anchors = [ + item["payload"]["native_input"] + for item in events + if item.get("kind") == "turn.started" and item.get("payload", {}).get("native_input") + ] + single_anchor = deepcopy(anchors[0]) if len(anchors) == 1 else None + steering = any(e.get("kind") == "turn.steered" for e in events) + interrupted = any(e.get("kind") in {"turn.interrupt_requested", "turn.interrupted"} for e in events) + return single_anchor, steering, interrupted + + +def observed_turn_event_summary( + store: Any, + session_id: str, + turn_id: str, + cache_store: QuotaBacklogObservationCache | None, + *, + real_events: bool, +) -> tuple[dict[str, Any] | None, bool, bool]: + """Read compact Turn event summary with fingerprint/pending-buffer guard on real stores.""" + if not real_events or cache_store is None: + return _summarize_turn_events(store.events_after(session_id, turn_id, None)) + key = (session_id, turn_id) + try: + event_path = store._event_path(session_id, turn_id) + except ValueError: + return _summarize_turn_events(store.events_after(session_id, turn_id, None)) + flush_lock = store._event_flush_lock(key) + fp_before: tuple[int, ...] | None = None + if flush_lock.acquire(blocking=False): + try: + with store._event_lock: + has_pending = bool(store._event_pending.get(key)) + if not has_pending: + fp_before = optional_file_fingerprint(event_path) + if fp_before is not None: + cached = cache_store.get_event_summary(session_id, turn_id, fp_before) + if cached is not None: + return cached + finally: + flush_lock.release() + summary = _summarize_turn_events(store.events_after(session_id, turn_id, None)) + if fp_before is not None and flush_lock.acquire(blocking=False): + try: + with store._event_lock: + has_pending_after = bool(store._event_pending.get(key)) + fp_after = None if has_pending_after else optional_file_fingerprint(event_path) + if fp_after == fp_before: + cache_store.put_event_summary(session_id, turn_id, fp_after, summary) + else: + cache_store.drop_event_summary(session_id, turn_id) + finally: + flush_lock.release() + return summary + + +def observed_turn_interrupt( + store: Any, + session_id: str, + turn_id: str, + cache_store: QuotaBacklogObservationCache | None, + *, + real_events: bool, +) -> bool: + """Check Turn interrupt events with fingerprint/pending-buffer guard on real stores.""" + return observed_turn_event_summary( + store, session_id, turn_id, cache_store, real_events=real_events, + )[2] + + +def _same_store_method(store: Any, name: str, expected_cls: type[Any]) -> bool: + return isinstance(store, expected_cls) and getattr(getattr(store, name, None), "__func__", None) is getattr(expected_cls, name) + + +def observe_quota_backlog_batch( + *, + root: Path, + store: Any, + rows: Sequence[dict[str, Any]], + build_inputs: Callable[[dict[str, Any]], dict[str, Any]], + eval_flags: Callable[[Sequence[dict[str, Any]]], list[bool]], + stop_controls: Callable[[], tuple[tuple[Any, Any, datetime], ...]], + evaluator_id: int, + cache_store: QuotaBacklogObservationCache | None, +) -> list[bool]: + """Evaluate backlog pending flags with optional revision-bound caching.""" + from ...chat_store import ChatSessionStore + + real_load_turn = cache_store is not None and _same_store_method(store, "load_turn", ChatSessionStore) + real_events = cache_store is not None and _same_store_method(store, "events_after", ChatSessionStore) + flags: list[bool] = [False] * len(rows) + turn_statuses: list[str | None] = [None] * len(rows) + params: list[dict[str, Any] | None] = [None] * len(rows) + meta: list[tuple[Any, ...]] = [(None, None, None, None, None)] * len(rows) + for index, row in enumerate(rows): + entry = (cache_store.build_key_prefix(root, store, row, evaluator_id, req_fp_override=getattr(row, "req_fp", None)) + if real_load_turn and cache_store is not None else (None, None, None, None, None)) + meta[index] = entry + if entry[0] is not None and cache_store is not None and cache_store.has_base_prefix(row["request_ref"], entry[0]): + continue + params[index] = build_inputs(row) + load_session = store.load_session + sessions: dict[str, dict[str, Any] | None] = {} + for session_id in dict.fromkeys(row["session_id"] for row in rows): + # A transient read failure must not mask a closed Session. + session = load_session(session_id) or load_session(session_id) + sessions[session_id] = ( + {key: session[key] for key in ("session_id", "status") if key in session} + if session is not None else None + ) + misses: list[tuple[int, dict[str, Any]]] = [] + for index, row in enumerate(rows): + session_row = sessions[row["session_id"]] + prefix, req_path, turn_path, req_fp, turn_fp = meta[index] + if prefix is not None and cache_store is not None: + session_status = session_row.get("status") if session_row is not None else None + full_key = (*prefix, session_status) + meta[index] = (full_key, req_path, turn_path, req_fp, turn_fp) + cached = cache_store.get_base(row["request_ref"], full_key) + if cached is not None: + flags[index], turn_statuses[index] = cached + continue + values = params[index] if params[index] is not None else build_inputs(row) + values["current_session"] = session_row + params[index] = values + misses.append((index, values)) + if misses: + for (index, values), pending in zip(misses, eval_flags([v for _, v in misses]), strict=True): + turn_dict = values["pending_observation"]["turn"] + turn_status = turn_dict.get("status") if isinstance(turn_dict, dict) else None + flags[index], turn_statuses[index] = pending, turn_status + if cache_store is not None: + cache_store.commit(rows[index]["request_ref"], meta[index], pending, turn_status=turn_status) + stopped_misses: list[tuple[int, dict[str, Any]]] = [] + for index, (row, pending) in enumerate(zip(rows, flags, strict=True)): + if not pending: + continue + if not observed_turn_interrupt(store, row["session_id"], row["turn_id"], cache_store, real_events=real_events): + if turn_statuses[index] not in {"failed", "completed"}: + continue + row_created = datetime.fromisoformat(row["created_at"]) + if not any(b == row["binding_id"] and s == row["source"] and c >= row_created + for b, s, c in stop_controls()): + continue + cache_key = meta[index][0] + if cache_store is not None and cache_key is not None: + cached_stop = cache_store.get_stopped(row["request_ref"], cache_key) + if cached_stop is not None: + flags[index] = cached_stop + continue + values = params[index] + if values is None: + values = build_inputs(row) + values["current_session"] = sessions[row["session_id"]] + values["pending_observation"]["stop_observed"] = True + stopped_misses.append((index, values)) + if stopped_misses: + for (index, _), pending in zip(stopped_misses, eval_flags([v for _, v in stopped_misses]), strict=True): + flags[index] = pending + if cache_store is not None: + cache_store.commit(rows[index]["request_ref"], meta[index], pending, stopped=True) + return flags + + class FileChatDiscovery: """Keep only caller-selected metadata while the complete file is unchanged. @@ -130,26 +427,33 @@ def __init__(self) -> None: self._lock = Lock() self._directories: dict[Path, dict[Path, tuple[tuple[int, ...], dict[str, Any]]]] = {} - def rows(self, directory: Path, paths: Iterable[Path], *, - read: Callable[[Path], dict[str, Any]], - project: Callable[[Path, dict[str, Any]], dict[str, Any] | None], - ) -> Iterator[tuple[Path, dict[str, Any]]]: + def rows_with_fingerprints( + self, + directory: Path, + paths: Iterable[Path], + *, + read: Callable[[Path], dict[str, Any]], + project: Callable[[Path, dict[str, Any]], dict[str, Any] | None], + known_fingerprints: Mapping[Path, tuple[int, ...]] | None = None, + ) -> Iterator[tuple[Path, dict[str, Any], tuple[int, ...] | None]]: seen = set() with self._lock: entries = self._directories.setdefault(directory, {}) for path in paths: seen.add(path) - try: - before = file_change_fingerprint(path) - except OSError: - before = None + before = known_fingerprints.get(path) if known_fingerprints is not None else None + if before is None: + try: + before = file_change_fingerprint(path) + except OSError: + before = None with self._lock: saved = entries.get(path) if before is None or saved is None or saved[0] != before: entries.pop(path, None) saved = None if saved is not None: - yield path, deepcopy(saved[1]) + yield path, deepcopy(saved[1]), before continue # Preserve the owning reader's missing/unreadable/strict behavior. value = read(path) @@ -162,11 +466,18 @@ def rows(self, directory: Path, paths: Iterable[Path], *, if unchanged: with self._lock: entries[path] = before, deepcopy(view) - yield path, value if view is None else view + yield path, (value if view is None else view), before with self._lock: for path in entries.keys() - seen: del entries[path] + def rows(self, directory: Path, paths: Iterable[Path], *, + read: Callable[[Path], dict[str, Any]], + project: Callable[[Path, dict[str, Any]], dict[str, Any] | None], + ) -> Iterator[tuple[Path, dict[str, Any]]]: + for path, view, _ in self.rows_with_fingerprints(directory, paths, read=read, project=project): + yield path, view + _REPLY_IMAGE_TYPES = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", diff --git a/loopx/capabilities/native_chat/github_pr_evidence.py b/loopx/capabilities/native_chat/github_pr_evidence.py index 765c3b95c3..a718608889 100644 --- a/loopx/capabilities/native_chat/github_pr_evidence.py +++ b/loopx/capabilities/native_chat/github_pr_evidence.py @@ -378,30 +378,177 @@ def summarize_github_pr_signal(row: Mapping[str, Any]) -> dict[str, Any]: return summary +_GITHUB_PR_LIST_PAGE_SIZE = 20 +_GITHUB_PR_LIST_MAX_PAGES = 5 +_GITHUB_OPEN_PRS_GRAPHQL_QUERY = """query($owner: String!, $name: String!, $cursor: String) { + repository(owner: $owner, name: $name) { + pullRequests( + states: OPEN, + first: 20, + after: $cursor, + orderBy: {field: UPDATED_AT, direction: DESC} + ) { + pageInfo { + hasNextPage + endCursor + } + nodes { + number + title + url + headRefOid + headRefName + baseRefName + isDraft + reviewDecision + mergeStateStatus + mergeable + updatedAt + author { + login + } + reviews(last: 20) { + nodes { + id + state + submittedAt + body + author { + login + } + commit { + oid + } + } + } + commits(last: 1) { + nodes { + commit { + statusCheckRollup { + contexts(first: 30) { + nodes { + __typename + ... on CheckRun { + name + status + conclusion + } + ... on StatusContext { + context + state + } + } + } + } + } + } + } + } + } + } +}""" + + +def _normalize_graphql_pr_node(node: Mapping[str, Any]) -> dict[str, Any]: + raw_reviews = node.get("reviews") + if isinstance(raw_reviews, Mapping) and isinstance(raw_reviews.get("nodes"), list): + reviews = [item for item in raw_reviews["nodes"] if isinstance(item, Mapping)] + elif isinstance(raw_reviews, list): + reviews = [item for item in raw_reviews if isinstance(item, Mapping)] + else: + reviews = [] + + rollup = node.get("statusCheckRollup") + if rollup is None: + raw_commits = node.get("commits") + commit_nodes = raw_commits.get("nodes") if isinstance(raw_commits, Mapping) else None + first_commit = ( + commit_nodes[0].get("commit") + if isinstance(commit_nodes, list) and commit_nodes and isinstance(commit_nodes[0], Mapping) + else None + ) + raw_rollup = first_commit.get("statusCheckRollup") if isinstance(first_commit, Mapping) else None + raw_contexts = raw_rollup.get("contexts") if isinstance(raw_rollup, Mapping) else None + context_nodes = raw_contexts.get("nodes") if isinstance(raw_contexts, Mapping) else None + rollup = [item for item in context_nodes if isinstance(item, Mapping)] if isinstance(context_nodes, list) else [] + + return { + "number": node.get("number"), + "title": node.get("title"), + "url": node.get("url"), + "headRefOid": node.get("headRefOid"), + "headRefName": node.get("headRefName"), + "baseRefName": node.get("baseRefName"), + "isDraft": node.get("isDraft"), + "author": node.get("author"), + "reviewDecision": node.get("reviewDecision"), + "mergeStateStatus": node.get("mergeStateStatus"), + "mergeable": node.get("mergeable"), + "updatedAt": node.get("updatedAt"), + "reviews": reviews, + "statusCheckRollup": rollup, + } + + def observe_github_repository_prs( repo_slug: str, *, runner: Callable[..., Any] = subprocess.run ) -> dict[str, Any]: """Read bounded public/authorized open PR signal metadata without side effects.""" + normalized = normalize_github_repo_slug(repo_slug) + if not normalized or "/" not in normalized: + return {"status": "unavailable", "repository": repo_slug} + owner, name = normalized.split("/", 1) + by_number: dict[int, dict[str, Any]] = {} + cursor: str | None = None try: - completed = runner( - [ - "gh", "pr", "list", "--repo", repo_slug, "--state", "open", - "--limit", "100", "--json", GITHUB_PR_LIST_FIELDS, - ], - capture_output=True, text=True, check=False, timeout=15, - ) - if completed.returncode != 0: - return {"status": "unavailable", "repository": repo_slug} - rows = json.loads(completed.stdout or "[]") - if not isinstance(rows, list): - return {"status": "unavailable", "repository": repo_slug} - items = [ - summarize_github_pr_signal(row) - for row in sorted( - (r for r in rows if isinstance(r, Mapping)), - key=lambda r: int(r.get("number") or 0), + for page_index in range(_GITHUB_PR_LIST_MAX_PAGES): + cmd = [ + "gh", + "api", + "graphql", + "-f", + f"query={_GITHUB_OPEN_PRS_GRAPHQL_QUERY}", + "-f", + f"owner={owner}", + "-f", + f"name={name}", + ] + if cursor is not None: + cmd.extend(["-f", f"cursor={cursor}"]) + completed = runner( + cmd, + capture_output=True, + text=True, + check=False, + timeout=20, ) - ] + if completed.returncode != 0: + return {"status": "unavailable", "repository": repo_slug} + payload = json.loads(completed.stdout or "{}") + if not isinstance(payload, Mapping) or payload.get("errors"): + return {"status": "unavailable", "repository": repo_slug} + data = payload.get("data") + repo = data.get("repository") if isinstance(data, Mapping) else None + pull_requests = repo.get("pullRequests") if isinstance(repo, Mapping) else None + nodes = pull_requests.get("nodes") if isinstance(pull_requests, Mapping) else None + if not isinstance(nodes, list): + return {"status": "unavailable", "repository": repo_slug} + for node in nodes: + if not isinstance(node, Mapping): + continue + number = int(node.get("number") or 0) + if number > 0: + by_number[number] = summarize_github_pr_signal(_normalize_graphql_pr_node(node)) + page_info = pull_requests.get("pageInfo") + if not isinstance(page_info, Mapping) or page_info.get("hasNextPage") is not True: + break + next_cursor = str(page_info.get("endCursor") or "").strip() + if not next_cursor or next_cursor == cursor: + if page_index < _GITHUB_PR_LIST_MAX_PAGES - 1: + return {"status": "unavailable", "repository": repo_slug} + break + cursor = next_cursor + items = [by_number[num] for num in sorted(by_number)] return {"status": "ok", "repository": repo_slug, "items": items} except (OSError, ValueError, TypeError, subprocess.TimeoutExpired): return {"status": "unavailable", "repository": repo_slug} diff --git a/loopx/capabilities/native_chat/steward_monitor.py b/loopx/capabilities/native_chat/steward_monitor.py index 18372f55b6..4bfb11b612 100644 --- a/loopx/capabilities/native_chat/steward_monitor.py +++ b/loopx/capabilities/native_chat/steward_monitor.py @@ -445,13 +445,30 @@ def pump_steward_monitors(controller: Any, *, cancelled=lambda: False) -> list[d _LOG.warning("An original steward report is unavailable; its window and other work are retained") due = service.due_monitors(actor) if due: - todo = due[0] - accepted, created = StewardTurnHost(controller).submit(binding["binding_id"], - turn_instance_id=monitor_turn_id(todo), todo_id=todo["todo_id"], work_kind="monitor", - monitor_deadline=todo["next_due_at"]) - if created: - changed.append(accepted) - continue + dispatched = False + for todo in due: + try: + accepted, created = StewardTurnHost(controller).submit( + binding["binding_id"], + turn_instance_id=monitor_turn_id(todo), + todo_id=todo["todo_id"], + work_kind="monitor", + monitor_deadline=todo["next_due_at"], + ) + except (OSError, ValueError, KeyError, TypeError, RuntimeError): + _LOG.warning( + "A native steward monitor is unavailable; original work retained" + ) + continue + if created: + changed.append(accepted) + dispatched = True + break + if isinstance(accepted, dict) and accepted.get("status") in {"queued", "running"}: + dispatched = True + break + if dispatched: + continue for adv in service.runnable_advancement_todos(actor): from .steward_turn import native_steward_client_id turn_id_token = advancement_turn_id(adv) @@ -473,8 +490,9 @@ def pump_steward_monitors(controller: Any, *, cancelled=lambda: False) -> list[d if all_monitors: soonest = min(all_monitors, key=lambda row: datetime.fromisoformat(row["next_due_at"].replace("Z", "+00:00"))) due_dt = datetime.fromisoformat(soonest["next_due_at"].replace("Z", "+00:00")) - remaining = max(0, (due_dt - datetime.now(timezone.utc)).total_seconds()) - cache[binding["binding_id"]] = time.monotonic() + min(60, remaining) + remaining = (due_dt - datetime.now(timezone.utc)).total_seconds() + if remaining > 0: + cache[binding["binding_id"]] = time.monotonic() + min(60, remaining) except (OSError, ValueError, KeyError, TypeError, RuntimeError): # One unreadable conversation or unavailable provider must not starve # other native monitors; retain its original data and deadline. diff --git a/loopx/chat/todo_actions.py b/loopx/chat/todo_actions.py index 4bdc1150f2..026b4d82bf 100644 --- a/loopx/chat/todo_actions.py +++ b/loopx/chat/todo_actions.py @@ -106,8 +106,58 @@ def _read_task_start( raise ValueError("The original Task Session is ambiguous") return (sessions[0] if sessions else None), None + def _pending_applied_task_receipt(self, proposal: dict[str, Any]) -> dict[str, Any] | None: + """Find the exact accepted Turn whose worker has yet to start.""" + parameters = proposal.get("normalized_parameters") or {} + receipt = proposal.get("receipt") or {} + if (proposal.get("status") != "applied" or proposal.get("action_kind") != "todo.create" + or parameters.get("start_execution") is not True + or proposal.get("permission_classification") != "durable_write" + or (proposal.get("context") or {}).get("binding_id") + or receipt.get("outcome") != "task_execution_started" + or receipt.get("projection_verified") is not True): + return None + resources = receipt.get("resource_ids") + canonical = receipt.get("canonical_receipt") + if not isinstance(resources, dict) or not isinstance(canonical, dict): + return None + goal_id = parameters.get("goal_id") + todo_id, sid, tid = (resources.get(key) for key in ("todo_id", "session_id", "turn_id")) + if (not all(isinstance(value, str) and value for value in (goal_id, todo_id, sid, tid)) + or resources.get("goal_id") != goal_id or canonical.get("goal_id") != goal_id + or canonical.get("todo_id") != todo_id or canonical.get("status") != "applied" + or canonical.get("preview_id") != proposal.get("expected_state_fingerprint")): + return None + chat = getattr(self.runtime_controller, "store", None) or self.chat_store + if chat is None: + return None + try: + turn = chat.load_turn(sid, tid) + except (OSError, ValueError): + return None + if (turn and turn.get("session_id") == sid and turn.get("turn_id") == tid + and turn.get("client_turn_id") == f"task-start-{proposal['proposal_id']}" + and turn.get("status") == "queued" and not (turn.get("started_at") or turn.get("upstream_turn_id"))): + return resources + return None + + def project_task_dispatch(self, proposal: dict[str, Any]) -> dict[str, Any]: + if self._pending_applied_task_receipt(proposal) is not None: + return {**proposal, "task_execution_dispatch_pending": True} + return proposal + + def _recover_applied_task(self, proposal: dict[str, Any]) -> dict[str, Any] | None: + execution = self._pending_applied_task_receipt(proposal) + if execution is None: + return self._turn_from_receipt(proposal.get("receipt")) + return self._start_confirmed_task( + proposal["proposal_id"], proposal["normalized_parameters"], execution["todo_id"], {}, + applied_execution=execution, + ) + def _start_confirmed_task( self, proposal_id: str, parameters: dict[str, Any], todo_id: str, steps: dict[str, Any], + *, applied_execution: dict[str, Any] | None = None, ) -> dict[str, Any]: from ..chat_actions import _opaque @@ -122,7 +172,7 @@ def _start_confirmed_task( channel_id = f"task.{todo_id}" client_turn_id = f"task-start-{proposal_id}" opened = steps.get("task_session_opened") - execution = steps.get("execution_started") + execution = applied_execution if applied_execution is not None else steps.get("execution_started") session, turn = self._read_task_start( chat, goal_id=goal_id, agent_id=endpoint, channel_id=channel_id, client_turn_id=client_turn_id, checkpoint=execution if execution is not None else opened, @@ -161,7 +211,7 @@ def _start_confirmed_task( # A past soft claim still needs a read-only current-owner check. if update_goal_todo(**claim_args, dry_run=True)["changed"]: raise ValueError("The reviewed Task assignment changed before launch") - if not isinstance(steps.get("todo_assigned"), dict): + if applied_execution is None and not isinstance(steps.get("todo_assigned"), dict): self.store.save_checkpoint( proposal_id, step="todo_assigned", receipt={"todo_id": todo_id, "agent_id": owner}, ) diff --git a/loopx/chat_actions.py b/loopx/chat_actions.py index 4b0e716a22..9f57ea7ae4 100644 --- a/loopx/chat_actions.py +++ b/loopx/chat_actions.py @@ -1651,7 +1651,7 @@ def _apply_owner_action(self, proposal_id: str, *, steward_context: dict[str, ob proposal = self.store.load(proposal_id) if proposal is None: raise KeyError("typed Chat action proposal was not found") - if proposal.get("status") == "applied": + if proposal.get("status") == "applied" and self._pending_applied_task_receipt(proposal) is None: return { "proposal": proposal, "turn": self._turn_from_receipt(proposal.get("receipt")), @@ -1698,7 +1698,7 @@ def _apply_owner_action(self, proposal_id: str, *, steward_context: dict[str, ob def _apply_confirmed_action(self, proposal_id: str) -> dict[str, object]: proposal = self.store.start_apply(proposal_id) if proposal["status"] == "applied": - return {"proposal": proposal, "turn": self._turn_from_receipt(proposal.get("receipt"))} + return {"proposal": proposal, "turn": self._recover_applied_task(proposal)} action_kind = str(proposal.get("action_kind") or "") parameters = proposal.get("normalized_parameters") if not isinstance(parameters, dict): diff --git a/loopx/chat_server.py b/loopx/chat_server.py index e9a1a7a064..99027ca214 100644 --- a/loopx/chat_server.py +++ b/loopx/chat_server.py @@ -1412,7 +1412,7 @@ def _action_snapshot(self, proposal_id: str) -> None: { "ok": True, "schema_version": "loopx_chat_action_v1", - "proposal": proposal, + "proposal": self.server.action_service.project_task_dispatch(proposal), } ) @@ -1436,7 +1436,7 @@ def _action_list(self) -> None: { "ok": True, "schema_version": "loopx_chat_action_list_v1", - "proposals": proposals, + "proposals": [self.server.action_service.project_task_dispatch(item) for item in proposals], } ) diff --git a/loopx/control_plane/collaboration/steward_managed_mode.ts b/loopx/control_plane/collaboration/steward_managed_mode.ts index 391e689f88..05ad062b08 100644 --- a/loopx/control_plane/collaboration/steward_managed_mode.ts +++ b/loopx/control_plane/collaboration/steward_managed_mode.ts @@ -62,10 +62,26 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se && Array.isArray(basis.phases) && basis.phases.length === 1 && requireJsonObject(basis.phases[0], "original watch phase").phase === "manager_context" && Array.isArray(events) && events.filter(kind => kind === "agent.phase").length === 1; + // A custom signal monitor may fail inside the read-only external signal + // check after manager_context completes and before any poll receipt is written. + // Allow at most one bounded recovery when zero-effect proofs hold. + let signalUnavailable = false; + if (original.error_code === "runtime_error" + && original.error === "steward monitor external signal source temporarily unavailable" + && requireJsonObject(prior.actor, "original watch actor").schema_version === "loopx_steward_managed_actor_v0" + && Array.isArray(basis.contexts) && basis.contexts.length === 1 + && Array.isArray(basis.phases) && basis.phases.length === 1 + && requireJsonObject(basis.phases[0], "original watch phase").phase === "manager_context" + && Array.isArray(events) && events.filter(kind => kind === "agent.phase").length === 1 + && Array.isArray(events) && events.filter(kind => kind === "manager.context").length === 1) { + const context = requireJsonObject(basis.contexts[0], "signal unavailable watch context"); + signalUnavailable = context.schema_version === "manager_turn_context_v1"; + } if (Object.keys(recovery).join() !== "original_turn_id" || recovery.original_turn_id !== original.turn_id || typeof original.turn_id !== "string" || !/^[a-f0-9]{32}$/.test(original.turn_id) || original.session_id !== session.session_id - || original.status !== "failed" || !(original.error_code === "server_restarted" || scopeRefused || identityRefused) + || original.status !== "failed" + || !(original.error_code === "server_restarted" || scopeRefused || identityRefused || signalUnavailable) || original.origin !== "native_loopx" || original.loopx_execution !== true || original.response !== null || original.upstream_turn_id !== null || original.delta_count !== 0 || prior.work_kind !== "monitor" || request.work_kind !== "monitor" || request.validation_command !== null diff --git a/loopx/control_plane/presentation/action_review_plan.ts b/loopx/control_plane/presentation/action_review_plan.ts index 9bd7518488..214c280ce4 100644 --- a/loopx/control_plane/presentation/action_review_plan.ts +++ b/loopx/control_plane/presentation/action_review_plan.ts @@ -609,6 +609,11 @@ export function compileActionReviewPlan(proposalValue: unknown, nowMs?: number): return held("repair", "readback_unverified"); } const taskExecution = taskExecutionReadback(proposal, receipt, identity.sourceFingerprint); + if (taskExecution && proposal.task_execution_dispatch_pending === true + && proposal.permission_classification === "durable_write" && identity.proposalId + && !objectValue(proposal.context)?.binding_id) { + return {...finish({interaction: "review", canApply: true, reason: "task_start_retry"}), retryOriginal: true}; + } const completed = held("completed", "readback_verified"); return taskExecution ? {...completed, taskExecution} : completed; } diff --git a/loopx/control_plane/quota/monitor_poll_admission.ts b/loopx/control_plane/quota/monitor_poll_admission.ts index 0e05fa9e4d..7ce319ae3f 100644 --- a/loopx/control_plane/quota/monitor_poll_admission.ts +++ b/loopx/control_plane/quota/monitor_poll_admission.ts @@ -69,11 +69,14 @@ function dueMonitorAllowed( ): boolean { if (!facts.must_attempt_work) return false; const candidate = decision.due_monitor_candidates.find((item) => candidateMatches(item, observation)); - if (facts.obligation === "attempt_due_monitor") return Boolean(candidate); - if (facts.reason_codes.has("due_monitor_context") && candidate) return true; const registry = decision.registry_due_monitor; - return facts.reason_codes.has("due_monitor_context") && registry.due === true && + const registryAllowed = registry.due === true && registry.claimed_by === decision.agent_id && candidateMatches(registry, observation); + if (facts.obligation === "attempt_due_monitor") { + return Boolean(candidate) || (!decision.requires_user_action && registryAllowed); + } + if (facts.reason_codes.has("due_monitor_context") && candidate) return true; + return facts.reason_codes.has("due_monitor_context") && registryAllowed; } function monitorGateAllowsObservation(decision: MonitorDecision): boolean { diff --git a/loopx/extensions/lark/goal_topic_runtime_service.py b/loopx/extensions/lark/goal_topic_runtime_service.py index 4541892d01..25a8016974 100644 --- a/loopx/extensions/lark/goal_topic_runtime_service.py +++ b/loopx/extensions/lark/goal_topic_runtime_service.py @@ -110,6 +110,8 @@ class ProfileWorkerFingerprint: class LarkGoalTopicRuntimeService: """Own one event-consumer worker per reusable Lark App profile.""" + _LISTENER_SNAPSHOT_TTL_SECONDS = 5.0 + def __init__( self, *, @@ -129,6 +131,11 @@ def __init__( self._profile_poller = profile_poller or self._poll_profile self.manager_route_reconciler = manager_route_reconciler self._lock = threading.Lock() + self._listener_snapshot_cond = threading.Condition(threading.Lock()) + self._listener_snapshot_in_flight = False + self._listener_snapshot_value: PayloadView | None = None + self._listener_snapshot_at = 0.0 + self._listener_snapshot_generation = 0 self._workers: dict[str, ProfileWorker] = {} self._message_consumers: dict[str, MessageConsumer] = {} self._health: dict[str, Payload] = {} @@ -357,6 +364,44 @@ def _handle_private_steward_callback( self._delivery_wakeup.set() return result + def _invalidate_listener_snapshot(self) -> int: + with self._listener_snapshot_cond: + self._listener_snapshot_generation += 1 + self._listener_snapshot_value = None + self._listener_snapshot_at = 0.0 + self._listener_snapshot_cond.notify_all() + return self._listener_snapshot_generation + + def _seed_listener_snapshot(self, generation: int, snapshot: PayloadView) -> None: + with self._listener_snapshot_cond: + if self._listener_snapshot_generation == generation and not self._closed.is_set(): + self._listener_snapshot_value = snapshot + self._listener_snapshot_at = time.monotonic() + + def _read_listener_snapshot(self) -> PayloadView: + with self._listener_snapshot_cond: + while self._listener_snapshot_in_flight: + self._listener_snapshot_cond.wait() + if ( + not self._closed.is_set() + and self._listener_snapshot_value is not None + and time.monotonic() - self._listener_snapshot_at < self._LISTENER_SNAPSHOT_TTL_SECONDS + ): + return self._listener_snapshot_value + generation = self._listener_snapshot_generation + self._listener_snapshot_in_flight = True + try: + fresh = self.snapshot_provider() + finally: + with self._listener_snapshot_cond: + self._listener_snapshot_in_flight = False + self._listener_snapshot_cond.notify_all() + with self._listener_snapshot_cond: + if self._listener_snapshot_generation == generation and not self._closed.is_set(): + self._listener_snapshot_value = fresh + self._listener_snapshot_at = time.monotonic() + return fresh + def _refresh_on_start(self) -> None: while not self._closed.is_set(): try: @@ -720,7 +765,7 @@ def start_consumer(args: list[str]) -> Popen[str]: continue def listener_snapshot() -> PayloadView: - current = self.snapshot_provider() + current = self._read_listener_snapshot() with self._lock: # Reuse the same observations used to admit # each worker. A successful refresh adds new @@ -830,7 +875,9 @@ def _refresh_profile_workers(self, *, resume_queues: bool = True) -> bool: return False self._profile_refresh_sequence += 1 sequence = self._profile_refresh_sequence + snapshot_generation = self._invalidate_listener_snapshot() snapshot = self.snapshot_provider() + self._seed_listener_snapshot(snapshot_generation, snapshot) profile_configs, unavailable = self._listener_configs(snapshot) with self._lock: if self._closed.is_set(): @@ -944,6 +991,7 @@ def active_profiles(self) -> list[str]: def close(self) -> None: self._closed.set() + self._invalidate_listener_snapshot() self._delivery_wakeup.set() with self._lock: if self._refresh_retry is not None: diff --git a/loopx/extensions/lark/private_conversations.py b/loopx/extensions/lark/private_conversations.py index 516f826685..dc325fcfb5 100644 --- a/loopx/extensions/lark/private_conversations.py +++ b/loopx/extensions/lark/private_conversations.py @@ -12,6 +12,7 @@ import re import subprocess import time +import zlib from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone @@ -26,7 +27,7 @@ STEWARD_BINDING_ADOPTION_PENDING, ) from ...capabilities.native_chat.file_discovery import ( - REPLY_IMAGE_NAME_PATTERN, FileChatDiscovery, reply_image_workspace, + REPLY_IMAGE_NAME_PATTERN, FileChatDiscovery, file_change_fingerprint, reply_image_workspace, ) from ...chat_attachments import CHAT_IMAGE_MAX_COUNT from ...chat_store import TERMINAL_TURN_STATES, _atomic_write_json, _read_json @@ -122,7 +123,10 @@ def __init__(self, *, controller: Any, runtime_root: Path, runner: Any, cli_bin: self.steward_callback_streams: dict[tuple[str, str], TeamPlanReviewCallbackStream] = {} self.root = controller.store.root / "lark-private-deliveries" self._context_recovery_probe_at: dict[Path, float] = {} + self._context_recovery_noop_backoff: dict[Path, tuple[tuple[int, ...] | None, tuple[int, ...] | None, int]] = {} self._historical_image_probe_at: dict[Path, float] = {} + self._planned_repairs_stamp: tuple[bool, int | None] | None = None + self._planned_repairs_names: frozenset[str] = frozenset() self._delivery_discovery = PrivateDeliveryDiscovery(self.root) self._body_discovery = PrivateDeliveryDiscovery(self.root) self._web_projection_scan_at = 0.0 @@ -186,7 +190,8 @@ def _health_delivery_discovery(self) -> FileChatDiscovery: @effect_runtime_request_scope() def health(self) -> dict[str, dict[str, int]]: health: dict[str, dict[str, int]] = {} - quota_pending = self.core.quota_backlog_observer() + observed_fps: dict[Path, tuple[int, ...]] = {} + quota_pending = self.core.quota_backlog_observer(use_cache=True, observed_fps=observed_fps) completed: list[tuple[dict[str, Any], dict[str, Any]]] = [] def count(row: dict[str, Any]) -> None: @@ -208,9 +213,7 @@ def health_fields(_path: Path, row: dict[str, Any]) -> dict[str, Any] | None: or (row.get("historical_answer_images") or {}).get("status") == "pending"): return None deliveries = row.get("deliveries") - if not isinstance(deliveries, dict) or not all( - isinstance(phase, dict) for phase in deliveries.values() - ): + if not isinstance(deliveries, dict) or not all(isinstance(phase, dict) for phase in deliveries.values()): return None # Settings uses only correlation and retry metadata. Never retain # delivered bodies or use these fields for admission or sending. @@ -222,9 +225,7 @@ def health_fields(_path: Path, row: dict[str, Any]) -> dict[str, Any] | None: def delivered_requests() -> Iterator[dict[str, Any]]: paths = [*self.root.glob("*.json"), *self.root.glob("web-projections/*.json")] - for path, row in self._health_delivery_discovery.rows( - self.root, paths, read=_read_json, project=health_fields, - ): + for path, row in self._health_delivery_discovery.rows(self.root, paths, read=_read_json, project=health_fields): health.setdefault(row["binding_id"], {"pending_count": 0, "recovery_count": 0}) if row["status"] != "delivered": if triage.pending(row): @@ -236,7 +237,9 @@ def delivered_requests() -> Iterator[dict[str, Any]]: # Pair the original receipts with the current Core correlation and Turn. receipts, refs = tee(delivered_requests()) - natives = self.core.read_backlog_requests(row["request_ref"] for row in refs) + natives = self.core.read_backlog_requests( + (row["request_ref"] for row in refs), defer_recovery=True, observed_fps=observed_fps, + ) for row, native in zip(receipts, natives, strict=True): if (native.get("status") == "accepted" and native.get("command") is None and _same_fields(native, row, "binding_id", "source", "session_id", "turn_id")): @@ -1525,19 +1528,81 @@ def authorized() -> bool: _atomic_write_json(path, record) return 1 + def _planned_historical_repairs(self) -> frozenset[str]: + repairs_dir = self.root / "historical-answer-images" + try: + stamp: tuple[bool, int | None] = (True, repairs_dir.stat().st_mtime_ns) + except OSError: + stamp = (False, None) + if stamp == getattr(self, "_planned_repairs_stamp", None): + return getattr(self, "_planned_repairs_names", frozenset()) + if not stamp[0]: + names: frozenset[str] = frozenset() + else: + names = frozenset(item.name for item in repairs_dir.glob("*.json")) + try: + stamp = (True, repairs_dir.stat().st_mtime_ns) + except OSError: + stamp = (False, None) + names = frozenset() + self._planned_repairs_stamp = stamp + self._planned_repairs_names = names + return names + def pending_body_paths(self) -> list[Path]: """Find unsent bodies without waiting for historical recovery probes.""" paths = [path for path, row, _ in self._body_discovery.rows() if row["status"] != "delivered" and triage.pending(row) and not _is_answer_image_followup(row)] return [*paths, *self._pending_web_projection_paths()] + def _noop_backoff_cache(self) -> dict[Path, tuple[tuple[int, ...] | None, tuple[int, ...] | None, int]]: + cache = getattr(self, "_context_recovery_noop_backoff", None) + if cache is None: + cache = {} + self._context_recovery_noop_backoff = cache + return cache + + def _context_recovery_fingerprints( + self, path: Path, request_ref: str, + ) -> tuple[tuple[int, ...] | None, tuple[int, ...] | None]: + core_root = getattr(self.core, "root", None) + native_fp: tuple[int, ...] | None = None + if isinstance(core_root, Path): + try: + native_fp = file_change_fingerprint(core_root / f"{request_ref}.json") + except OSError: + native_fp = None + try: + delivery_fp = file_change_fingerprint(path) + except OSError: + delivery_fp = None + return delivery_fp, native_fp + + def _clear_context_recovery_noop_backoff(self, path: Path) -> None: + self._noop_backoff_cache().pop(path, None) + + def _record_context_recovery_noop_backoff(self, path: Path, request_ref: str) -> None: + cache = self._noop_backoff_cache() + delivery_fp, native_fp = self._context_recovery_fingerprints(path, request_ref) + prev = cache.get(path) + streak = prev[2] + 1 if prev is not None and (prev[0], prev[1]) == (delivery_fp, native_fp) else 1 + cache[path] = (delivery_fp, native_fp, streak) + if streak <= 1: + delay = 60.0 + else: + base = min(1800, 60 * (2 ** min(streak - 1, 5))) + jitter = zlib.crc32(path.name.encode("utf-8")) % 60 + delay = float(base + jitter) + self._context_recovery_probe_at[path] = time.monotonic() + delay + @effect_runtime_request_scope() def pending_delivery_paths(self) -> list[Path]: """The durable transport store is the queue; no in-memory admission.""" paths = [] recovery_candidate = self.core.context_recovery_observer() now = datetime.now(timezone.utc).timestamp() - planned_repairs = {item.name for item in (self.root / "historical-answer-images").glob("*.json")} + planned_repairs = self._planned_historical_repairs() + noop_backoff = self._noop_backoff_cache() for path, row, repairs in self._delivery_discovery.rows(): if row["status"] != "delivered": if triage.pending(row) and not _answer_image_retry_waits(row, now): @@ -1547,14 +1612,25 @@ def pending_delivery_paths(self) -> list[Path]: elif (not row.get("historical_images_admitted") and path.name in planned_repairs and time.monotonic() >= self._historical_image_probe_at.get(path, 0)): paths.append(path) - elif (row.get("session_id") and row.get("turn_id") - and time.monotonic() >= self._context_recovery_probe_at.get(path, 0)): - # Only a discovery/backoff hint, never execution authority. - # The canonical request, provider effect inventory and source - # are independently re-read by recover_failed_request. - self._context_recovery_probe_at[path] = time.monotonic() + 60 - if repairs or recovery_candidate(row["request_ref"]): - paths.append(path) + elif row.get("session_id") and row.get("turn_id"): + saved_backoff = noop_backoff.get(path) + if (saved_backoff is not None + and self._context_recovery_fingerprints(path, row["request_ref"]) != saved_backoff[:2]): + noop_backoff.pop(path, None) + self._context_recovery_probe_at.pop(path, None) + if time.monotonic() >= self._context_recovery_probe_at.get(path, 0): + # Only a discovery/backoff hint, never execution authority. + # The canonical request, provider effect inventory and source + # are independently re-read by recover_failed_request. + # Stagger steady-state ineligible re-probes across [60, 120)s by filename + # hash so cold-start bursts do not stay synchronized on the same tick, + # while keeping eligible candidates on the exact 60s retry cadence. + now_mono = time.monotonic() + stagger = zlib.crc32(path.name.encode("utf-8")) % 60 + self._context_recovery_probe_at[path] = now_mono + 60 + stagger + if repairs or recovery_candidate(row["request_ref"]): + self._context_recovery_probe_at[path] = now_mono + 60 + paths.append(path) return [*paths, *self._web_projection_paths()] def is_recovery_path(self, path: Path) -> bool: @@ -1691,9 +1767,15 @@ def reconcile_request(self, path: Path) -> int: source_observation=lambda selected: self._recovery_source_observation(row, selected)) if (row["status"] == "delivered" and recovered.get("native_result_recovery") and recovered["native_result_recovery"]["turn_id"] == recovered.get("turn_id")): - return self._reconcile_native_result(path, recovered) + delivered_native = self._reconcile_native_result(path, recovered) + if delivered_native: + self._clear_context_recovery_noop_backoff(path) + else: + self._record_context_recovery_noop_backoff(path, row["request_ref"]) + return delivered_native if (recovered.get("context_recovery") and not recovered.get("delivery_verified") and recovered.get("turn_id") != row["turn_id"]): + self._clear_context_recovery_noop_backoff(path) with exclusive_file_lock(path, operation="deliver_private_chat_request"): row = _read_json(path) if not _same_fields(row, recovered, "request_ref", "binding_id", "source", "session_id"): @@ -1701,6 +1783,8 @@ def reconcile_request(self, path: Path) -> int: row.update(status="accepted", turn_id=recovered["turn_id"], context_recovery=recovered["context_recovery"]) _atomic_write_json(path, row) + elif row["status"] == "delivered": + self._record_context_recovery_noop_backoff(path, row["request_ref"]) if row["status"] == "captured": # Intake may still own this capture. Wait on its existing fence and # read its result before retrying admission from a stale snapshot. diff --git a/loopx/extensions/lark/private_delivery_discovery.py b/loopx/extensions/lark/private_delivery_discovery.py index 1011de8439..5a42030c70 100644 --- a/loopx/extensions/lark/private_delivery_discovery.py +++ b/loopx/extensions/lark/private_delivery_discovery.py @@ -230,6 +230,12 @@ def _observe(self, path: Path) -> tuple[dict[str, Any], bool]: self._completed.pop(path, None) row = _read_json(path) if row["status"] != "delivered": + triage = row.get("triage") + if isinstance(triage, dict) and triage.get("retired"): + view = {"status": row["status"], "triage": {"retired": triage["retired"]}} + if before is not None and file_change_fingerprint(path) == before: + self._completed[path] = (before, view, False) + return view, False return row, False repairs = bool(row.get("session_id") and row.get("turn_id") and legacy_recovery_repairs(row)) diff --git a/tests/control_plane/test_chat_task_start_recovery.py b/tests/control_plane/test_chat_task_start_recovery.py index 56dd2af5cf..f36aa4292a 100644 --- a/tests/control_plane/test_chat_task_start_recovery.py +++ b/tests/control_plane/test_chat_task_start_recovery.py @@ -292,3 +292,80 @@ def after_completion(proposal_id, *, step, receipt): assert calls.count("thread/start") == 1 assert calls.count("turn/start") == (0 if interruption == "accepted-then-stopped" else 1) assert _snapshot(runtime) == before + + +@pytest.mark.parametrize("stopped", [False, True], ids=["active", "stopped"]) +def test_applied_confirmation_retries_the_original_undispatched_turn(task_start, monkeypatch, stopped): + actions, request, chat, controller, reopen, runtime, capture, proposal_id = task_start + path = f"/api/actions/{proposal_id}" + + # The server persisted acceptance and returned it, but its scheduled worker + # never wrote the first durable start fact before the process went away. + with monkeypatch.context() as faults: + faults.setattr(controller, "_start_accepted_turn_worker", lambda **kwargs: True) + code, original = request(f"{path}/apply", {}) + assert code == 202 and original["proposal"]["status"] == "applied", original + receipt = original["proposal"]["receipt"] + ids = receipt["resource_ids"] + sid, tid = ids["session_id"], ids["turn_id"] + queued = chat.load_turn(sid, tid) + assert queued["status"] == "queued" and queued["started_at"] is None + if stopped: + _stop_goal(request) + before = _snapshot(runtime) + chat, controller = reopen() + + # Looking at the original confirmation or its goal's list must never start it. + code, detail = request(path) + assert code == 200 and detail["proposal"]["receipt"] == receipt + code, history = request("/api/actions?goal_id=report") + assert code == 200, history + listed = next(row for row in history["proposals"] if row["proposal_id"] == proposal_id) + assert chat.load_turn(sid, tid) == queued + assert all(json.loads(line)["method"] != "turn/start" for line in capture.read_text().splitlines()) + + if stopped: + code, retried = request(f"{path}/apply", {}) + assert code == 400 and "Goal is no longer active" in retried["error"], retried + assert chat.load_turn(sid, tid) == queued + else: + entered, proceed = Event(), Event() + start = controller._start_accepted_turn_worker + + def pause_ready_dispatch(**kwargs): + entered.set() + assert proceed.wait(20), "the original retry did not resume" + return start(**kwargs) + + monkeypatch.setattr(controller, "_start_accepted_turn_worker", pause_ready_dispatch) + with ThreadPoolExecutor(max_workers=1) as pool: + original_retry = pool.submit(request, f"{path}/apply", {}) + try: + assert entered.wait(10), "the original retry did not reach dispatch" + code, concurrent = request(f"{path}/apply", {}) + assert code == 409 and concurrent["error_code"] == "action_conflict", concurrent + finally: + proceed.set() + code, retried = original_retry.result(timeout=25) + assert code == 202 and retried["proposal"]["receipt"] == receipt, retried + assert retried["turn"]["created"] is False + assert retried["turn"]["turn_id"] == tid + assert controller.wait_for_turn(session_id=sid, turn_id=tid, timeout_sec=3)["status"] == "completed" + code, again = request(f"{path}/apply", {}) + assert code == 202 and again["proposal"]["receipt"] == receipt + assert again["turn"]["status"] == "completed" + code, complete = request(path) + assert code == 200 and "task_execution_dispatch_pending" not in complete["proposal"] + + assert detail["proposal"]["task_execution_dispatch_pending"] is True + assert listed["task_execution_dispatch_pending"] is True + assert "task_execution_dispatch_pending" not in actions.load(proposal_id) + assert actions.load(proposal_id)["receipt"] == receipt + assert _snapshot(runtime) == before + assert [row["session_id"] for row in chat.session_candidates( + goal_id="report", agent_id="codex", channel_id=f"task.{ids['todo_id']}", + )] == [sid] + assert len([row for row in chat.messages(sid) if row.get("role") == "user"]) == 1 + calls = [json.loads(line)["method"] for line in capture.read_text().splitlines()] + assert calls.count("thread/start") == 1 + assert calls.count("turn/start") == (0 if stopped else 1) diff --git a/tests/control_plane/test_new_goal_storage_defaults.py b/tests/control_plane/test_new_goal_storage_defaults.py index b294eec86a..f2bc388873 100644 --- a/tests/control_plane/test_new_goal_storage_defaults.py +++ b/tests/control_plane/test_new_goal_storage_defaults.py @@ -342,10 +342,13 @@ def app(environment): thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() - def request(path, body): + def request(path, body=None): connection = http.client.HTTPConnection(*server.server_address, timeout=30) try: - connection.request("POST", path, json.dumps(body), {"Content-Type": "application/json"}) + if body is None: + connection.request("GET", path) + else: + connection.request("POST", path, json.dumps(body), {"Content-Type": "application/json"}) response = connection.getresponse() return response.status, json.loads(response.read()) finally: diff --git a/tests/control_plane_ts/action_review_plan.test.ts b/tests/control_plane_ts/action_review_plan.test.ts index bc52c22407..219ce826d9 100644 --- a/tests/control_plane_ts/action_review_plan.test.ts +++ b/tests/control_plane_ts/action_review_plan.test.ts @@ -66,6 +66,18 @@ test("confirmed Task dispatch resolves only to the receipt's own execution Turn" ]) { assert.equal(compileActionReviewPlan({...proposal, ...change}).taskExecution, undefined); } + const pending = {...proposal, permission_classification: "durable_write", task_execution_dispatch_pending: true}; + assert.equal(compileActionReviewPlan(pending).reason, "task_start_retry"); + assert.equal(compileActionReviewPlan(pending).canApply, true); + assert.equal(compileActionReviewPlan(pending).retryOriginal, true); + for (const change of [ + {task_execution_dispatch_pending: "true"}, + {context: {binding_id: "bound-source"}}, + {permission_classification: "protected"}, + {receipt: {...receipt, resource_ids: {...resources, turn_id: ""}}}, + ]) { + assert.equal(compileActionReviewPlan({...pending, ...change}).canApply, false); + } // Older saved actions can still report creation while the Session is unknown. const legacy = compileActionReviewPlan({...proposal, receipt: {projection_verified: true, outcome: "todo_added"}}); assert.equal(legacy.interaction, "completed"); diff --git a/tests/control_plane_ts/quota_monitor_poll_commit.test.ts b/tests/control_plane_ts/quota_monitor_poll_commit.test.ts index 69d7c418ed..62e85380e7 100644 --- a/tests/control_plane_ts/quota_monitor_poll_commit.test.ts +++ b/tests/control_plane_ts/quota_monitor_poll_commit.test.ts @@ -291,6 +291,67 @@ test("admission revalidates due, external, and exact blocked-wait modes", async ); }); +test("attempt_due_monitor admits secondary claimed due monitors via registry_due_monitor unless user gate applies", async () => { + const secondaryCandidate = { + todo_id: "todo_secondary_monitor", + target_key: "secondary-watch", + task_class: "continuous_monitor", + claimed_by: "codex-main-control", + due: true, + }; + const baseDecision = (overrides: Record = {}) => decision({ + should_run: true, + agent_id: "codex-main-control", + effective_action: "normal_run", + requires_user_action: false, + work_lane_contract: { + must_attempt_work: true, + obligation: "attempt_due_monitor", + }, + due_monitor_candidates: [{ + todo_id: "todo_primary_monitor", + target_key: "primary-watch", + task_class: "continuous_monitor", + }], + registry_due_monitor: secondaryCandidate, + ...overrides, + }); + const admitted = await evaluateQuotaMonitorPollCommit(request({ + decision: baseDecision(), + observation: observation({ + todo_id: "todo_secondary_monitor", + target_key: "secondary-watch", + result_hash: "unchanged-secondary", + }), + })); + assert.equal( + (admitted.record?.monitor_event as Record).monitor_mode, + "due_monitor_observed_without_material_transition", + ); + + for (const invalid of [ + { requires_user_action: true }, + { registry_due_monitor: { ...secondaryCandidate, due: false } }, + { registry_due_monitor: { ...secondaryCandidate, claimed_by: "other-agent" } }, + ]) { + await assert.rejects( + () => evaluateQuotaMonitorPollCommit(request({ + decision: baseDecision(invalid), + observation: observation({ + todo_id: "todo_secondary_monitor", + target_key: "secondary-watch", + result_hash: "unchanged-secondary", + }), + })), + (error: unknown) => { + assert.ok(error instanceof EffectRuntimeRequestError); + assert.equal(error.code, "monitor_poll_admission_rejected"); + return true; + }, + ); + } +}); + test("Todo preflight rejects invalid admission before journaling provider intent", async (t) => { const runtimeRoot = await tempRuntime(t); const effectId = "quota-monitor-poll:rejected-provider"; diff --git a/tests/control_plane_ts/steward_managed_mode.test.ts b/tests/control_plane_ts/steward_managed_mode.test.ts index fea6e2422e..2ee64fe2a8 100644 --- a/tests/control_plane_ts/steward_managed_mode.test.ts +++ b/tests/control_plane_ts/steward_managed_mode.test.ts @@ -82,6 +82,45 @@ test("identity-refused watches need independent collector evidence and retain cu } }); +test("signal-unavailable custom monitors admit exactly one zero-effect recovery after manager_context", () => { + const {params, goal} = fixture(); + const actor = bindStewardManagedActor({...params, goal}); + const prior = {operation: "native_turn", host: "codex-app-server", actor, + work_kind: "monitor", turn_instance_id: "monitor.signal", todo_id: "todo_signal_watch", validation_command: null, + monitor_deadline: "2026-11-19T13:21:54Z", monitor_lease_version: 0}; + const original = {turn_id: "2".repeat(32), session_id: params.session.session_id, + status: "failed", error_code: "runtime_error", + error: "steward monitor external signal source temporarily unavailable", + origin: "native_loopx", loopx_execution: true, upstream_turn_id: null, response: null, + delta_count: 0, loopx_request: prior}; + const basis = {original, contexts: [{schema_version: "manager_turn_context_v1", goals: [goal]}], + phases: [{phase: "manager_context"}], + event_kinds: ["turn.queued", "agent.phase", "manager.context", "turn.failed"], + poll_absent: true, old_worker_excluded: true, lease_available: true}; + const monitor = {todo_id: "todo_signal_watch", task_class: "continuous_monitor", + target_key: "steward-stalled-pr-monitor", claimed_by: "steward", status: "open", + next_due_at: prior.monitor_deadline}; + const input = {...params, goal, goal_active: true, binding: {...params.binding, managed_actor: actor}, + request: {...prior, monitor_restart_recovery: {original_turn_id: original.turn_id}}, + monitor_todo: monitor, monitor_restart_basis: basis}; + assert.equal(admitStewardManagedTurn(input).observation_only, true); + + for (const changed of [ + {contexts: []}, + {contexts: [{schema_version: "wrong_schema"}]}, + {contexts: [...basis.contexts, ...basis.contexts]}, + {phases: [{phase: "model_start"}]}, + {event_kinds: ["turn.queued", "agent.phase", "turn.failed"]}, + {poll_absent: false}, + {lease_available: false}, + {original: {...original, error: "other transient failure"}}, + {original: {...original, upstream_turn_id: "model-started"}}, + {original: {...original, loopx_request: {...prior, monitor_restart_recovery: {original_turn_id: "3".repeat(32)}}}}, + ]) { + assert.throws(() => admitStewardManagedTurn({...input, monitor_restart_basis: {...basis, ...changed}})); + } +}); + test("monitor configuration retains the original actor and native Todo authority", () => { const {params, goal} = fixture(); const actor = bindStewardManagedActor({...params, goal}); diff --git a/tests/test_lark_private_conversations.py b/tests/test_lark_private_conversations.py index 7271944959..52f9c22a9a 100644 --- a/tests/test_lark_private_conversations.py +++ b/tests/test_lark_private_conversations.py @@ -8178,7 +8178,7 @@ def observed(path): assert "private-body-marker" not in repr(transport._delivery_discovery._completed) _atomic_write_json(new, {"status": "pending"}) assert transport.pending_delivery_paths() == [new, pending] - clock[0] += 61 + clock[0] += 120 # Ineligible recovery probes stagger across [60, 120)s. eligible["old"] = True assert transport.pending_delivery_paths() == [new, old, pending] clock[0] += 61 diff --git a/tests/test_lark_steward_directory_scan_perf.py b/tests/test_lark_steward_directory_scan_perf.py index aee1ff8b8f..34d0b3af56 100644 --- a/tests/test_lark_steward_directory_scan_perf.py +++ b/tests/test_lark_steward_directory_scan_perf.py @@ -298,3 +298,794 @@ def test_external_conversations_recover_does_not_retain_delivered_image_payloads retained_bytes = sum(stat.size_diff for stat in after.compare_to(before, "lineno") if stat.size_diff > 0) assert retained_bytes < 512 * 1024, f"recover() retained {retained_bytes} bytes of delivered image payloads" + + +def test_private_conversations_caches_historical_repairs_and_staggers_recovery_probes(tmp_path, monkeypatch): + from loopx.extensions.lark import private_conversations as private + from loopx.extensions.lark.private_delivery_discovery import PrivateDeliveryDiscovery + + repairs_dir = tmp_path / "historical-answer-images" + repairs_dir.mkdir() + _atomic_write_json(repairs_dir / f"{_hex_ref(1)}.json", {"turn_id": "t1"}) + + clock = [100.0] + monkeypatch.setattr(private, "time", SimpleNamespace(monotonic=lambda: clock[0])) + + transport = private.LarkPrivateConversations.__new__(private.LarkPrivateConversations) + transport.root = tmp_path + transport._context_recovery_probe_at = {} + transport._historical_image_probe_at = {} + transport._delivery_discovery = PrivateDeliveryDiscovery(tmp_path) + transport._web_projection_paths = lambda: [] + transport.core = SimpleNamespace(context_recovery_observer=lambda: lambda _ref: False) + + for idx in range(2, 62): + ref = _hex_ref(idx) + _atomic_write_json( + tmp_path / f"{ref}.json", + { + "status": "delivered", + "request_ref": ref, + "session_id": f"sess_{idx}", + "turn_id": f"turn_{idx}", + "profile": "fixture", + "event": {"message_id": f"om_{idx}"}, + }, + ) + + glob_calls = 0 + original_glob = Path.glob + + def counted_glob(self: Path, pattern: str): + nonlocal glob_calls + if self == repairs_dir: + glob_calls += 1 + return original_glob(self, pattern) + + monkeypatch.setattr(Path, "glob", counted_glob) + + assert transport.pending_delivery_paths() == [] + assert glob_calls == 1 + assert transport.pending_delivery_paths() == [] + assert glob_calls == 1, "unchanged historical-answer-images dir must hit mtime cache without globbing" + + # Adding a new repair manifest updates dir mtime_ns and refreshes the cached set once. + _atomic_write_json(repairs_dir / f"{_hex_ref(2)}.json", {"turn_id": "t2"}) + assert transport.pending_delivery_paths() == [tmp_path / f"{_hex_ref(2)}.json"] + assert glob_calls == 2 + + # Delivered probe deadlines must stagger across [60, 120)s rather than locking every file onto T+60. + deadlines = set(transport._context_recovery_probe_at.values()) + assert len(deadlines) > 10 + assert all(160.0 <= value < 220.0 for value in deadlines) + + +def test_reconcile_request_backs_off_repeated_noop_recovery_and_resets_on_change(tmp_path, monkeypatch): + import zlib + from loopx.extensions.lark import private_conversations as private + from loopx.extensions.lark.private_delivery_discovery import PrivateDeliveryDiscovery + + deliv_dir = tmp_path / "lark-private-deliveries" + ext_dir = tmp_path / "external-requests" + deliv_dir.mkdir() + ext_dir.mkdir() + + ref = _hex_ref(42) + deliv_path = deliv_dir / f"{ref}.json" + ext_path = ext_dir / f"{ref}.json" + deliv_row = { + "status": "delivered", + "request_ref": ref, + "session_id": f"sess_{ref}", + "turn_id": f"turn_{ref}", + "profile": "fixture", + "event": {"message_id": f"om_{ref}"}, + } + _atomic_write_json(deliv_path, deliv_row) + _atomic_write_json(ext_path, {"request_ref": ref, "status": "accepted"}) + + clock = [1000.0] + monkeypatch.setattr(private, "time", SimpleNamespace(monotonic=lambda: clock[0])) + monkeypatch.setattr(private, "record_event_timing", lambda *_a, **_kw: None) + + transport = private.LarkPrivateConversations.__new__(private.LarkPrivateConversations) + transport.root = deliv_dir + transport.reaction_feedback = False + transport._context_recovery_probe_at = {} + transport._historical_image_probe_at = {} + transport._delivery_discovery = PrivateDeliveryDiscovery(deliv_dir) + transport._web_projection_paths = lambda: [] + + noop_calls = 0 + + def fake_recover(_request_ref: str, **_kwargs): + nonlocal noop_calls + noop_calls += 1 + return {"request_ref": ref, "status": "accepted", "turn_id": f"turn_{ref}", "delivery_verified": True} + + transport.core = SimpleNamespace( + root=ext_dir, + context_recovery_observer=lambda: lambda candidate_ref: candidate_ref == ref, + context_recovery_candidate=lambda candidate_ref: candidate_ref == ref, + recover_failed_request=fake_recover, + recover=lambda **_kwargs: None, + ) + + jitter = zlib.crc32(deliv_path.name.encode("utf-8")) % 60 + + # Discovery finds candidate -> first noop reconcile keeps exact +60s retry. + assert transport.pending_delivery_paths() == [deliv_path] + assert transport.reconcile_request(deliv_path) == 0 + assert noop_calls == 1 + assert transport._context_recovery_probe_at[deliv_path] == clock[0] + 60.0 + + # Before +60s, unchanged fingerprints remain backed off. + clock[0] = 1059.0 + assert transport.pending_delivery_paths() == [] + + # Second consecutive noop backs off to 120 + jitter; third to 240 + jitter. + clock[0] = 1060.0 + assert transport.pending_delivery_paths() == [deliv_path] + assert transport.reconcile_request(deliv_path) == 0 + assert transport._context_recovery_probe_at[deliv_path] == 1060.0 + 120.0 + jitter + + clock[0] = 1060.0 + 120.0 + jitter + assert transport.pending_delivery_paths() == [deliv_path] + assert transport.reconcile_request(deliv_path) == 0 + assert transport._context_recovery_probe_at[deliv_path] == clock[0] + 240.0 + jitter + + # Changing the external-request file resets backoff immediately on the next discovery. + clock[0] += 5.0 + _atomic_write_json(ext_path, {"request_ref": ref, "status": "accepted", "updated": 1}) + assert transport.pending_delivery_paths() == [deliv_path] + + +def test_source_request_digests_cache_unchanged_requests_and_detect_mutation(tmp_path, monkeypatch): + import loopx.capabilities.native_chat.external_conversations as ext_mod + + core = ChatExternalConversations.__new__(ChatExternalConversations) + core.root = tmp_path + refs = [_hex_ref(1), _hex_ref(2), _hex_ref(3)] + for idx, ref in enumerate(refs): + _atomic_write_json( + tmp_path / f"{ref}.json", + { + "request_ref": ref, + "binding_id": "binding_one", + "source": {"source_ref": "source_one"}, + "seq": idx, + }, + ) + target = {"binding_id": "binding_one", "source": {"source_ref": "source_one"}} + + reads: list[Path] = [] + original_read = ext_mod._read_json + + def counted_read(path: Path): + reads.append(path) + return original_read(path) + + monkeypatch.setattr(ext_mod, "_read_json", counted_read) + + first = core._source_request_digests(target) + assert [row["request_ref"] for row in first] == refs + assert len(reads) == 3 + + reads.clear() + second = core._source_request_digests(target) + assert second == first + assert reads == [], "unchanged external-request files must not be re-read" + + # Mutating a single request invalidates only that file and changes its digest. + mutated_path = tmp_path / f"{refs[1]}.json" + _atomic_write_json( + mutated_path, + { + "request_ref": refs[1], + "binding_id": "binding_one", + "source": {"source_ref": "source_one"}, + "seq": 99, + }, + ) + third = core._source_request_digests(target) + assert reads == [mutated_path] + assert third[1]["sha256"] != first[1]["sha256"] + assert third[0]["sha256"] == first[0]["sha256"] + + +def test_recovery_discovery_reads_caches_session_user_messages_without_attachments(tmp_path, monkeypatch): + from loopx.capabilities.native_chat.external_conversations import _RecoveryDiscoveryReads + from loopx.chat_store import ChatSessionStore + + store = ChatSessionStore(tmp_path) + session = store.create_session( + goal_id="goal-perf-test", + agent_id="codex", + adapter_kind="codex", + upstream_thread_id="thread_perf_1", + ) + sid = session["session_id"] + png_1x1 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + ) + store.append_message( + sid, + role="user", + text="hello", + origin="lark", + turn_id="turn_1", + attachments=[{"id": "att_1", "name": "a.png", "mime_type": "image/png", "size": 68, "data_url": f"data:image/png;base64,{png_1x1}"}], + ) + store.append_message(sid, role="agent", text="reply", turn_id="turn_1") + + core = ChatExternalConversations.__new__(ChatExternalConversations) + core.controller = SimpleNamespace(store=store) + + msg_reads = 0 + original_messages = ChatSessionStore.messages + + def counted_messages(self, session_id: str, *, strict: bool = False): + nonlocal msg_reads + msg_reads += 1 + return original_messages(self, session_id, strict=strict) + + monkeypatch.setattr(ChatSessionStore, "messages", counted_messages) + + msgs_pass1 = _RecoveryDiscoveryReads(core).user_messages(sid) + assert msg_reads == 1 + assert [m["text"] for m in msgs_pass1] == ["hello"] + + # Fresh discovery pass must hit fingerprint cache without re-reading messages.jsonl. + msgs_pass2 = _RecoveryDiscoveryReads(core).user_messages(sid) + assert msg_reads == 1 + assert len(msgs_pass2) == 1 + assert msgs_pass2[0]["text"] == "hello" + assert "attachments" not in msgs_pass2[0] + + # Appending a new user message updates messages.jsonl fingerprint and refreshes cache. + store.append_message(sid, role="user", text="world", origin="lark", turn_id="turn_2") + msgs_pass3 = _RecoveryDiscoveryReads(core).user_messages(sid) + assert msg_reads == 2 + assert [m["text"] for m in msgs_pass3] == ["hello", "world"] + + +def test_private_delivery_discovery_caches_retired_triage_captures(tmp_path, monkeypatch): + import loopx.extensions.lark.private_delivery_discovery as pdd_mod + from loopx.extensions.lark.private_delivery_discovery import PrivateDeliveryDiscovery + + retired_path = tmp_path / f"{_hex_ref(10)}.json" + active_path = tmp_path / f"{_hex_ref(11)}.json" + _atomic_write_json(retired_path, {"status": "captured", "triage": {"retired": "card_delivery_expired"}}) + _atomic_write_json(active_path, {"status": "captured", "profile": "steward"}) + + reads: list[Path] = [] + original_read = pdd_mod._read_json + + def counted_read(path: Path): + reads.append(path) + return original_read(path) + + monkeypatch.setattr(pdd_mod, "_read_json", counted_read) + + discovery = PrivateDeliveryDiscovery(tmp_path) + first = {path: row for path, row, _ in discovery.rows()} + assert set(first) == {retired_path, active_path} + assert sorted(reads) == sorted([retired_path, active_path]) + + reads.clear() + second = {path: row for path, row, _ in discovery.rows()} + assert second[retired_path]["triage"]["retired"] == "card_delivery_expired" + assert reads == [active_path], "retired triage capture must stay cached while active capture stays fresh" + + +def _seed_backlog_health_store(tmp_path: Path, *, failed_count: int = 20, completed_count: int = 20): + from loopx.chat_store import CHAT_TURN_SCHEMA_VERSION, ChatSessionStore + from loopx.extensions.lark.private_conversations import LarkPrivateConversations + + store = ChatSessionStore(tmp_path) + session = store.create_session( + goal_id="goal-health-cache", + agent_id="codex", + adapter_kind="codex_app_server", + upstream_thread_id="thread-health-cache", + session_id="a" * 32, + ) + sid = session["session_id"] + controller = SimpleNamespace( + store=store, + project_contexts=SimpleNamespace(conversation_bindings=SimpleNamespace()), + ) + core = ChatExternalConversations(controller) + transport = LarkPrivateConversations.__new__(LarkPrivateConversations) + transport.core = core + transport.root = store.root / "lark-private-deliveries" + core.root.mkdir(parents=True, exist_ok=True) + transport.root.mkdir(parents=True, exist_ok=True) + + failed_rows: list[dict[str, object]] = [] + for idx in range(failed_count + completed_count): + is_failed = idx < failed_count + ref = _hex_ref(1000 + idx) + tid = f"{1000 + idx:032x}" + row = { + "request_ref": ref, + "binding_id": "b" * 24, + "source": {"source_ref": "c" * 24}, + "session_id": sid, + "turn_id": tid, + "command": None, + "status": "accepted", + "delivery_verified": True, + "message": f"req {idx}", + "created_at": "2026-01-01T00:00:00+00:00", + } + turn = { + "schema_version": CHAT_TURN_SCHEMA_VERSION, + "session_id": sid, + "turn_id": tid, + "client_turn_id": f"external-{ref}", + "status": "failed" if is_failed else "completed", + **({"error_code": "usage_limit_exceeded", "response": None} if is_failed else { + "error_code": None, + "response": {"message": "done", "proposals": []}, + }), + } + _atomic_write_json(store._turn_path(sid, tid), turn) + _atomic_write_json(core.root / f"{ref}.json", row) + _atomic_write_json(transport.root / f"{ref}.json", {**row, "status": "delivered", "deliveries": {}}) + if is_failed: + failed_rows.append(row) + return store, core, transport, sid, failed_rows + + +def test_backlog_health_caches_unchanged_turns_and_avoids_ipc_and_event_lock_churn(tmp_path, monkeypatch): + from loopx.chat_store import ChatSessionStore + from loopx.control_plane import effect_runtime + + store, core, transport, _sid, failed_rows = _seed_backlog_health_store(tmp_path) + original_rpc = effect_runtime.effect_runtime_result + original_load_turn = ChatSessionStore.load_turn + original_events_after = ChatSessionStore.events_after + counts = {"rpc": 0, "load_turn": 0, "events_after": 0} + + def counted_rpc(method, params, **kwargs): + if method == "collaboration.conversation.request": + counts["rpc"] += 1 + return original_rpc(method, params, **kwargs) + + def counted_load_turn(self, session_id: str, turn_id: str): + counts["load_turn"] += 1 + return original_load_turn(self, session_id, turn_id) + + def counted_events_after(self, session_id: str, turn_id: str, event_id: str | None): + counts["events_after"] += 1 + return original_events_after(self, session_id, turn_id, event_id) + + monkeypatch.setattr(effect_runtime, "effect_runtime_result", counted_rpc) + monkeypatch.setattr(ChatSessionStore, "load_turn", counted_load_turn) + monkeypatch.setattr(ChatSessionStore, "events_after", counted_events_after) + + assert transport.health()["b" * 24] == {"pending_count": 20, "recovery_count": 0} + assert counts["rpc"] == 1 + assert counts["load_turn"] == 40 + assert counts["events_after"] == 20 + + counts = {"rpc": 0, "load_turn": 0, "events_after": 0} + assert transport.health()["b" * 24] == {"pending_count": 20, "recovery_count": 0} + assert counts == {"rpc": 0, "load_turn": 0, "events_after": 0} + + # Uncached direct observer (default use_cache=False) still evaluates in-memory mutations fresh. + mutated = [{**failed_rows[0], "command": "stop"}] + assert core.quota_backlog_observer()(mutated) == [False] + + +def test_backlog_health_cache_invalidates_on_turn_event_recovery_session_and_stop_mutations(tmp_path, monkeypatch): + from loopx.chat_store import CHAT_TURN_SCHEMA_VERSION + from loopx.control_plane import effect_runtime + + store, core, transport, sid, failed_rows = _seed_backlog_health_store(tmp_path, failed_count=6, completed_count=4) + original_rpc = effect_runtime.effect_runtime_result + rpc_calls = 0 + + def counted_rpc(method, params, **kwargs): + nonlocal rpc_calls + if method == "collaboration.conversation.request": + rpc_calls += 1 + return original_rpc(method, params, **kwargs) + + monkeypatch.setattr(effect_runtime, "effect_runtime_result", counted_rpc) + assert transport.health()["b" * 24]["pending_count"] == 6 + assert rpc_calls == 1 + + # 1. Completing one failed Turn invalidates only that row immediately. + r0 = failed_rows[0] + _atomic_write_json(store._turn_path(sid, str(r0["turn_id"])), { + "schema_version": CHAT_TURN_SCHEMA_VERSION, + "session_id": sid, + "turn_id": r0["turn_id"], + "client_turn_id": f"external-{r0['request_ref']}", + "status": "completed", + "error_code": None, + "response": {"message": "ok", "proposals": []}, + }) + assert transport.health()["b" * 24]["pending_count"] == 5 + + # 2. Appending a flushed turn.interrupted event invalidates that Turn immediately. + r1 = failed_rows[1] + store.append_event(sid, str(r1["turn_id"]), kind="turn.interrupted", payload={}, buffered=False) + assert transport.health()["b" * 24]["pending_count"] == 4 + + # 3. Buffered (unflushed) turn.interrupt_requested in _event_pending must never serve stale cache. + r2 = failed_rows[2] + store.append_event(sid, str(r2["turn_id"]), kind="turn.interrupt_requested", payload={}, buffered=True) + assert transport.health()["b" * 24]["pending_count"] == 3 + + # 4. Recording a verified native_result_recovery on disk invalidates that request immediately. + r3 = failed_rows[3] + _atomic_write_json(core.root / f"{r3['request_ref']}.json", { + **r3, + "native_result_recovery": { + "turn_id": r3["turn_id"], + "message_id": f"native.{r3['turn_id']}.recovered", + "message_text": "recovered ok", + "delivery_verified": True, + "native_observation": {"response": {"message": "recovered ok", "proposals": []}}, + }, + }) + assert transport.health()["b" * 24]["pending_count"] == 2 + + # 5. Writing a later /stop command clears remaining pending items using only stopped RPC, then hits _stopped cache. + rpc_calls = 0 + _atomic_write_json(core.root / f"{_hex_ref(9999)}.json", { + "request_ref": _hex_ref(9999), + "binding_id": "b" * 24, + "source": {"source_ref": "c" * 24}, + "session_id": sid, + "turn_id": None, + "command": "stop", + "status": "command_completed", + "created_at": "2026-01-02T00:00:00+00:00", + }) + assert transport.health()["b" * 24]["pending_count"] == 0 + assert rpc_calls == 1 + + rpc_calls = 0 + assert transport.health()["b" * 24]["pending_count"] == 0 + assert rpc_calls == 0 + + # Replacing effect_runtime_result invalidates cached verdicts by evaluator identity. + def wrapper_rpc(method, params, **kwargs): + nonlocal rpc_calls + if method == "collaboration.conversation.request": + rpc_calls += 1 + return original_rpc(method, params, **kwargs) + + monkeypatch.setattr(effect_runtime, "effect_runtime_result", wrapper_rpc) + assert transport.health()["b" * 24]["pending_count"] == 0 + assert rpc_calls > 0 + rpc_calls = 0 + assert transport.health()["b" * 24]["pending_count"] == 0 + assert rpc_calls == 0 + + # 6. Closing a Session also drops remaining cached pending counts. + _store2, _core2, transport2, sid2, _ = _seed_backlog_health_store(tmp_path / "s2", failed_count=3, completed_count=0) + assert transport2.health()["b" * 24]["pending_count"] == 3 + sess_path = _store2._session_path(sid2) + _atomic_write_json(sess_path, {**store_read_json(sess_path), "status": "closed"}) + assert transport2.health()["b" * 24]["pending_count"] == 0 + + +def test_listener_snapshot_coalesces_concurrent_watchers_and_invalidates_on_refresh(tmp_path, monkeypatch): + import threading + from loopx.extensions.lark import goal_topic_runtime_service as svc_mod + from loopx.extensions.lark.goal_topic_runtime_service import LarkGoalTopicRuntimeService + + clock = [100.0] + monkeypatch.setattr(svc_mod.time, "monotonic", lambda: clock[0]) + + entered = threading.Event() + release = threading.Event() + provider_calls = 0 + snapshot_seq = 0 + fail_next = False + + def snapshot_provider(): + nonlocal provider_calls, snapshot_seq, fail_next + provider_calls += 1 + if fail_next: + fail_next = False + raise RuntimeError("transient snapshot error") + if provider_calls == 1: + entered.set() + assert release.wait(2) + snapshot_seq += 1 + return { + "private_profiles": {}, + "target_payload": {"targets": {}}, + "binding_payloads": {}, + "seq": snapshot_seq, + } + + service = LarkGoalTopicRuntimeService( + snapshot_provider=snapshot_provider, + runtime_root=tmp_path, + runtime_controller=SimpleNamespace(store=None), + profile_poller=lambda _profile, stop: stop.wait(), + ) + results: list[dict[str, object]] = [] + errors: list[BaseException] = [] + + def reader(): + try: + results.append(dict(service._read_listener_snapshot())) + except BaseException as exc: + errors.append(exc) + + threads = [threading.Thread(target=reader) for _ in range(3)] + for t in threads: + t.start() + assert entered.wait(2) + # Give sibling threads a moment to block on single-flight condition before releasing. + threading.Event().wait(0.05) + release.set() + for t in threads: + t.join(2) + assert not errors + assert provider_calls == 1 + assert [row["seq"] for row in results] == [1, 1, 1] + + # Within TTL, repeated calls hit the coalesced cache. + clock[0] += 4.5 + assert service._read_listener_snapshot()["seq"] == 1 + assert provider_calls == 1 + + # Past TTL (5.0s), failures are not cached and the following call retries cleanly. + clock[0] += 0.6 + fail_next = True + with pytest.raises(RuntimeError, match="transient snapshot error"): + service._read_listener_snapshot() + assert provider_calls == 2 + assert service._read_listener_snapshot()["seq"] == 2 + assert provider_calls == 3 + + # refresh() invalidates and seeds the cache atomically. + assert service._refresh_profile_workers(resume_queues=False) is False + assert provider_calls == 4 + assert service._read_listener_snapshot()["seq"] == 3 + assert provider_calls == 4 + service.close() + + +def test_context_recovery_discovery_caches_turn_event_summary_without_event_lock_churn(tmp_path, monkeypatch): + from loopx.chat_manager import MANAGER_CONTEXT_VERSION + from loopx.chat_store import CHAT_TURN_SCHEMA_VERSION, ChatSessionStore + + store = ChatSessionStore(tmp_path) + binding_id = "b" * 24 + source_ref = "c" * 24 + binding = { + "schema_version": "loopx_chat_conversation_binding_v0", + "binding_id": binding_id, + "enabled": True, + "transport_ref": "steward-app", + "context_kind": "steward", + "goal_ids": [], + "grant": "portfolio_read", + "provider_ref": "a" * 24, + "operator_ref": "e" * 24, + "project_ref": "f" * 24, + "executor_endpoint_id": "antigravity", + } + session = store.create_session( + goal_id="loopx-manager", + agent_id="antigravity", + adapter_kind="antigravity", + upstream_thread_id="thread-recovery-events", + upstream_mode="default", + session_id="d" * 32, + channel_id=f"manager.external.native.{binding_id}.{source_ref}", + steward_context={ + "kind": "bound_steward", + "binding_id": binding_id, + "audience": "bound_owner", + "grant": "portfolio_read", + "goal_ids": [], + "provider_ref": "a" * 24, + "operator_ref": "e" * 24, + "project_ref": "f" * 24, + "source_ref": source_ref, + "workspace_path": str(tmp_path), + }, + ) + sid = session["session_id"] + sess_path = store._session_path(sid) + _atomic_write_json(sess_path, { + **store.load_session(sid), + "session_mode": "managed_runtime", + "executor_endpoint_id": "antigravity", + "manager_context_version": MANAGER_CONTEXT_VERSION, + }) + controller = SimpleNamespace( + store=store, + lock=SimpleNamespace(__enter__=lambda s: None, __exit__=lambda s, *a: False), + adapters={}, + codex_home=tmp_path, + project_contexts=SimpleNamespace( + conversation_bindings=SimpleNamespace( + read=lambda: {"bindings": [binding]}, + resolve=lambda **_: {"binding": binding}, + ), + ), + ) + core = ChatExternalConversations(controller) + core.root.mkdir(parents=True, exist_ok=True) + + refs: list[str] = [] + tids: list[str] = [] + # 12 failed turns (> ChatEventCache's 8-turn limit) would thrash events_after without summary caching. + for idx in range(12): + ref = _hex_ref(2000 + idx) + tid = f"{2000 + idx:032x}" + refs.append(ref) + tids.append(tid) + _atomic_write_json(store._turn_path(sid, tid), { + "schema_version": CHAT_TURN_SCHEMA_VERSION, + "session_id": sid, + "turn_id": tid, + "client_turn_id": f"external-{ref}", + "message": f"question {idx}", + "origin": "lark", + "status": "failed", + "error_code": "antigravity_step_error", + "delta_count": 0, + "upstream_turn_id": f"upstream-{idx}", + "created_at": "2026-01-01T00:00:00+00:00", + "completed_at": "2026-01-01T00:01:00+00:00", + "response": None, + }) + store.append_event(sid, tid, kind="turn.started", payload={ + "native_input": { + "thread_id": "thread-recovery-events", + "start_step_index": idx, + "input_sha256": "a" * 64, + }, + }, buffered=False) + if 4 <= idx < 8: + store.append_event(sid, tid, kind="turn.steered", payload={}, buffered=False) + elif idx >= 8: + store.append_event(sid, tid, kind="turn.interrupted", payload={}, buffered=False) + _atomic_write_json(core.root / f"{ref}.json", { + "request_ref": ref, + "binding_id": binding_id, + "source": {"source_ref": source_ref, "sender_ref": "e" * 24, "private_human_message": True}, + "session_id": sid, + "turn_id": tid, + "command": None, + "origin": "lark", + "status": "accepted", + "delivery_verified": True, + "message": f"question {idx}", + "created_at": "2026-01-01T00:00:00+00:00", + }) + store.update_session(sid, manager_context_version=MANAGER_CONTEXT_VERSION) + + event_reads: list[str] = [] + original_events_after = ChatSessionStore.events_after + + def counted_events_after(self_store, session_id: str, turn_id: str, event_id: str | None): + event_reads.append(turn_id) + return original_events_after(self_store, session_id, turn_id, event_id) + + monkeypatch.setattr(ChatSessionStore, "events_after", counted_events_after) + + observe1 = core.context_recovery_observer() + verdicts1 = [observe1(ref) for ref in refs] + assert len(event_reads) == 12 + assert verdicts1 == [True] * 4 + [False] * 8 + + # Pass 2 across all 12 historical turns hits _event_summaries with zero events_after reads. + event_reads.clear() + observe2 = core.context_recovery_observer() + assert [observe2(ref) for ref in refs] == verdicts1 + assert event_reads == [] + + # Flushed turn.interrupted event invalidates only that Turn immediately. + store.append_event(sid, tids[0], kind="turn.interrupted", payload={}, buffered=False) + observe3 = core.context_recovery_observer() + assert observe3(refs[0]) is False + assert observe3(refs[1]) is True + assert event_reads == [tids[0]] + + # Buffered unflushed event in _event_pending bypasses cache immediately. + event_reads.clear() + store.append_event(sid, tids[1], kind="turn.interrupt_requested", payload={}, buffered=True) + observe4 = core.context_recovery_observer() + assert observe4(refs[1]) is False + assert event_reads == [tids[1]] + + # Non-discovery admission and preparation failures always read fresh full events_after. + event_reads.clear() + row2 = core.read_request(refs[2]) + core._context_recovery_plan(row2, discovery_only=False, selected_binding=binding) + assert event_reads == [tids[2]] + + event_reads.clear() + _atomic_write_json(store._turn_path(sid, tids[3]), { + "schema_version": CHAT_TURN_SCHEMA_VERSION, + "session_id": sid, + "turn_id": tids[3], + "client_turn_id": f"external-{refs[3]}", + "status": "failed", + "error_code": "manager_authorization_unavailable", + "created_at": "2026-01-01T00:00:00+00:00", + "response": None, + }) + core.context_recovery_observer()(refs[3]) + assert event_reads == [tids[3]] + + +def test_backlog_health_defers_recovery_body_reads_and_reuses_request_fingerprints(tmp_path, monkeypatch): + import loopx.capabilities.native_chat.external_conversations as ext_mod + import loopx.capabilities.native_chat.file_discovery as fd_mod + + store, core, transport, sid, failed_rows = _seed_backlog_health_store(tmp_path, failed_count=3, completed_count=2) + # Add two requests carrying context_recovery metadata so deferred recovery reads are exercised. + for row in failed_rows[:2]: + _atomic_write_json(core.root / f"{row['request_ref']}.json", { + **row, + "context_recovery": {"failed_turn_id": row["turn_id"], "client_turn_id": f"external-{row['request_ref']}-recovery-1"}, + }) + # Add an earlier /stop command for an unrelated source so stop_controls() is queried while pending stays 3. + _atomic_write_json(core.root / f"{_hex_ref(8888)}.json", { + "request_ref": _hex_ref(8888), + "binding_id": "b" * 24, + "source": {"source_ref": "e" * 24}, + "session_id": sid, + "turn_id": None, + "command": "stop", + "status": "command_completed", + "created_at": "2026-01-01T00:00:00+00:00", + }) + + # Cold pass warms both FileChatDiscovery and QuotaBacklogObservationCache. + assert transport.health()["b" * 24]["pending_count"] == 3 + + reads: list[Path] = [] + stats_by_path: dict[Path, int] = {} + original_read = ext_mod._read_json + original_fp = fd_mod.file_change_fingerprint + + def counted_read(path: Path): + reads.append(path) + return original_read(path) + + def counted_fp(path: Path): + stats_by_path[path] = stats_by_path.get(path, 0) + 1 + return original_fp(path) + + monkeypatch.setattr(ext_mod, "_read_json", counted_read) + monkeypatch.setattr(fd_mod, "file_change_fingerprint", counted_fp) + + assert transport.health()["b" * 24]["pending_count"] == 3 + assert reads == [], "warm health() must not re-read recovery requests" + for row in failed_rows: + req_path = core.root / f"{row['request_ref']}.json" + assert stats_by_path.get(req_path) == 1, f"expected 1 stat for {req_path}, got {stats_by_path.get(req_path)}" + + # Mutating a recovery request (marking recovered on row 0) with preserved mtime invalidates and drops pending to 2. + target_path = core.root / f"{failed_rows[0]['request_ref']}.json" + stamp = target_path.stat() + _atomic_write_json(target_path, { + **store_read_json(target_path), + "native_result_recovery": { + "turn_id": failed_rows[0]["turn_id"], + "message_id": f"native.{failed_rows[0]['turn_id']}.recovered", + "message_text": "recovered ok", + "delivery_verified": True, + "native_observation": {"response": {"message": "recovered ok", "proposals": []}}, + }, + }) + os.utime(target_path, ns=(stamp.st_atime_ns, stamp.st_mtime_ns)) + reads.clear() + assert transport.health()["b" * 24]["pending_count"] == 2 + assert set(reads) == {target_path} diff --git a/tests/test_native_steward_monitor.py b/tests/test_native_steward_monitor.py index 6051b0dffb..ba51e97a76 100644 --- a/tests/test_native_steward_monitor.py +++ b/tests/test_native_steward_monitor.py @@ -1247,3 +1247,128 @@ def test_fresh_scope_includes_a_new_stopped_goal_without_reusing_or_rewriting_mo assert not result["response"]["native_loopx"]["monitor"]["material_change"] assert capture.read_bytes() == calls and store.load_session(sid)["manager_authorization_scope_id"] == original_scope assert monitor.current(actor)["result_hash"] == first_hash + + +def test_signal_unavailable_monitor_recovers_once_and_does_not_starve_other_due_monitors( + steward, tmp_path, monkeypatch, +): + from test_steward_generic_monitor_todo import _set_signal_source_policy + + store, runtime, _, transport, binding, _, _ = steward + sid, actor, monitor = prepared(steward, tmp_path, monkeypatch, "sqlite") + session = store.load_session(sid) + _set_signal_source_policy(runtime, session, binding, goal_id=actor["goal_id"], agent_id=actor["agent_id"]) + + # Push the default portfolio watch into the future so our two custom monitors control due ordering. + default_watch = monitor.current(actor) + edit_local_watch( + runtime, + actor, + default_watch["todo_id"], + "--next-due-at", + (datetime.now(timezone.utc) + timedelta(hours=2)).isoformat(), + ) + + from loopx.capabilities.native_chat.steward_commands import native_command + + first_due = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat() + second_due = (datetime.now(timezone.utc) - timedelta(seconds=5)).isoformat() + added_first = native_command( + runtime, + [ + "todo", "add", "--goal-id", actor["goal_id"], "--role", "agent", + "--text", "Monitor open PR signals", + "--claimed-by", actor["agent_id"], + "--task-class", "continuous_monitor", + "--action-kind", "local_runtime_observe", + "--continuation-policy", "same_agent_non_delivery", + "--required-capability", "filesystem", + "--target-key", "steward-stalled-pr-monitor", + "--task-repository", "git:github.com/org/repo", + "--watch-only", + "--cadence", "30m", + "--next-due-at", first_due, + "--execute", + ], + ) + added_second = native_command( + runtime, + [ + "todo", "add", "--goal-id", actor["goal_id"], "--role", "agent", + "--text", "Monitor secondary inbox state", + "--claimed-by", actor["agent_id"], + "--task-class", "continuous_monitor", + "--action-kind", "local_runtime_observe", + "--continuation-policy", "same_agent_non_delivery", + "--required-capability", "filesystem", + "--target-key", "steward-secondary-watch", + "--watch-only", + "--cadence", "30m", + "--next-due-at", second_due, + "--execute", + ], + ) + pr_todo_id = added_first["todo_id"] + secondary_todo_id = added_second["todo_id"] + transport.reconcile() + + # 1. Fail due[0] inside _observe_github_repository_prs before any poll receipt. + calls = {"count": 0} + + def flaky_repo_observe(repo_slug, **_kwargs): + calls["count"] += 1 + if calls["count"] == 1: + return {"status": "unavailable", "repository": repo_slug} + return {"status": "ok", "repository": repo_slug, "items": []} + + monkeypatch.setattr( + "loopx.capabilities.native_chat.steward_monitor._observe_github_repository_prs", + flaky_repo_observe, + ) + runtime._steward_monitor_next_check.clear() + initial = pump_steward_monitors(runtime) + assert len(initial) == 1 and initial[0]["loopx_request"]["todo_id"] == pr_todo_id + failed = finish(runtime, initial[0]) + assert failed["status"] == "failed" + assert failed["error_code"] == "runtime_error" + assert failed["error"] == "steward monitor external signal source temporarily unavailable" + + # 2. If due[0] stays unavailable on recovery, it executes its ONE bounded retry and stops failing over itself. + # Replace flaky mock to fail a second time so we prove 1-retry cap AND non-starvation of due[1]! + monkeypatch.setattr( + "loopx.capabilities.native_chat.steward_monitor._observe_github_repository_prs", + lambda repo_slug, **_kw: {"status": "unavailable", "repository": repo_slug}, + ) + runtime._steward_monitor_next_check.clear() + recovery = pump_steward_monitors(runtime) + assert len(recovery) == 1 + assert recovery[0]["loopx_request"]["todo_id"] == pr_todo_id + assert recovery[0]["loopx_request"]["monitor_restart_recovery"]["original_turn_id"] == failed["turn_id"] + failed_again = finish(runtime, recovery[0]) + assert failed_again["status"] == "failed" + + # 3. Next pump skips the unrecoverable due[0] and dispatches due[1] instead of starving it! + runtime._steward_monitor_next_check.clear() + next_due = pump_steward_monitors(runtime) + assert len(next_due) == 1 and next_due[0]["loopx_request"]["todo_id"] == secondary_todo_id + done_secondary = finish(runtime, next_due[0]) + assert done_secondary["status"] == "completed", done_secondary + assert local_watch_row(runtime, actor, secondary_todo_id)["next_due_at"] != second_due + + # 4. Also verify recovery succeeds when GitHub recovers after the first failure. + third_due = (datetime.now(timezone.utc) - timedelta(seconds=3)).isoformat() + edit_local_watch(runtime, actor, pr_todo_id, "--next-due-at", third_due) + monkeypatch.setattr( + "loopx.capabilities.native_chat.steward_monitor._observe_github_repository_prs", + flaky_repo_observe, + ) + calls["count"] = 0 + runtime._steward_monitor_next_check.clear() + first_try = pump_steward_monitors(runtime) + assert len(first_try) == 1 and finish(runtime, first_try[0])["status"] == "failed" + runtime._steward_monitor_next_check.clear() + recovered_try = pump_steward_monitors(runtime) + assert len(recovered_try) == 1 + recovered_done = finish(runtime, recovered_try[0]) + assert recovered_done["status"] == "completed", recovered_done + assert local_watch_row(runtime, actor, pr_todo_id)["next_due_at"] != third_due diff --git a/tests/test_steward_pr_merge_and_review_signal.py b/tests/test_steward_pr_merge_and_review_signal.py index e8e96ba746..0d216334da 100644 --- a/tests/test_steward_pr_merge_and_review_signal.py +++ b/tests/test_steward_pr_merge_and_review_signal.py @@ -202,36 +202,120 @@ def test_summarize_pr_distinguishes_gpt_gemini_fallback_stale_casual_and_conflic assert conflicted["review_summary"]["effective_verdict"] == "REQUEST_CHANGES" +def _graphql_page( + nodes: list[dict[str, Any]], *, has_next_page: bool = False, end_cursor: str | None = None +) -> dict[str, Any]: + return { + "data": { + "repository": { + "pullRequests": { + "pageInfo": {"hasNextPage": has_next_page, "endCursor": end_cursor}, + "nodes": nodes, + } + } + } + } + + +def _graphql_node(row: dict[str, Any], *, contexts: list[dict[str, Any]] | None = None) -> dict[str, Any]: + copied = dict(row) + default_contexts = copied.pop("statusCheckRollup", []) + rollup_contexts = contexts if contexts is not None else default_contexts + copied["reviews"] = {"nodes": copied.get("reviews") or []} + copied["commits"] = { + "nodes": [ + { + "commit": { + "statusCheckRollup": { + "contexts": { + "nodes": rollup_contexts, + } + } + } + } + ] + } + return copied + + def test_observe_github_repository_prs_enriches_signal_summary(monkeypatch) -> None: - head = "c" * 40 + head_99 = "c" * 40 + head_101 = "1" * 40 gpt_decl = "model_agent · gpt-6.2-sol · OpenAI (runtime_reported, max)" - payload = [ + pr_99 = _graphql_node( _sample_pr( number=99, - head_oid=head, + head_oid=head_99, reviews=[ { "id": "PRR_99", "author": {"login": "loopx-agent"}, "state": "COMMENTED", - "commit": {"oid": head}, + "commit": {"oid": head_99}, "submittedAt": "2030-01-01T08:30:00Z", "body": _valid_bilingual_body( - head_oid=head, verdict="REQUEST_CHANGES", reviewer=gpt_decl, author_owned=True + head_oid=head_99, verdict="REQUEST_CHANGES", reviewer=gpt_decl, author_owned=True ), } ], ) - ] + ) + pr_101 = _graphql_node( + _sample_pr(number=101, head_oid=head_101), + contexts=[ + {"__typename": "CheckRun", "name": "ci", "status": "COMPLETED", "conclusion": "SUCCESS"}, + {"__typename": "StatusContext", "context": "gate", "state": "FAILURE"}, + ], + ) + seen_cursors: list[str | None] = [] def fake_run(cmd, **_kwargs): - assert cmd[:3] == ["gh", "pr", "list"] - return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + assert cmd[:3] == ["gh", "api", "graphql"] + cursor_args = [arg for arg in cmd if arg.startswith("cursor=")] + cursor = cursor_args[0].split("=", 1)[1] if cursor_args else None + seen_cursors.append(cursor) + if cursor is None: + body = _graphql_page([pr_101], has_next_page=True, end_cursor="cursor-page-2") + else: + assert cursor == "cursor-page-2" + # Include duplicate PR 101 across page shift + new PR 99 to test deduplication & sorting. + body = _graphql_page([pr_101, pr_99], has_next_page=False, end_cursor="cursor-end") + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(body), stderr="") monkeypatch.setattr("loopx.capabilities.native_chat.steward_monitor.subprocess.run", fake_run) observed = _observe_github_repository_prs("org/repo") - assert observed["status"] == "ok" and len(observed["items"]) == 1 + assert seen_cursors == [None, "cursor-page-2"] + assert observed["status"] == "ok" and [item["number"] for item in observed["items"]] == [99, 101] assert observed["items"][0]["review_summary"]["effective_verdict"] == "REQUEST_CHANGES" + assert observed["items"][1]["checks_summary"] == { + "total": 2, + "pass_count": 1, + "fail_count": 1, + "pending_count": 0, + } + + +def test_observe_github_repository_prs_fails_closed_on_page_error(monkeypatch) -> None: + calls = 0 + + def fake_run(cmd, **_kwargs): + nonlocal calls + calls += 1 + if calls == 1: + body = _graphql_page([_graphql_node(_sample_pr(number=10))], has_next_page=True, end_cursor="p2") + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(body), stderr="") + return subprocess.CompletedProcess(cmd, 1, stdout="", stderr="502 Bad Gateway") + + monkeypatch.setattr("loopx.capabilities.native_chat.steward_monitor.subprocess.run", fake_run) + observed = _observe_github_repository_prs("org/repo") + assert observed == {"status": "unavailable", "repository": "org/repo"} + assert calls == 2 + + def graphql_errors_run(cmd, **_kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps({"errors": [{"message": "timeout"}]}), stderr="") + + monkeypatch.setattr("loopx.capabilities.native_chat.steward_monitor.subprocess.run", graphql_errors_run) + assert _observe_github_repository_prs("org/repo") == {"status": "unavailable", "repository": "org/repo"} def test_inspect_github_pull_request_attaches_rest_html_url() -> None: