Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 109 additions & 9 deletions loopx/capabilities/native_chat/external_conversations.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import hashlib
import json
import re
import threading
from datetime import datetime as datetime, timezone as timezone
from typing import Any

Expand All @@ -21,7 +22,12 @@
ensure_external_admission_scope,
resolve_external_admission_source,
)
from ...chat_store import _atomic_write_json as _atomic_write_json, _read_json as _read_json, RESUMABLE_SESSION_STATES
from ...chat_store import (
ChatSessionStore,
_atomic_write_json as _atomic_write_json,
_read_json as _read_json,
RESUMABLE_SESSION_STATES,
)
from ...chat_attachments import normalize_chat_image_attachments
from ...control_plane import effect_runtime
from ...control_plane.quota.automation_cadence_readback import (
Expand All @@ -31,7 +37,14 @@
from ...file_lock import exclusive_file_lock as exclusive_file_lock, try_exclusive_file_lock
from . import result_review_evidence
from .codex_context import checked_session_home
from .file_discovery import FileChatDiscovery, external_source_route_path, later_equivalent_source_input, recovery_journal_fields, source_control_fields
from .file_discovery import (
FileChatDiscovery,
external_source_route_path,
file_change_fingerprint,
later_equivalent_source_input,
recovery_journal_fields,
source_control_fields,
)
from .steward_commands import ExternalStewardCommissionMixin

# Request journals are named by their 24-hex request_ref; lock metadata and
Expand All @@ -55,6 +68,24 @@ def _preparation_image_identity(value: Any) -> dict[str, Any] | None:
return None


def _project_source_request_digest(_path: Path, item: dict[str, Any]) -> dict[str, Any]:
canonical = json.dumps(item, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
return {
"request_ref": item.get("request_ref"),
"binding_id": item.get("binding_id"),
"source": item.get("source"),
"sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(),
}


def _project_user_message_hint(message: Mapping[str, Any]) -> dict[str, Any]:
return {
key: message[key]
for key in ("role", "turn_id", "text", "origin", "created_at")
if key in message
}


class _RecoveryDiscoveryReads:
"""Lazy facts for one queue-discovery pass, never recovery admission.

Expand All @@ -77,8 +108,7 @@ def queued(self, session_id: str) -> bool:

def user_messages(self, session_id: str) -> list[dict[str, Any]]:
if session_id not in self.histories:
self.histories[session_id] = [message for message in self.core.controller.store.messages(session_id)
if message.get("role") == "user"]
self.histories[session_id] = list(self.core._cached_user_messages(session_id))
return self.histories[session_id]

def source_controls(self) -> list[dict[str, Any]]:
Expand All @@ -102,8 +132,11 @@ def __init__(self, controller: Any) -> None:
self.bindings.controller = controller
self.root = controller.store.root / "external-requests"
self._source_control_discovery = FileChatDiscovery()
self._source_request_discovery = FileChatDiscovery()
self._backlog_request_discovery = FileChatDiscovery()
self._recovery_discovery = FileChatDiscovery()
self._user_messages_lock = threading.Lock()
self._session_user_messages_cache: dict[str, tuple[tuple[int, ...], tuple[dict[str, Any], ...]]] = {}
self.actions: Any | None = None

def _retained_result_review(self, row: dict[str, Any]) -> dict[str, Any] | None:
Expand Down Expand Up @@ -270,6 +303,73 @@ def _source_requests(self, row: dict[str, Any]) -> list[dict[str, Any]]:
return [item for item in self.pending()
if item.get("binding_id") == row["binding_id"] and item.get("source") == row["source"]]

def _source_request_digests(self, row: dict[str, Any]) -> list[dict[str, Any]]:
discovery = getattr(self, "_source_request_discovery", None)
if discovery is None:
discovery = FileChatDiscovery()
self._source_request_discovery = discovery
binding_id, source = row["binding_id"], row["source"]
return [
item for _, item in discovery.rows(
self.root, self._request_paths(), read=_read_json, project=_project_source_request_digest,
)
if item.get("binding_id") == binding_id and item.get("source") == source
]

def _cached_user_messages(self, session_id: str) -> tuple[dict[str, Any], ...]:
store = self.controller.store
is_real_store = (
isinstance(store, ChatSessionStore)
and getattr(store.messages, "__func__", None) is ChatSessionStore.messages
)
if not is_real_store:
return tuple(
_project_user_message_hint(message)
for message in store.messages(session_id)
if message.get("role") == "user"
)
try:
msg_path = store._session_dir(session_id) / "messages.jsonl"
before = file_change_fingerprint(msg_path)
except (OSError, ValueError):
before = None
msg_path = None
lock = getattr(self, "_user_messages_lock", None)
if lock is None:
lock = threading.Lock()
self._user_messages_lock = lock
cache_map = getattr(self, "_session_user_messages_cache", None)
if cache_map is None:
cache_map = {}
self._session_user_messages_cache = cache_map
if before is None:
with lock:
cache_map.pop(session_id, None)
if msg_path is None:
return ()
return tuple(
_project_user_message_hint(message)
for message in store.messages(session_id)
if message.get("role") == "user"
)
with lock:
saved = cache_map.get(session_id)
if saved is not None and saved[0] == before:
return saved[1]
rows = tuple(
_project_user_message_hint(message)
for message in store.messages(session_id)
if message.get("role") == "user"
)
try:
unchanged = msg_path is not None and file_change_fingerprint(msg_path) == before
except OSError:
unchanged = False
if unchanged:
with lock:
cache_map[session_id] = (before, rows)
return rows

def _codex_quota_route_capacity(self, session: dict[str, Any]) -> dict[str, Any] | None:
from ...chat_manager import manager_session_quota_route
route = manager_session_quota_route(self.controller, session)
Expand Down Expand Up @@ -462,7 +562,7 @@ def recover_failed_request(self, request_ref: str, *,
else checked_session_home(self.controller.codex_home, session))
context = self.controller.project_contexts.session_context(session)

requests = self._source_requests(row)
requests = self._source_request_digests(row)
# The read-only native client can consume its entire RPC deadline.
# Retain this request's fence, but let new source input/control in.
# Its observation is only usable if the original local state still
Expand Down Expand Up @@ -564,7 +664,7 @@ def recover_failed_request(self, request_ref: str, *,
or self.controller.store.load_turn(row["session_id"], current_turn["turn_id"]) != current_turn
or (plan.get("completed_review_turn_id") or plan.get("original_result_turn_id"))
and self.controller.store.load_turn(row["session_id"], turn["turn_id"]) != turn
or self._source_requests(row) != requests):
or self._source_request_digests(row) != requests):
return row
plan = self._context_recovery_plan(row, observation=proof, source_verified=source is not None,
selected_binding=selected["binding"], preparation_scope_available=scope_available,
Expand Down Expand Up @@ -725,7 +825,7 @@ def _recover_native_result(self, path: Path, row: dict[str, Any], session: dict[
turn: dict[str, Any], source_observation: Callable) -> dict[str, Any]:
"""Adopt an original provider result, without reopening/dispatching a Turn."""
store = self.controller.store
requests = self._source_requests(row)
requests = self._source_request_digests(row)
context = self.controller.project_contexts.session_context(session)
recovered = row.get("native_result_recovery")
proof = self._original_native_result(row, session, turn, context) if recovered is None else recovered.get("native_observation")
Expand All @@ -741,7 +841,7 @@ def _recover_native_result(self, path: Path, row: dict[str, Any], session: dict[
selected = self.bindings._resolve_observed(current, selected["binding"], selected["observation"], row["source"])
if (store.load_session(session["session_id"]) != session
or store.load_turn(session["session_id"], turn["turn_id"]) != turn
or self._source_requests(row) != requests):
or self._source_request_digests(row) != requests):
return row
plan = self._context_recovery_plan(row, observation=proof, source_verified=source is not None,
selected_binding=selected["binding"])
Expand Down Expand Up @@ -973,7 +1073,7 @@ def _checked_metadata_review(self, session: dict[str, Any], turn: dict[str, Any]
error_message="The original source, grant or queue changed; metadata review is held.",
require_empty_queue=True,
)
refs = sorted(item["request_ref"] for item in self._source_requests(row))
refs = sorted(item["request_ref"] for item in self._source_request_digests(row))
native_session = self._original_native_session(current, [continuation["native_observation"]])
if (native_session.get("agent_id") != "antigravity"
or native_session.get("upstream_thread_id") != continuation["native_observation"]["thread_id"]
Expand Down
87 changes: 71 additions & 16 deletions loopx/extensions/lark/private_conversations.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
STEWARD_BINDING_ADOPTION_PENDING,
)
from ...capabilities.native_chat.file_discovery import (
REPLY_IMAGE_NAME_PATTERN, FileChatDiscovery, reply_image_workspace,
REPLY_IMAGE_NAME_PATTERN, FileChatDiscovery, file_change_fingerprint, reply_image_workspace,
)
from ...chat_attachments import CHAT_IMAGE_MAX_COUNT
from ...chat_store import TERMINAL_TURN_STATES, _atomic_write_json, _read_json
Expand Down Expand Up @@ -123,6 +123,7 @@ def __init__(self, *, controller: Any, runtime_root: Path, runner: Any, cli_bin:
self.steward_callback_streams: dict[tuple[str, str], TeamPlanReviewCallbackStream] = {}
self.root = controller.store.root / "lark-private-deliveries"
self._context_recovery_probe_at: dict[Path, float] = {}
self._context_recovery_noop_backoff: dict[Path, tuple[tuple[int, ...] | None, tuple[int, ...] | None, int]] = {}
self._historical_image_probe_at: dict[Path, float] = {}
self._planned_repairs_stamp: tuple[bool, int | None] | None = None
self._planned_repairs_names: frozenset[str] = frozenset()
Expand Down Expand Up @@ -1555,13 +1556,54 @@ def pending_body_paths(self) -> list[Path]:
if row["status"] != "delivered" and triage.pending(row) and not _is_answer_image_followup(row)]
return [*paths, *self._pending_web_projection_paths()]

def _noop_backoff_cache(self) -> dict[Path, tuple[tuple[int, ...] | None, tuple[int, ...] | None, int]]:
cache = getattr(self, "_context_recovery_noop_backoff", None)
if cache is None:
cache = {}
self._context_recovery_noop_backoff = cache
return cache

def _context_recovery_fingerprints(
self, path: Path, request_ref: str,
) -> tuple[tuple[int, ...] | None, tuple[int, ...] | None]:
core_root = getattr(self.core, "root", None)
native_fp: tuple[int, ...] | None = None
if isinstance(core_root, Path):
try:
native_fp = file_change_fingerprint(core_root / f"{request_ref}.json")
except OSError:
native_fp = None
try:
delivery_fp = file_change_fingerprint(path)
except OSError:
delivery_fp = None
return delivery_fp, native_fp

def _clear_context_recovery_noop_backoff(self, path: Path) -> None:
self._noop_backoff_cache().pop(path, None)

def _record_context_recovery_noop_backoff(self, path: Path, request_ref: str) -> None:
cache = self._noop_backoff_cache()
delivery_fp, native_fp = self._context_recovery_fingerprints(path, request_ref)
prev = cache.get(path)
streak = prev[2] + 1 if prev is not None and (prev[0], prev[1]) == (delivery_fp, native_fp) else 1
cache[path] = (delivery_fp, native_fp, streak)
if streak <= 1:
delay = 60.0
else:
base = min(1800, 60 * (2 ** min(streak - 1, 5)))
jitter = zlib.crc32(path.name.encode("utf-8")) % 60
delay = float(base + jitter)
self._context_recovery_probe_at[path] = time.monotonic() + delay

@effect_runtime_request_scope()
def pending_delivery_paths(self) -> list[Path]:
"""The durable transport store is the queue; no in-memory admission."""
paths = []
recovery_candidate = self.core.context_recovery_observer()
now = datetime.now(timezone.utc).timestamp()
planned_repairs = self._planned_historical_repairs()
noop_backoff = self._noop_backoff_cache()
for path, row, repairs in self._delivery_discovery.rows():
if row["status"] != "delivered":
if triage.pending(row) and not _answer_image_retry_waits(row, now):
Expand All @@ -1571,20 +1613,25 @@ def pending_delivery_paths(self) -> list[Path]:
elif (not row.get("historical_images_admitted") and path.name in planned_repairs
and time.monotonic() >= self._historical_image_probe_at.get(path, 0)):
paths.append(path)
elif (row.get("session_id") and row.get("turn_id")
and time.monotonic() >= self._context_recovery_probe_at.get(path, 0)):
# Only a discovery/backoff hint, never execution authority.
# The canonical request, provider effect inventory and source
# are independently re-read by recover_failed_request.
# Stagger steady-state ineligible re-probes across [60, 120)s by filename
# hash so cold-start bursts do not stay synchronized on the same tick,
# while keeping eligible candidates on the exact 60s retry cadence.
now_mono = time.monotonic()
stagger = zlib.crc32(path.name.encode("utf-8")) % 60
self._context_recovery_probe_at[path] = now_mono + 60 + stagger
if repairs or recovery_candidate(row["request_ref"]):
self._context_recovery_probe_at[path] = now_mono + 60
paths.append(path)
elif row.get("session_id") and row.get("turn_id"):
saved_backoff = noop_backoff.get(path)
if (saved_backoff is not None
and self._context_recovery_fingerprints(path, row["request_ref"]) != saved_backoff[:2]):
noop_backoff.pop(path, None)
self._context_recovery_probe_at.pop(path, None)
if time.monotonic() >= self._context_recovery_probe_at.get(path, 0):
# Only a discovery/backoff hint, never execution authority.
# The canonical request, provider effect inventory and source
# are independently re-read by recover_failed_request.
# Stagger steady-state ineligible re-probes across [60, 120)s by filename
# hash so cold-start bursts do not stay synchronized on the same tick,
# while keeping eligible candidates on the exact 60s retry cadence.
now_mono = time.monotonic()
stagger = zlib.crc32(path.name.encode("utf-8")) % 60
self._context_recovery_probe_at[path] = now_mono + 60 + stagger
if repairs or recovery_candidate(row["request_ref"]):
self._context_recovery_probe_at[path] = now_mono + 60
paths.append(path)
return [*paths, *self._web_projection_paths()]

def is_recovery_path(self, path: Path) -> bool:
Expand Down Expand Up @@ -1721,16 +1768,24 @@ def reconcile_request(self, path: Path) -> int:
source_observation=lambda selected: self._recovery_source_observation(row, selected))
if (row["status"] == "delivered" and recovered.get("native_result_recovery")
and recovered["native_result_recovery"]["turn_id"] == recovered.get("turn_id")):
return self._reconcile_native_result(path, recovered)
delivered_native = self._reconcile_native_result(path, recovered)
if delivered_native:
self._clear_context_recovery_noop_backoff(path)
else:
self._record_context_recovery_noop_backoff(path, row["request_ref"])
return delivered_native
if (recovered.get("context_recovery") and not recovered.get("delivery_verified")
and recovered.get("turn_id") != row["turn_id"]):
self._clear_context_recovery_noop_backoff(path)
with exclusive_file_lock(path, operation="deliver_private_chat_request"):
row = _read_json(path)
if not _same_fields(row, recovered, "request_ref", "binding_id", "source", "session_id"):
raise ValueError("original recovery source changed")
row.update(status="accepted", turn_id=recovered["turn_id"],
context_recovery=recovered["context_recovery"])
_atomic_write_json(path, row)
elif row["status"] == "delivered":
self._record_context_recovery_noop_backoff(path, row["request_ref"])
if row["status"] == "captured":
# Intake may still own this capture. Wait on its existing fence and
# read its result before retrying admission from a stale snapshot.
Expand Down
6 changes: 6 additions & 0 deletions loopx/extensions/lark/private_delivery_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,12 @@ def _observe(self, path: Path) -> tuple[dict[str, Any], bool]:
self._completed.pop(path, None)
row = _read_json(path)
if row["status"] != "delivered":
triage = row.get("triage")
if isinstance(triage, dict) and triage.get("retired"):
view = {"status": row["status"], "triage": {"retired": triage["retired"]}}
if before is not None and file_change_fingerprint(path) == before:
self._completed[path] = (before, view, False)
return view, False
return row, False

repairs = bool(row.get("session_id") and row.get("turn_id") and legacy_recovery_repairs(row))
Expand Down
2 changes: 1 addition & 1 deletion tests/test_lark_private_conversations.py
Original file line number Diff line number Diff line change
Expand Up @@ -8178,7 +8178,7 @@ def observed(path):
assert "private-body-marker" not in repr(transport._delivery_discovery._completed)
_atomic_write_json(new, {"status": "pending"})
assert transport.pending_delivery_paths() == [new, pending]
clock[0] += 61
clock[0] += 120 # Ineligible recovery probes stagger across [60, 120)s.
eligible["old"] = True
assert transport.pending_delivery_paths() == [new, old, pending]
clock[0] += 61
Expand Down
Loading
Loading