From 61849c78945267ef457bf9e5e22a0c9ac0a96788 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:26:40 +0800 Subject: [PATCH] perf(lark,native_chat): coalesce listener snapshots and cache recovery event summaries Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../native_chat/external_conversations.py | 107 ++++-- .../native_chat/file_discovery.py | 124 +++++-- .../lark/goal_topic_runtime_service.py | 50 ++- .../extensions/lark/private_conversations.py | 15 +- .../test_lark_steward_directory_scan_perf.py | 316 ++++++++++++++++++ 5 files changed, 548 insertions(+), 64 deletions(-) diff --git a/loopx/capabilities/native_chat/external_conversations.py b/loopx/capabilities/native_chat/external_conversations.py index 45b3230642..4cb2488e9a 100644 --- a/loopx/capabilities/native_chat/external_conversations.py +++ b/loopx/capabilities/native_chat/external_conversations.py @@ -40,10 +40,12 @@ from .file_discovery import ( FileChatDiscovery, QuotaBacklogObservationCache, + _same_store_method, external_source_route_path, file_change_fingerprint, later_equivalent_source_input, observe_quota_backlog_batch, + observed_turn_event_summary, recovery_journal_fields, source_control_fields, ) @@ -56,6 +58,23 @@ _QUOTA_BACKLOG_BATCH_BYTES = 1024 * 1024 +class _DeferredBacklogRequest(dict[str, Any]): + """Bounded backlog request view carrying its observed file fingerprint.""" + + __slots__ = ("req_fp", "needs_full_read") + + def __init__( + self, + data: Mapping[str, Any], + *, + req_fp: tuple[int, ...] | None, + needs_full_read: bool, + ) -> None: + super().__init__(data) + self.req_fp = req_fp + self.needs_full_read = needs_full_read + + def _preparation_image_identity(value: Any) -> dict[str, Any] | None: """Fingerprint already normalized Core images without exporting their bytes.""" if not isinstance(value, list) or not value: @@ -142,6 +161,13 @@ def __init__(self, controller: Any) -> None: self._quota_backlog_cache = QuotaBacklogObservationCache() self.actions: Any | None = None + def _event_summary_cache(self) -> QuotaBacklogObservationCache: + cache_store = getattr(self, "_quota_backlog_cache", None) + if cache_store is None: + cache_store = QuotaBacklogObservationCache() + self._quota_backlog_cache = cache_store + return cache_store + def _retained_result_review(self, row: dict[str, Any]) -> dict[str, Any] | None: """Read correlation only; the typed owner still checks continuation eligibility.""" reservation = row.get("context_recovery") or {} @@ -198,7 +224,19 @@ def _context_recovery_plan(self, row: dict[str, Any], *, and reservation.get("failed_turn_id") == turn["turn_id"] and (retained_review is None or reservation.get("work_attempt") == 1 or reservation.get("legacy_result_review")) else None) - events, anchors = self._started_native_inputs(session["session_id"], turn["turn_id"]) + image_failure = turn.get("error_code") == "runtime_error" and bool(row.get("attachments")) + runtime_failure = turn.get("error_code") in {"model_auth_unavailable", "resume_failed"} + needs_full_events = (not discovery_only or retained_review is not None + or turn.get("error_code") == "manager_authorization_unavailable" + or image_failure or runtime_failure) + if needs_full_events: + events, anchors = self._started_native_inputs(session["session_id"], turn["turn_id"]) + steering_observed = interrupt_observed = False + else: + single_anchor, steering_observed, interrupt_observed = observed_turn_event_summary( + store, session["session_id"], turn["turn_id"], self._event_summary_cache(), + real_events=_same_store_method(store, "events_after", ChatSessionStore)) + events, anchors = [], ([single_anchor] if single_anchor is not None else []) session = self._original_native_session(session, anchors) unfinished = turn.get("status") == "completed" and bool((turn.get("response") or {}).get("unfinished_work")) related_review = (result_review_evidence.collect_related_result_review(self, row, turn, discovery_reads=discovery_reads) @@ -221,8 +259,6 @@ def _context_recovery_plan(self, row: dict[str, Any], *, later_equivalent = None read_source_controls = (discovery_reads.source_controls if discovery_reads is not None else cache(self.source_controls)) - image_failure = turn.get("error_code") == "runtime_error" and bool(row.get("attachments")) - runtime_failure = turn.get("error_code") in {"model_auth_unavailable", "resume_failed"} if turn.get("error_code") == "manager_authorization_unavailable" or image_failure or runtime_failure: # Collect canonical facts only. The typed planner owns the exact # pre-dispatch sequence, retry budget and authority decision. @@ -275,9 +311,11 @@ def _context_recovery_plan(self, row: dict[str, Any], *, "queued_input_present": (discovery_reads.queued(session["session_id"]) if discovery_reads is not None else bool(store.queued_turns(session["session_id"]))), "attachments_present": bool(row.get("attachments")), - "steering_observed": any(event.get("kind") == "turn.steered" for event in events), - "stop_observed": self._source_stop_observed(row, turn, events, - source_controls=read_source_controls), + "steering_observed": (any(event.get("kind") == "turn.steered" for event in events) + if needs_full_events else steering_observed), + "stop_observed": (self._source_stop_observed(row, turn, events, source_controls=read_source_controls) + if needs_full_events else (interrupt_observed + or self._source_stop_observed(row, turn, (), source_controls=read_source_controls))), "preparation_observation": preparation, "later_equivalent_input_present": later_equivalent, "preparation_scope_available": preparation_scope_available, @@ -398,7 +436,7 @@ def _original_native_session(session: dict[str, Any], anchors: list[dict[str, An "upstream_thread_id": matches[0]["thread_id"]} def _source_stop_observed(self, row: dict[str, Any], turn: dict[str, Any], - events: list[dict[str, Any]], *, + events: Sequence[dict[str, Any]], *, source_controls: Callable[[], list[dict[str, Any]]] | None = None) -> bool: """Read the same durable source controls for recovery and display.""" return (any(event.get("kind") in {"turn.interrupt_requested", "turn.interrupted"} for event in events) @@ -410,9 +448,11 @@ def _source_stop_observed(self, row: dict[str, Any], turn: dict[str, Any], def quota_backlog_observer( self, *, source_controls: Callable[[], list[dict[str, Any]]] | None = None, use_cache: bool = False, + observed_fps: Mapping[Path, tuple[int, ...]] | None = None, ) -> Callable[[Sequence[dict[str, Any]]], list[bool]]: """Read Session state after each batch's Turns and share source controls across batches.""" - read_controls = cache(source_controls or self.source_controls) + read_controls = cache(source_controls or (self.source_controls if observed_fps is None + else (lambda: self.source_controls(known_fingerprints=observed_fps)))) @cache def stop_controls() -> tuple[tuple[Any, Any, datetime], ...]: @@ -425,12 +465,7 @@ def stop_controls() -> tuple[tuple[Any, Any, datetime], ...]: def observe(rows: Sequence[dict[str, Any]]) -> list[bool]: if len(rows) > QUOTA_BACKLOG_BATCH_SIZE: raise ValueError("select at most 64 pending requests per observation") - cache_store = None - if use_cache: - cache_store = getattr(self, "_quota_backlog_cache", None) - if cache_store is None: - cache_store = QuotaBacklogObservationCache() - self._quota_backlog_cache = cache_store + cache_store = self._event_summary_cache() if use_cache else None return observe_quota_backlog_batch( root=self.root, store=self.controller.store, rows=rows, build_inputs=self._quota_pending_inputs, eval_flags=self._quota_pending_flags, @@ -477,6 +512,8 @@ def flush() -> None: return flags def _quota_pending_inputs(self, row: dict[str, Any]) -> dict[str, Any]: + if getattr(row, "needs_full_read", False): + row = self.read_request(row["request_ref"]) turn = self.controller.store.load_turn(row["session_id"], row["turn_id"]) return { "request": {key: value for key, value in row.items() if key != "attachments"}, @@ -1848,28 +1885,52 @@ def _recovery_rows(self) -> list[dict[str, Any]]: return [r for _, r in self._recovery_discovery.rows( self.root, self._request_paths(), read=_read_json, project=recovery_journal_fields)] - def source_controls(self) -> list[dict[str, Any]]: + def source_controls(self, *, known_fingerprints: Mapping[Path, tuple[int, ...]] | None = None) -> list[dict[str, Any]]: """Observe source controls and private text digests without reparsing unchanged bodies.""" - return [r for _, r in self._source_control_discovery.rows( - self.root, self._request_paths(), read=_read_json, project=source_control_fields)] + return [r for _, r, _ in self._source_control_discovery.rows_with_fingerprints( + self.root, self._request_paths(), read=_read_json, project=source_control_fields, + known_fingerprints=known_fingerprints)] def read_request(self, request_ref: str) -> dict[str, Any]: return _read_json(self.root / f"{request_ref}.json") - def read_backlog_requests(self, request_refs: Iterable[str]) -> Iterator[dict[str, Any]]: + def read_backlog_requests( + self, request_refs: Iterable[str], *, defer_recovery: bool = False, + observed_fps: dict[Path, tuple[int, ...]] | None = None, + ) -> Iterator[dict[str, Any]]: """Observe settings correlation, without retaining ordinary request bodies.""" fields = ("request_ref", "binding_id", "source", "session_id", "turn_id", "status", "command", "created_at") + fresh: dict[Path, dict[str, Any]] = {} + + def read_current(path: Path) -> dict[str, Any]: + payload = _read_json(path) + if not defer_recovery: + fresh[path] = payload + return payload def project(_path: Path, row: dict[str, Any]) -> dict[str, Any] | None: - # Recovery evidence is consumed by the typed pending rule. - if any(row.get(k) is not None for k in ("context_recovery", "native_result_recovery")): + if not all(key in row for key in fields): return None - return {key: row[key] for key in fields} if all(key in row for key in fields) else None + return { + **{key: row[key] for key in fields}, + "_has_recovery": any(row.get(k) is not None for k in ("context_recovery", "native_result_recovery")), + } paths = (self.root / f"{ref}.json" for ref in request_refs) - for _, row in self._backlog_request_discovery.rows(self.root, paths, read=_read_json, project=project): - yield row + for path, row, fp in self._backlog_request_discovery.rows_with_fingerprints( + self.root, paths, read=read_current, project=project, + ): + payload = fresh.pop(path, None) + is_recovery = bool(row.pop("_has_recovery", False)) + if observed_fps is not None and fp is not None: + observed_fps[path] = fp + if is_recovery and not defer_recovery: + yield payload if payload is not None else _read_json(path) + elif defer_recovery: + yield _DeferredBacklogRequest(row, req_fp=fp, needs_full_read=is_recovery) + else: + yield row def record_delivery(self, request_ref: str, *, session_id: str | None, turn_id: str | None, message_text: str | None = None) -> None: diff --git a/loopx/capabilities/native_chat/file_discovery.py b/loopx/capabilities/native_chat/file_discovery.py index e4ef53610a..39bc49ae3f 100644 --- a/loopx/capabilities/native_chat/file_discovery.py +++ b/loopx/capabilities/native_chat/file_discovery.py @@ -1,7 +1,7 @@ """File-backed Chat journal discovery and bounded reply image artifacts.""" from __future__ import annotations -from collections.abc import Callable, Iterable, Iterator, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from copy import deepcopy from datetime import datetime import base64 @@ -152,7 +152,7 @@ def __init__(self, *, max_entries: int = 8192) -> None: self._max_entries = max(1, max_entries) self._base: dict[str, tuple[tuple[Any, ...], tuple[bool, str | None]]] = {} self._stopped: dict[str, tuple[tuple[Any, ...], bool]] = {} - self._interrupt_events: dict[tuple[str, str], tuple[tuple[int, ...], bool]] = {} + self._event_summaries: dict[tuple[str, str], tuple[tuple[int, ...], tuple[dict[str, Any] | None, bool, bool]]] = {} def get_base(self, request_ref: str, key: tuple[Any, ...]) -> tuple[bool, str | None] | None: with self._lock: @@ -188,11 +188,12 @@ def drop_request(self, request_ref: str) -> None: @staticmethod def build_key_prefix( root: Path, store: Any, row: dict[str, Any], evaluator_id: int, + *, req_fp_override: tuple[int, ...] | None = None, ) -> tuple[Any, Path | None, Path | None, tuple[int, ...] | None, tuple[int, ...] | None]: try: req_path = root / f"{row['request_ref']}.json" turn_path = store._turn_path(row["session_id"], row["turn_id"]) - req_fp = optional_file_fingerprint(req_path) + req_fp = req_fp_override if isinstance(req_fp_override, tuple) else optional_file_fingerprint(req_path) turn_fp = optional_file_fingerprint(turn_path) except (OSError, ValueError): return None, None, None, None, None @@ -215,37 +216,68 @@ def commit(self, request_ref: str, meta_entry: tuple[Any, ...], pending: bool, * else: self.put_base(request_ref, key, pending, turn_status) - def get_interrupt(self, session_id: str, turn_id: str, events_fp: tuple[int, ...]) -> bool | None: + def get_event_summary( + self, session_id: str, turn_id: str, events_fp: tuple[int, ...], + ) -> tuple[dict[str, Any] | None, bool, bool] | None: + with self._lock: + entry = self._event_summaries.get((session_id, turn_id)) + if entry is None or entry[0] != events_fp: + return None + anchor, steering, interrupted = entry[1] + return (deepcopy(anchor) if anchor is not None else None), steering, interrupted + + def put_event_summary( + self, session_id: str, turn_id: str, events_fp: tuple[int, ...], + summary: tuple[dict[str, Any] | None, bool, bool], + ) -> None: + anchor, steering, interrupted = summary + stored = (deepcopy(anchor) if anchor is not None else None), steering, interrupted with self._lock: - entry = self._interrupt_events.get((session_id, turn_id)) - return entry[1] if entry is not None and entry[0] == events_fp else None + _set_bounded_cache(self._event_summaries, (session_id, turn_id), (events_fp, stored), self._max_entries) - def put_interrupt(self, session_id: str, turn_id: str, events_fp: tuple[int, ...], observed: bool) -> None: + def drop_event_summary(self, session_id: str, turn_id: str) -> None: with self._lock: - _set_bounded_cache(self._interrupt_events, (session_id, turn_id), (events_fp, observed), self._max_entries) + self._event_summaries.pop((session_id, turn_id), None) + + def get_interrupt(self, session_id: str, turn_id: str, events_fp: tuple[int, ...]) -> bool | None: + summary = self.get_event_summary(session_id, turn_id, events_fp) + return summary[2] if summary is not None else None + + def put_interrupt(self, session_id: str, turn_id: str, events_fp: tuple[int, ...], observed: bool) -> None: + self.put_event_summary(session_id, turn_id, events_fp, (None, False, observed)) def drop_interrupt(self, session_id: str, turn_id: str) -> None: - with self._lock: - self._interrupt_events.pop((session_id, turn_id), None) + self.drop_event_summary(session_id, turn_id) -def observed_turn_interrupt( +def _summarize_turn_events(events: Sequence[dict[str, Any]]) -> tuple[dict[str, Any] | None, bool, bool]: + anchors = [ + item["payload"]["native_input"] + for item in events + if item.get("kind") == "turn.started" and item.get("payload", {}).get("native_input") + ] + single_anchor = deepcopy(anchors[0]) if len(anchors) == 1 else None + steering = any(e.get("kind") == "turn.steered" for e in events) + interrupted = any(e.get("kind") in {"turn.interrupt_requested", "turn.interrupted"} for e in events) + return single_anchor, steering, interrupted + + +def observed_turn_event_summary( store: Any, session_id: str, turn_id: str, cache_store: QuotaBacklogObservationCache | None, *, real_events: bool, -) -> bool: - """Check Turn interrupt events with fingerprint/pending-buffer guard on real stores.""" - interrupt_kinds = {"turn.interrupt_requested", "turn.interrupted"} +) -> tuple[dict[str, Any] | None, bool, bool]: + """Read compact Turn event summary with fingerprint/pending-buffer guard on real stores.""" if not real_events or cache_store is None: - return any(e.get("kind") in interrupt_kinds for e in store.events_after(session_id, turn_id, None)) + return _summarize_turn_events(store.events_after(session_id, turn_id, None)) key = (session_id, turn_id) try: event_path = store._event_path(session_id, turn_id) except ValueError: - return any(e.get("kind") in interrupt_kinds for e in store.events_after(session_id, turn_id, None)) + return _summarize_turn_events(store.events_after(session_id, turn_id, None)) flush_lock = store._event_flush_lock(key) fp_before: tuple[int, ...] | None = None if flush_lock.acquire(blocking=False): @@ -255,24 +287,38 @@ def observed_turn_interrupt( if not has_pending: fp_before = optional_file_fingerprint(event_path) if fp_before is not None: - cached = cache_store.get_interrupt(session_id, turn_id, fp_before) + cached = cache_store.get_event_summary(session_id, turn_id, fp_before) if cached is not None: return cached finally: flush_lock.release() - observed = any(e.get("kind") in interrupt_kinds for e in store.events_after(session_id, turn_id, None)) + summary = _summarize_turn_events(store.events_after(session_id, turn_id, None)) if fp_before is not None and flush_lock.acquire(blocking=False): try: with store._event_lock: has_pending_after = bool(store._event_pending.get(key)) fp_after = None if has_pending_after else optional_file_fingerprint(event_path) if fp_after == fp_before: - cache_store.put_interrupt(session_id, turn_id, fp_after, observed) + cache_store.put_event_summary(session_id, turn_id, fp_after, summary) else: - cache_store.drop_interrupt(session_id, turn_id) + cache_store.drop_event_summary(session_id, turn_id) finally: flush_lock.release() - return observed + return summary + + +def observed_turn_interrupt( + store: Any, + session_id: str, + turn_id: str, + cache_store: QuotaBacklogObservationCache | None, + *, + real_events: bool, +) -> bool: + """Check Turn interrupt events with fingerprint/pending-buffer guard on real stores.""" + return observed_turn_event_summary( + store, session_id, turn_id, cache_store, real_events=real_events, + )[2] def _same_store_method(store: Any, name: str, expected_cls: type[Any]) -> bool: @@ -300,7 +346,7 @@ def observe_quota_backlog_batch( params: list[dict[str, Any] | None] = [None] * len(rows) meta: list[tuple[Any, ...]] = [(None, None, None, None, None)] * len(rows) for index, row in enumerate(rows): - entry = (cache_store.build_key_prefix(root, store, row, evaluator_id) + entry = (cache_store.build_key_prefix(root, store, row, evaluator_id, req_fp_override=getattr(row, "req_fp", None)) if real_load_turn and cache_store is not None else (None, None, None, None, None)) meta[index] = entry if entry[0] is not None and cache_store is not None and cache_store.has_base_prefix(row["request_ref"], entry[0]): @@ -381,26 +427,33 @@ def __init__(self) -> None: self._lock = Lock() self._directories: dict[Path, dict[Path, tuple[tuple[int, ...], dict[str, Any]]]] = {} - def rows(self, directory: Path, paths: Iterable[Path], *, - read: Callable[[Path], dict[str, Any]], - project: Callable[[Path, dict[str, Any]], dict[str, Any] | None], - ) -> Iterator[tuple[Path, dict[str, Any]]]: + def rows_with_fingerprints( + self, + directory: Path, + paths: Iterable[Path], + *, + read: Callable[[Path], dict[str, Any]], + project: Callable[[Path, dict[str, Any]], dict[str, Any] | None], + known_fingerprints: Mapping[Path, tuple[int, ...]] | None = None, + ) -> Iterator[tuple[Path, dict[str, Any], tuple[int, ...] | None]]: seen = set() with self._lock: entries = self._directories.setdefault(directory, {}) for path in paths: seen.add(path) - try: - before = file_change_fingerprint(path) - except OSError: - before = None + before = known_fingerprints.get(path) if known_fingerprints is not None else None + if before is None: + try: + before = file_change_fingerprint(path) + except OSError: + before = None with self._lock: saved = entries.get(path) if before is None or saved is None or saved[0] != before: entries.pop(path, None) saved = None if saved is not None: - yield path, deepcopy(saved[1]) + yield path, deepcopy(saved[1]), before continue # Preserve the owning reader's missing/unreadable/strict behavior. value = read(path) @@ -413,11 +466,18 @@ def rows(self, directory: Path, paths: Iterable[Path], *, if unchanged: with self._lock: entries[path] = before, deepcopy(view) - yield path, value if view is None else view + yield path, (value if view is None else view), before with self._lock: for path in entries.keys() - seen: del entries[path] + def rows(self, directory: Path, paths: Iterable[Path], *, + read: Callable[[Path], dict[str, Any]], + project: Callable[[Path, dict[str, Any]], dict[str, Any] | None], + ) -> Iterator[tuple[Path, dict[str, Any]]]: + for path, view, _ in self.rows_with_fingerprints(directory, paths, read=read, project=project): + yield path, view + _REPLY_IMAGE_TYPES = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", diff --git a/loopx/extensions/lark/goal_topic_runtime_service.py b/loopx/extensions/lark/goal_topic_runtime_service.py index 4541892d01..25a8016974 100644 --- a/loopx/extensions/lark/goal_topic_runtime_service.py +++ b/loopx/extensions/lark/goal_topic_runtime_service.py @@ -110,6 +110,8 @@ class ProfileWorkerFingerprint: class LarkGoalTopicRuntimeService: """Own one event-consumer worker per reusable Lark App profile.""" + _LISTENER_SNAPSHOT_TTL_SECONDS = 5.0 + def __init__( self, *, @@ -129,6 +131,11 @@ def __init__( self._profile_poller = profile_poller or self._poll_profile self.manager_route_reconciler = manager_route_reconciler self._lock = threading.Lock() + self._listener_snapshot_cond = threading.Condition(threading.Lock()) + self._listener_snapshot_in_flight = False + self._listener_snapshot_value: PayloadView | None = None + self._listener_snapshot_at = 0.0 + self._listener_snapshot_generation = 0 self._workers: dict[str, ProfileWorker] = {} self._message_consumers: dict[str, MessageConsumer] = {} self._health: dict[str, Payload] = {} @@ -357,6 +364,44 @@ def _handle_private_steward_callback( self._delivery_wakeup.set() return result + def _invalidate_listener_snapshot(self) -> int: + with self._listener_snapshot_cond: + self._listener_snapshot_generation += 1 + self._listener_snapshot_value = None + self._listener_snapshot_at = 0.0 + self._listener_snapshot_cond.notify_all() + return self._listener_snapshot_generation + + def _seed_listener_snapshot(self, generation: int, snapshot: PayloadView) -> None: + with self._listener_snapshot_cond: + if self._listener_snapshot_generation == generation and not self._closed.is_set(): + self._listener_snapshot_value = snapshot + self._listener_snapshot_at = time.monotonic() + + def _read_listener_snapshot(self) -> PayloadView: + with self._listener_snapshot_cond: + while self._listener_snapshot_in_flight: + self._listener_snapshot_cond.wait() + if ( + not self._closed.is_set() + and self._listener_snapshot_value is not None + and time.monotonic() - self._listener_snapshot_at < self._LISTENER_SNAPSHOT_TTL_SECONDS + ): + return self._listener_snapshot_value + generation = self._listener_snapshot_generation + self._listener_snapshot_in_flight = True + try: + fresh = self.snapshot_provider() + finally: + with self._listener_snapshot_cond: + self._listener_snapshot_in_flight = False + self._listener_snapshot_cond.notify_all() + with self._listener_snapshot_cond: + if self._listener_snapshot_generation == generation and not self._closed.is_set(): + self._listener_snapshot_value = fresh + self._listener_snapshot_at = time.monotonic() + return fresh + def _refresh_on_start(self) -> None: while not self._closed.is_set(): try: @@ -720,7 +765,7 @@ def start_consumer(args: list[str]) -> Popen[str]: continue def listener_snapshot() -> PayloadView: - current = self.snapshot_provider() + current = self._read_listener_snapshot() with self._lock: # Reuse the same observations used to admit # each worker. A successful refresh adds new @@ -830,7 +875,9 @@ def _refresh_profile_workers(self, *, resume_queues: bool = True) -> bool: return False self._profile_refresh_sequence += 1 sequence = self._profile_refresh_sequence + snapshot_generation = self._invalidate_listener_snapshot() snapshot = self.snapshot_provider() + self._seed_listener_snapshot(snapshot_generation, snapshot) profile_configs, unavailable = self._listener_configs(snapshot) with self._lock: if self._closed.is_set(): @@ -944,6 +991,7 @@ def active_profiles(self) -> list[str]: def close(self) -> None: self._closed.set() + self._invalidate_listener_snapshot() self._delivery_wakeup.set() with self._lock: if self._refresh_retry is not None: diff --git a/loopx/extensions/lark/private_conversations.py b/loopx/extensions/lark/private_conversations.py index 7df64a6e18..dc325fcfb5 100644 --- a/loopx/extensions/lark/private_conversations.py +++ b/loopx/extensions/lark/private_conversations.py @@ -190,7 +190,8 @@ def _health_delivery_discovery(self) -> FileChatDiscovery: @effect_runtime_request_scope() def health(self) -> dict[str, dict[str, int]]: health: dict[str, dict[str, int]] = {} - quota_pending = self.core.quota_backlog_observer(use_cache=True) + observed_fps: dict[Path, tuple[int, ...]] = {} + quota_pending = self.core.quota_backlog_observer(use_cache=True, observed_fps=observed_fps) completed: list[tuple[dict[str, Any], dict[str, Any]]] = [] def count(row: dict[str, Any]) -> None: @@ -212,9 +213,7 @@ def health_fields(_path: Path, row: dict[str, Any]) -> dict[str, Any] | None: or (row.get("historical_answer_images") or {}).get("status") == "pending"): return None deliveries = row.get("deliveries") - if not isinstance(deliveries, dict) or not all( - isinstance(phase, dict) for phase in deliveries.values() - ): + if not isinstance(deliveries, dict) or not all(isinstance(phase, dict) for phase in deliveries.values()): return None # Settings uses only correlation and retry metadata. Never retain # delivered bodies or use these fields for admission or sending. @@ -226,9 +225,7 @@ def health_fields(_path: Path, row: dict[str, Any]) -> dict[str, Any] | None: def delivered_requests() -> Iterator[dict[str, Any]]: paths = [*self.root.glob("*.json"), *self.root.glob("web-projections/*.json")] - for path, row in self._health_delivery_discovery.rows( - self.root, paths, read=_read_json, project=health_fields, - ): + for path, row in self._health_delivery_discovery.rows(self.root, paths, read=_read_json, project=health_fields): health.setdefault(row["binding_id"], {"pending_count": 0, "recovery_count": 0}) if row["status"] != "delivered": if triage.pending(row): @@ -240,7 +237,9 @@ def delivered_requests() -> Iterator[dict[str, Any]]: # Pair the original receipts with the current Core correlation and Turn. receipts, refs = tee(delivered_requests()) - natives = self.core.read_backlog_requests(row["request_ref"] for row in refs) + natives = self.core.read_backlog_requests( + (row["request_ref"] for row in refs), defer_recovery=True, observed_fps=observed_fps, + ) for row, native in zip(receipts, natives, strict=True): if (native.get("status") == "accepted" and native.get("command") is None and _same_fields(native, row, "binding_id", "source", "session_id", "turn_id")): diff --git a/tests/test_lark_steward_directory_scan_perf.py b/tests/test_lark_steward_directory_scan_perf.py index 65dc8579f7..34d0b3af56 100644 --- a/tests/test_lark_steward_directory_scan_perf.py +++ b/tests/test_lark_steward_directory_scan_perf.py @@ -773,3 +773,319 @@ def wrapper_rpc(method, params, **kwargs): sess_path = _store2._session_path(sid2) _atomic_write_json(sess_path, {**store_read_json(sess_path), "status": "closed"}) assert transport2.health()["b" * 24]["pending_count"] == 0 + + +def test_listener_snapshot_coalesces_concurrent_watchers_and_invalidates_on_refresh(tmp_path, monkeypatch): + import threading + from loopx.extensions.lark import goal_topic_runtime_service as svc_mod + from loopx.extensions.lark.goal_topic_runtime_service import LarkGoalTopicRuntimeService + + clock = [100.0] + monkeypatch.setattr(svc_mod.time, "monotonic", lambda: clock[0]) + + entered = threading.Event() + release = threading.Event() + provider_calls = 0 + snapshot_seq = 0 + fail_next = False + + def snapshot_provider(): + nonlocal provider_calls, snapshot_seq, fail_next + provider_calls += 1 + if fail_next: + fail_next = False + raise RuntimeError("transient snapshot error") + if provider_calls == 1: + entered.set() + assert release.wait(2) + snapshot_seq += 1 + return { + "private_profiles": {}, + "target_payload": {"targets": {}}, + "binding_payloads": {}, + "seq": snapshot_seq, + } + + service = LarkGoalTopicRuntimeService( + snapshot_provider=snapshot_provider, + runtime_root=tmp_path, + runtime_controller=SimpleNamespace(store=None), + profile_poller=lambda _profile, stop: stop.wait(), + ) + results: list[dict[str, object]] = [] + errors: list[BaseException] = [] + + def reader(): + try: + results.append(dict(service._read_listener_snapshot())) + except BaseException as exc: + errors.append(exc) + + threads = [threading.Thread(target=reader) for _ in range(3)] + for t in threads: + t.start() + assert entered.wait(2) + # Give sibling threads a moment to block on single-flight condition before releasing. + threading.Event().wait(0.05) + release.set() + for t in threads: + t.join(2) + assert not errors + assert provider_calls == 1 + assert [row["seq"] for row in results] == [1, 1, 1] + + # Within TTL, repeated calls hit the coalesced cache. + clock[0] += 4.5 + assert service._read_listener_snapshot()["seq"] == 1 + assert provider_calls == 1 + + # Past TTL (5.0s), failures are not cached and the following call retries cleanly. + clock[0] += 0.6 + fail_next = True + with pytest.raises(RuntimeError, match="transient snapshot error"): + service._read_listener_snapshot() + assert provider_calls == 2 + assert service._read_listener_snapshot()["seq"] == 2 + assert provider_calls == 3 + + # refresh() invalidates and seeds the cache atomically. + assert service._refresh_profile_workers(resume_queues=False) is False + assert provider_calls == 4 + assert service._read_listener_snapshot()["seq"] == 3 + assert provider_calls == 4 + service.close() + + +def test_context_recovery_discovery_caches_turn_event_summary_without_event_lock_churn(tmp_path, monkeypatch): + from loopx.chat_manager import MANAGER_CONTEXT_VERSION + from loopx.chat_store import CHAT_TURN_SCHEMA_VERSION, ChatSessionStore + + store = ChatSessionStore(tmp_path) + binding_id = "b" * 24 + source_ref = "c" * 24 + binding = { + "schema_version": "loopx_chat_conversation_binding_v0", + "binding_id": binding_id, + "enabled": True, + "transport_ref": "steward-app", + "context_kind": "steward", + "goal_ids": [], + "grant": "portfolio_read", + "provider_ref": "a" * 24, + "operator_ref": "e" * 24, + "project_ref": "f" * 24, + "executor_endpoint_id": "antigravity", + } + session = store.create_session( + goal_id="loopx-manager", + agent_id="antigravity", + adapter_kind="antigravity", + upstream_thread_id="thread-recovery-events", + upstream_mode="default", + session_id="d" * 32, + channel_id=f"manager.external.native.{binding_id}.{source_ref}", + steward_context={ + "kind": "bound_steward", + "binding_id": binding_id, + "audience": "bound_owner", + "grant": "portfolio_read", + "goal_ids": [], + "provider_ref": "a" * 24, + "operator_ref": "e" * 24, + "project_ref": "f" * 24, + "source_ref": source_ref, + "workspace_path": str(tmp_path), + }, + ) + sid = session["session_id"] + sess_path = store._session_path(sid) + _atomic_write_json(sess_path, { + **store.load_session(sid), + "session_mode": "managed_runtime", + "executor_endpoint_id": "antigravity", + "manager_context_version": MANAGER_CONTEXT_VERSION, + }) + controller = SimpleNamespace( + store=store, + lock=SimpleNamespace(__enter__=lambda s: None, __exit__=lambda s, *a: False), + adapters={}, + codex_home=tmp_path, + project_contexts=SimpleNamespace( + conversation_bindings=SimpleNamespace( + read=lambda: {"bindings": [binding]}, + resolve=lambda **_: {"binding": binding}, + ), + ), + ) + core = ChatExternalConversations(controller) + core.root.mkdir(parents=True, exist_ok=True) + + refs: list[str] = [] + tids: list[str] = [] + # 12 failed turns (> ChatEventCache's 8-turn limit) would thrash events_after without summary caching. + for idx in range(12): + ref = _hex_ref(2000 + idx) + tid = f"{2000 + idx:032x}" + refs.append(ref) + tids.append(tid) + _atomic_write_json(store._turn_path(sid, tid), { + "schema_version": CHAT_TURN_SCHEMA_VERSION, + "session_id": sid, + "turn_id": tid, + "client_turn_id": f"external-{ref}", + "message": f"question {idx}", + "origin": "lark", + "status": "failed", + "error_code": "antigravity_step_error", + "delta_count": 0, + "upstream_turn_id": f"upstream-{idx}", + "created_at": "2026-01-01T00:00:00+00:00", + "completed_at": "2026-01-01T00:01:00+00:00", + "response": None, + }) + store.append_event(sid, tid, kind="turn.started", payload={ + "native_input": { + "thread_id": "thread-recovery-events", + "start_step_index": idx, + "input_sha256": "a" * 64, + }, + }, buffered=False) + if 4 <= idx < 8: + store.append_event(sid, tid, kind="turn.steered", payload={}, buffered=False) + elif idx >= 8: + store.append_event(sid, tid, kind="turn.interrupted", payload={}, buffered=False) + _atomic_write_json(core.root / f"{ref}.json", { + "request_ref": ref, + "binding_id": binding_id, + "source": {"source_ref": source_ref, "sender_ref": "e" * 24, "private_human_message": True}, + "session_id": sid, + "turn_id": tid, + "command": None, + "origin": "lark", + "status": "accepted", + "delivery_verified": True, + "message": f"question {idx}", + "created_at": "2026-01-01T00:00:00+00:00", + }) + store.update_session(sid, manager_context_version=MANAGER_CONTEXT_VERSION) + + event_reads: list[str] = [] + original_events_after = ChatSessionStore.events_after + + def counted_events_after(self_store, session_id: str, turn_id: str, event_id: str | None): + event_reads.append(turn_id) + return original_events_after(self_store, session_id, turn_id, event_id) + + monkeypatch.setattr(ChatSessionStore, "events_after", counted_events_after) + + observe1 = core.context_recovery_observer() + verdicts1 = [observe1(ref) for ref in refs] + assert len(event_reads) == 12 + assert verdicts1 == [True] * 4 + [False] * 8 + + # Pass 2 across all 12 historical turns hits _event_summaries with zero events_after reads. + event_reads.clear() + observe2 = core.context_recovery_observer() + assert [observe2(ref) for ref in refs] == verdicts1 + assert event_reads == [] + + # Flushed turn.interrupted event invalidates only that Turn immediately. + store.append_event(sid, tids[0], kind="turn.interrupted", payload={}, buffered=False) + observe3 = core.context_recovery_observer() + assert observe3(refs[0]) is False + assert observe3(refs[1]) is True + assert event_reads == [tids[0]] + + # Buffered unflushed event in _event_pending bypasses cache immediately. + event_reads.clear() + store.append_event(sid, tids[1], kind="turn.interrupt_requested", payload={}, buffered=True) + observe4 = core.context_recovery_observer() + assert observe4(refs[1]) is False + assert event_reads == [tids[1]] + + # Non-discovery admission and preparation failures always read fresh full events_after. + event_reads.clear() + row2 = core.read_request(refs[2]) + core._context_recovery_plan(row2, discovery_only=False, selected_binding=binding) + assert event_reads == [tids[2]] + + event_reads.clear() + _atomic_write_json(store._turn_path(sid, tids[3]), { + "schema_version": CHAT_TURN_SCHEMA_VERSION, + "session_id": sid, + "turn_id": tids[3], + "client_turn_id": f"external-{refs[3]}", + "status": "failed", + "error_code": "manager_authorization_unavailable", + "created_at": "2026-01-01T00:00:00+00:00", + "response": None, + }) + core.context_recovery_observer()(refs[3]) + assert event_reads == [tids[3]] + + +def test_backlog_health_defers_recovery_body_reads_and_reuses_request_fingerprints(tmp_path, monkeypatch): + import loopx.capabilities.native_chat.external_conversations as ext_mod + import loopx.capabilities.native_chat.file_discovery as fd_mod + + store, core, transport, sid, failed_rows = _seed_backlog_health_store(tmp_path, failed_count=3, completed_count=2) + # Add two requests carrying context_recovery metadata so deferred recovery reads are exercised. + for row in failed_rows[:2]: + _atomic_write_json(core.root / f"{row['request_ref']}.json", { + **row, + "context_recovery": {"failed_turn_id": row["turn_id"], "client_turn_id": f"external-{row['request_ref']}-recovery-1"}, + }) + # Add an earlier /stop command for an unrelated source so stop_controls() is queried while pending stays 3. + _atomic_write_json(core.root / f"{_hex_ref(8888)}.json", { + "request_ref": _hex_ref(8888), + "binding_id": "b" * 24, + "source": {"source_ref": "e" * 24}, + "session_id": sid, + "turn_id": None, + "command": "stop", + "status": "command_completed", + "created_at": "2026-01-01T00:00:00+00:00", + }) + + # Cold pass warms both FileChatDiscovery and QuotaBacklogObservationCache. + assert transport.health()["b" * 24]["pending_count"] == 3 + + reads: list[Path] = [] + stats_by_path: dict[Path, int] = {} + original_read = ext_mod._read_json + original_fp = fd_mod.file_change_fingerprint + + def counted_read(path: Path): + reads.append(path) + return original_read(path) + + def counted_fp(path: Path): + stats_by_path[path] = stats_by_path.get(path, 0) + 1 + return original_fp(path) + + monkeypatch.setattr(ext_mod, "_read_json", counted_read) + monkeypatch.setattr(fd_mod, "file_change_fingerprint", counted_fp) + + assert transport.health()["b" * 24]["pending_count"] == 3 + assert reads == [], "warm health() must not re-read recovery requests" + for row in failed_rows: + req_path = core.root / f"{row['request_ref']}.json" + assert stats_by_path.get(req_path) == 1, f"expected 1 stat for {req_path}, got {stats_by_path.get(req_path)}" + + # Mutating a recovery request (marking recovered on row 0) with preserved mtime invalidates and drops pending to 2. + target_path = core.root / f"{failed_rows[0]['request_ref']}.json" + stamp = target_path.stat() + _atomic_write_json(target_path, { + **store_read_json(target_path), + "native_result_recovery": { + "turn_id": failed_rows[0]["turn_id"], + "message_id": f"native.{failed_rows[0]['turn_id']}.recovered", + "message_text": "recovered ok", + "delivery_verified": True, + "native_observation": {"response": {"message": "recovered ok", "proposals": []}}, + }, + }) + os.utime(target_path, ns=(stamp.st_atime_ns, stamp.st_mtime_ns)) + reads.clear() + assert transport.health()["b" * 24]["pending_count"] == 2 + assert set(reads) == {target_path}