Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions loopx/extensions/lark/goal_topic_runtime_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ def _reconcile_private(self) -> None:
feedback_due = 0.0
discovery: Future[list[Path]] | None = None
discovery_due = 0.0
discovery_yielded = False
ingress: Future[list[str]] | None = None
ingress_due = time.monotonic() + 60
wakeup_scope = getattr(self.runtime_controller, "turn_release_wakeup", None)
Expand All @@ -135,7 +136,7 @@ def _reconcile_private(self) -> None:
(wakeup_scope(self._delivery_wakeup) if callable(wakeup_scope) else nullcontext()):
def step() -> None:
nonlocal attempted, ready, attention, attention_due, reports, reports_due
nonlocal ingress, ingress_due, feedback, feedback_due, discovery, discovery_due
nonlocal ingress, ingress_due, feedback, feedback_due, discovery, discovery_due, discovery_yielded
# Clear BEFORE scanning: an arrival during the scan must also
# wake the following wait, rather than disappear in that gap.
self._delivery_wakeup.clear()
Expand Down Expand Up @@ -208,6 +209,9 @@ def step() -> None:
for path in paths:
if path not in pending:
ready.setdefault(path, None)
active_background = sum(job is not None for job in (feedback, reports, attention, ingress, discovery))
discovery_ready = (callable(body_paths) and discovery is None
and time.monotonic() >= discovery_due)
owner_attention = getattr(transport, "owner_attention", None)
attention_ready = (owner_attention is not None and attention is None
and time.monotonic() >= attention_due and owner_attention.has_enabled())
Expand All @@ -233,21 +237,30 @@ def step() -> None:
# Keep an already discovered request eligible for the next
# scan. Its historical discovery may be backed off.
continue
active_background = sum(job is not None for job in (feedback, reports, attention, ingress, discovery))
due_background = sum((reports_ready, attention_ready, ingress_ready))
due_background = sum((reports_ready, attention_ready, ingress_ready, discovery_ready))
scheduled_body = False
for path in sorted(recovering, key=lambda path: (recovering[path], attempted.get(path, 0))):
occupied = len(pending) + active_background
if occupied >= 4 or self._closed.is_set():
break
# Give an unsent body the next available slot when due
# checks would otherwise reserve every free slot.
if occupied + due_background >= 4 and (recovering[path] or scheduled_body):
if occupied + due_background >= 4 and (recovering[path] or scheduled_body
or (discovery_ready and discovery_yielded)):
break
pending[path] = pool.submit(transport.reconcile_request, path)
scheduled_body = scheduled_body or not recovering[path]
del ready[path]
attempted[path] = attempted.get(path, 0) + 1
# Let an unsent original reply use the only free slot first.
# Once that body yields it, the due scan wins over recurring
# attempts, so newly committed App replies remain discoverable.
discovery_yielded = discovery_yielded or (discovery_ready and scheduled_body)
if (discovery_ready and len(pending) + active_background < 4
and not self._closed.is_set()):
discovery = pool.submit(transport.pending_delivery_paths)
discovery.add_done_callback(lambda _: self._delivery_wakeup.set())
discovery_yielded = False
if reports_ready and len(pending) + int(feedback is not None) + int(attention is not None or attention_ready) + int(ingress is not None or ingress_ready) + int(discovery is not None) < 4 and not self._closed.is_set():
reports = pool.submit(report_provider.reconcile)
if attention_ready and len(pending) + int(feedback is not None) + int(reports is not None) + int(ingress is not None or ingress_ready) + int(discovery is not None) < 4 and not self._closed.is_set():
Expand All @@ -261,14 +274,6 @@ def step() -> None:
and len(pending) + int(reports is not None) + int(attention is not None)
+ int(ingress is not None) + int(discovery is not None) < 3 and not self._closed.is_set()):
feedback = pool.submit(app_feedback)
# One discovery pass uses this same bounded pool. Its slow
# native/history observations cannot hold the scanner that
# discovers fresh captures and already generated answers.
if (callable(body_paths) and discovery is None and time.monotonic() >= discovery_due
and len(pending) + int(feedback is not None) + int(reports is not None)
+ int(attention is not None) + int(ingress is not None) < 4 and not self._closed.is_set()):
discovery = pool.submit(transport.pending_delivery_paths)
discovery.add_done_callback(lambda _: self._delivery_wakeup.set())
except Exception:
logging.getLogger(__name__).warning("Private Chat delivery discovery is pending")

Expand Down
137 changes: 131 additions & 6 deletions tests/test_lark_private_conversations.py
Original file line number Diff line number Diff line change
Expand Up @@ -8540,7 +8540,8 @@ def reconcile(path):
runtime.close()


def test_historical_discovery_uses_one_existing_slot_without_an_executor_backlog(tmp_path):
@pytest.mark.parametrize("bodies_already_pending", [False, True])
def test_historical_discovery_uses_one_existing_slot_without_an_executor_backlog(tmp_path, bodies_already_pending):
from loopx.extensions.lark.goal_topic_runtime_service import LarkGoalTopicRuntimeService

scanning, release_scan, full, release_bodies = (threading.Event() for _ in range(4))
Expand All @@ -8566,15 +8567,22 @@ def reconcile_request(self, path):

service = LarkGoalTopicRuntimeService(snapshot_provider=lambda: {}, runtime_root=tmp_path,
runtime_controller=None, private_conversations=Pending())
service.start()
closer = None
try:
assert scanning.wait(5)

def queue_original_requests():
for index in range(9):
path = tmp_path / str(index)
path.write_text("durable original request")
bodies.append(path)
service._delivery_wakeup.set()

if bodies_already_pending:
queue_original_requests()
service.start()
closer = None
try:
assert scanning.wait(5), "Pending bodies starved discovery of new App replies"
if not bodies_already_pending:
queue_original_requests()
service._delivery_wakeup.set()
assert full.wait(4)
assert len(calls) == 3 and len(set(calls)) == 3 # One scan plus three replies, total four.
closer = threading.Thread(target=service.close)
Expand All @@ -8589,3 +8597,120 @@ def reconcile_request(self, path):
service._delivery_thread.join(5)
assert not service._delivery_thread.is_alive()
assert len(calls) == 3 # The other six durable requests were never submitted to the executor.


def test_completed_original_reply_uses_only_free_slot_before_due_discovery(ordinary, monkeypatch): # noqa: F811
from loopx.chat_store import _read_json
from loopx.extensions.lark import goal_topic_runtime_service as module

store, runtime, provider, transport = connect(ordinary)
held, release, scan, delivered = (threading.Event() for _ in range(4))
clock, active, scans = [0.0], [], []
lock = threading.Lock()
blockers = [transport.root.parent / f"held-{i}.json" for i in range(3)]
for path in blockers:
path.write_text("durable historical fixture")
body_paths, reconcile = transport.pending_body_paths, transport.reconcile_request
monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: clock[0]))
monkeypatch.setattr(transport, "pending_body_paths", lambda: [*blockers, *body_paths()])
monkeypatch.setattr(transport, "is_recovery_path", lambda path: path in blockers)
monkeypatch.setattr(transport, "reconcile_app_feedback", None)
monkeypatch.setattr(transport.owner_attention, "has_enabled", lambda: False)
monkeypatch.setattr(transport.steward_reports, "pending", lambda: False)

def discover():
scans.append(clock[0])
if len(scans) > 1:
scan.set()
assert release.wait(12)
return []

def deliver(path):
if path in blockers:
with lock:
active.append(path)
if len(active) == 3:
held.set()
assert release.wait(12)
return 0
result = reconcile(path)
if _read_json(path).get("deliveries", {}).get("terminal", {}).get("verified"):
delivered.set()
return result

monkeypatch.setattr(transport, "pending_delivery_paths", discover)
monkeypatch.setattr(transport, "reconcile_request", deliver)
service = module.LarkGoalTopicRuntimeService(snapshot_provider=lambda: {}, runtime_root=transport.runtime_root,
runtime_controller=runtime, private_conversations=transport,
profile_poller=lambda _profile, stop: stop.wait(12))
service.start()
try:
assert held.wait(4)
event = provider.event("notes-app", "only-free-slot", "Answer the original question")
assert transport.admit("notes-app", event)["status"] == "durably_accepted"
ref = identity_ref(transport._binding("notes-app")["provider_ref"], event["message_id"])
request = transport.core.read_request(ref)
turn = runtime.wait_for_turn(session_id=request["session_id"], turn_id=request["turn_id"], timeout_sec=5)
assert turn["status"] == "completed"
clock[0] = 10.0
service._delivery_wakeup.set()
assert delivered.wait(3), "Due discovery took the only free slot ahead of the completed original reply"
assert scan.wait(3), "Discovery did not progress after the original reply released its slot"
assert not release.is_set()
terminal = _read_json(transport.root / f"{ref}.json")["deliveries"]["terminal"]
assert terminal["verified"] and terminal["text"] == turn["response"]["message"]
assert provider.messages[terminal["attempt"]["message_ref"]]["chat_id"] == event["chat_id"]
assert len(store.list_sessions()) == 1
packets = [json.loads(line) for line in ordinary[4].read_text().splitlines()]
assert sum(packet["method"] == "turn/start" for packet in packets) == 1
finally:
release.set()
service.close()
service._delivery_thread.join(5)
runtime.close()


def test_due_discovery_progresses_when_only_free_slot_keeps_retrying_a_body(tmp_path, monkeypatch):
from loopx.extensions.lark import goal_topic_runtime_service as module

held, release, discovered = (threading.Event() for _ in range(3))
clock, calls, scans = [0.0], [], []
lock = threading.Lock()
paths = [tmp_path / str(i) for i in range(4)]
for path in paths:
path.write_text("durable request")

class Pending:
def pending_body_paths(self):
return paths

def pending_delivery_paths(self):
scans.append(clock[0])
if len(scans) > 1:
discovered.set()
assert release.wait(12)
return []

def reconcile_request(self, path):
with lock:
calls.append(path)
if all(item in calls for item in paths):
held.set()
if path != paths[-1]:
assert release.wait(12)
return 0 # The unsent fourth body remains durably pending.

monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: clock[0]))
service = module.LarkGoalTopicRuntimeService(snapshot_provider=lambda: {}, runtime_root=tmp_path,
runtime_controller=None, private_conversations=Pending())
service.start()
try:
assert held.wait(4)
clock[0] = 10.0
service._delivery_wakeup.set()
assert discovered.wait(4), "Repeated unsent-body attempts starved due App reply discovery"
assert not release.is_set()
finally:
release.set()
service.close()
service._delivery_thread.join(5)
Loading