diff --git a/loopx/capabilities/manager_context/tracking.py b/loopx/capabilities/manager_context/tracking.py index 434091ab0c..b2cba8ac54 100644 --- a/loopx/capabilities/manager_context/tracking.py +++ b/loopx/capabilities/manager_context/tracking.py @@ -5,6 +5,8 @@ from __future__ import annotations +from typing import Any + from . import _read, _root, evidence_goal_scope from ...control_plane.collaboration.inbox import ( _entry as _entry, @@ -16,6 +18,7 @@ from ...control_plane.collaboration.goal_instance_scope import ( collaboration_goal_scope, decide_collaboration_lifecycle, + scope_for_caller_goal_ref, ) from ...control_plane.collaboration.links import read_linked_work from ...control_plane.content_digest import ( @@ -61,6 +64,7 @@ def query( except (OSError, ValueError, TypeError): continue rows, unreadable = [], 0 + candidates_by_goal: dict[str, list[dict[str, Any]]] = {} paths = sorted( (_root(root) / "entries").glob( "*/" + (request_id + ".json" if request_id else "*.json") @@ -69,7 +73,8 @@ def query( for path in paths[:2000]: try: row = _read(path) - if row.get("goal_id") not in goal_ids: + gid = row.get("goal_id") + if gid not in goal_ids: continue if agent_id is not None and row.get("agent_id") != agent_id: continue @@ -83,27 +88,42 @@ def query( and legacy_channels.get(row.get("source_id")) == {channel_id} ): continue + candidates_by_goal.setdefault(gid, []).append(row) + except (OSError, ValueError, KeyError, TypeError): + unreadable += 1 + # Hold each Goal's lifetime fence once per matched Goal rather than once per + # handoff entry so lock contention cannot amplify into O(N * timeout) stalls. + for gid, group_rows in candidates_by_goal.items(): + try: with collaboration_goal_scope( registry_path, - goal_id=row["goal_id"], + goal_id=gid, agents=(), - caller_goal_ref=row.get("goal_ref"), - ) as goal_scope: - entry = _entry( - root, - row["goal_id"], - row["agent_id"], - row["request_id"], - scope=goal_scope, - ) - decide_collaboration_lifecycle( - goal_scope, - operation="history_inspect", - record=entry, - ) - rows.append(row) + caller_goal_ref=None, + ) as base_scope: + for row in group_rows: + try: + goal_scope = scope_for_caller_goal_ref( + base_scope, + row.get("goal_ref"), + ) + entry = _entry( + root, + row["goal_id"], + row["agent_id"], + row["request_id"], + scope=goal_scope, + ) + decide_collaboration_lifecycle( + goal_scope, + operation="history_inspect", + record=entry, + ) + rows.append(row) + except (OSError, ValueError, KeyError, TypeError): + unreadable += 1 except (OSError, ValueError, KeyError, TypeError): - unreadable += 1 + unreadable += len(group_rows) rows.sort( key=lambda row: (row.get("delivered_at") or "", row["request_id"]), reverse=True ) diff --git a/loopx/control_plane/collaboration/goal_instance_scope.py b/loopx/control_plane/collaboration/goal_instance_scope.py index e36efabb15..2111d04023 100644 --- a/loopx/control_plane/collaboration/goal_instance_scope.py +++ b/loopx/control_plane/collaboration/goal_instance_scope.py @@ -2,7 +2,7 @@ from collections.abc import Iterator from contextlib import ExitStack, contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Any @@ -177,6 +177,28 @@ def capture_collaboration_goal_ref( ) +def scope_for_caller_goal_ref( + scope: CollaborationGoalScope, + caller_goal_ref: dict[str, str] | None, +) -> CollaborationGoalScope: + """Bind one entry's caller_goal_ref under an already-held Goal lifetime scope.""" + + if not scope.exact: + return scope + if caller_goal_ref is not None and not isinstance(caller_goal_ref, dict): + raise TypeError("source-session caller_goal_ref must be a mapping") + if scope.current_goal_ref is None: + raise ValueError("source-session Goal scope requires current_goal_ref") + return replace( + scope, + caller_goal_ref=_caller_ref( + goal_id=scope.goal_id, + requested=caller_goal_ref, + current=scope.current_goal_ref, + ), + ) + + class CollaborationLifecycleRejected(ValueError): """Retain the typed refusal for authorized diagnostic readers.""" diff --git a/tests/test_manager_context_tracking.py b/tests/test_manager_context_tracking.py index f8b204e0df..d7e2a9606c 100644 --- a/tests/test_manager_context_tracking.py +++ b/tests/test_manager_context_tracking.py @@ -586,3 +586,201 @@ def observe(): assert "Other receiver's private work" not in json.dumps(foreign) with pytest.raises(ValueError, match="must belong"): link(root, registry, "research", "worker", ids[1], ["todo_history"], []) + + + +def test_handoff_query_batches_goal_lifetime_scope_per_goal(fixture, monkeypatch): + import loopx.capabilities.manager_context.tracking as tracking + + root, registry, session, turn, target = fixture + expected_ids = [] + for idx in range(3): + rid = deliver( + root, + registry, + session=session, + turn=dict(turn, client_turn_id=f"batch-{idx}"), + request=target, + )["request_id"] + expected_ids.append(rid) + target_dir = next((_root(root) / "entries").iterdir()) + (target_dir / "corrupt.json").write_text("{damaged", encoding="utf-8") + + real_scope = tracking.collaboration_goal_scope + scope_calls = [] + + def spy_scope(reg_path, *, goal_id, agents, caller_goal_ref=None, **kwargs): + scope_calls.append(goal_id) + return real_scope( + reg_path, + goal_id=goal_id, + agents=agents, + caller_goal_ref=caller_goal_ref, + **kwargs, + ) + + monkeypatch.setattr(tracking, "collaboration_goal_scope", spy_scope) + result = query(root, registry, goal_ids=["research"], owner_scope=True) + + assert scope_calls == ["research"] + assert len(result["rows"]) == 3 + assert {row["request_id"] for row in result["rows"]} == set(expected_ids) + assert result["coverage"]["unreadable"] == 1 + + +def test_handoff_query_fails_fast_per_goal_on_lock_timeout(tmp_path, monkeypatch): + import loopx.capabilities.manager_context.tracking as tracking + from loopx.file_lock import LockAcquireTimeoutError + + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "goals": [ + { + "id": "contended", + "repo": str(tmp_path), + "coordination": {"registered_agents": ["worker"]}, + }, + { + "id": "healthy", + "repo": str(tmp_path), + "coordination": {"registered_agents": ["worker"]}, + }, + ] + } + ), + encoding="utf-8", + ) + session = {"session_id": "manager-session", "channel_id": "manager"} + for idx in range(5): + deliver( + tmp_path, + registry, + session=session, + turn={"client_turn_id": f"contended-{idx}", "origin": "web", "message": "Check lock."}, + request={"goal_id": "contended", "agent_id": "worker"}, + ) + healthy_ids = [] + for idx in range(2): + healthy_ids.append( + deliver( + tmp_path, + registry, + session=session, + turn={"client_turn_id": f"healthy-{idx}", "origin": "web", "message": "Check healthy."}, + request={"goal_id": "healthy", "agent_id": "worker"}, + )["request_id"] + ) + + real_scope = tracking.collaboration_goal_scope + calls = [] + + def contended_scope(reg_path, *, goal_id, agents, caller_goal_ref=None, **kwargs): + calls.append(goal_id) + if goal_id == "contended": + raise LockAcquireTimeoutError( + incident={"holder": {}, "operator_action": {"action": "inspect_lock_holder"}}, + incident_recorded=True, + incident_channel="test", + ) + return real_scope( + reg_path, + goal_id=goal_id, + agents=agents, + caller_goal_ref=caller_goal_ref, + **kwargs, + ) + + monkeypatch.setattr(tracking, "collaboration_goal_scope", contended_scope) + result = query(tmp_path, registry, goal_ids=["contended", "healthy"], owner_scope=True) + + assert calls.count("contended") == 1 + assert calls.count("healthy") == 1 + assert {row["request_id"] for row in result["rows"]} == set(healthy_ids) + assert result["coverage"]["unreadable"] == 5 + + +def test_handoff_query_batched_source_session_validates_per_row_goal_ref(tmp_path, monkeypatch): + import loopx.capabilities.manager_context.tracking as tracking + from loopx.control_plane.projects.registry_codec import source_session_registry_transaction + + instance_a = "ginst_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + instance_b = "ginst_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + registry = tmp_path / ".loopx" / "registry.json" + with source_session_registry_transaction( + registry, + operation="test_create_source_registry", + create=lambda: { + "profile_id": "source_session_v1", + "goals": [ + { + "id": "delivery", + "repo": str(tmp_path), + "status": "active", + "goal_instance_id": instance_a, + "coordination": {"registered_agents": ["worker"]}, + } + ], + "session_bindings": [], + "session_receipts": [], + "lifetime_receipts": [], + }, + ) as transaction: + transaction.commit(transaction.payload_copy()) + + session = {"session_id": "manager-session", "channel_id": "manager"} + valid_ids = [] + for idx in range(2): + valid_ids.append( + deliver( + tmp_path, + registry, + session=session, + turn={"client_turn_id": f"valid-{idx}", "origin": "web", "message": "Valid instance."}, + request={"goal_id": "delivery", "agent_id": "worker"}, + )["request_id"] + ) + stale_id = deliver( + tmp_path, + registry, + session=session, + turn={"client_turn_id": "stale", "origin": "web", "message": "Stale instance."}, + request={"goal_id": "delivery", "agent_id": "worker"}, + )["request_id"] + stale_path = next((_root(tmp_path) / "entries").glob(f"*/{stale_id}.json")) + stale_row = _read(stale_path) + stale_row["goal_ref"] = {"goal_id": "delivery", "goal_instance_id": instance_b} + _write(stale_path, stale_row) + + malformed_id = deliver( + tmp_path, + registry, + session=session, + turn={"client_turn_id": "malformed", "origin": "web", "message": "Malformed ref."}, + request={"goal_id": "delivery", "agent_id": "worker"}, + )["request_id"] + malformed_path = next((_root(tmp_path) / "entries").glob(f"*/{malformed_id}.json")) + malformed_row = _read(malformed_path) + malformed_row["goal_ref"] = "bad-ref" + _write(malformed_path, malformed_row) + + real_scope = tracking.collaboration_goal_scope + scope_calls = [] + + def spy_scope(reg_path, *, goal_id, agents, caller_goal_ref=None, **kwargs): + scope_calls.append(goal_id) + return real_scope( + reg_path, + goal_id=goal_id, + agents=agents, + caller_goal_ref=caller_goal_ref, + **kwargs, + ) + + monkeypatch.setattr(tracking, "collaboration_goal_scope", spy_scope) + result = query(tmp_path, registry, goal_ids=["delivery"], owner_scope=True) + + assert scope_calls == ["delivery"] + assert {row["request_id"] for row in result["rows"]} == set(valid_ids) + assert result["coverage"]["unreadable"] == 2