diff --git a/loopx/capabilities/native_chat/github_pr_evidence.py b/loopx/capabilities/native_chat/github_pr_evidence.py index 765c3b95c3..a718608889 100644 --- a/loopx/capabilities/native_chat/github_pr_evidence.py +++ b/loopx/capabilities/native_chat/github_pr_evidence.py @@ -378,30 +378,177 @@ def summarize_github_pr_signal(row: Mapping[str, Any]) -> dict[str, Any]: return summary +_GITHUB_PR_LIST_PAGE_SIZE = 20 +_GITHUB_PR_LIST_MAX_PAGES = 5 +_GITHUB_OPEN_PRS_GRAPHQL_QUERY = """query($owner: String!, $name: String!, $cursor: String) { + repository(owner: $owner, name: $name) { + pullRequests( + states: OPEN, + first: 20, + after: $cursor, + orderBy: {field: UPDATED_AT, direction: DESC} + ) { + pageInfo { + hasNextPage + endCursor + } + nodes { + number + title + url + headRefOid + headRefName + baseRefName + isDraft + reviewDecision + mergeStateStatus + mergeable + updatedAt + author { + login + } + reviews(last: 20) { + nodes { + id + state + submittedAt + body + author { + login + } + commit { + oid + } + } + } + commits(last: 1) { + nodes { + commit { + statusCheckRollup { + contexts(first: 30) { + nodes { + __typename + ... on CheckRun { + name + status + conclusion + } + ... on StatusContext { + context + state + } + } + } + } + } + } + } + } + } + } +}""" + + +def _normalize_graphql_pr_node(node: Mapping[str, Any]) -> dict[str, Any]: + raw_reviews = node.get("reviews") + if isinstance(raw_reviews, Mapping) and isinstance(raw_reviews.get("nodes"), list): + reviews = [item for item in raw_reviews["nodes"] if isinstance(item, Mapping)] + elif isinstance(raw_reviews, list): + reviews = [item for item in raw_reviews if isinstance(item, Mapping)] + else: + reviews = [] + + rollup = node.get("statusCheckRollup") + if rollup is None: + raw_commits = node.get("commits") + commit_nodes = raw_commits.get("nodes") if isinstance(raw_commits, Mapping) else None + first_commit = ( + commit_nodes[0].get("commit") + if isinstance(commit_nodes, list) and commit_nodes and isinstance(commit_nodes[0], Mapping) + else None + ) + raw_rollup = first_commit.get("statusCheckRollup") if isinstance(first_commit, Mapping) else None + raw_contexts = raw_rollup.get("contexts") if isinstance(raw_rollup, Mapping) else None + context_nodes = raw_contexts.get("nodes") if isinstance(raw_contexts, Mapping) else None + rollup = [item for item in context_nodes if isinstance(item, Mapping)] if isinstance(context_nodes, list) else [] + + return { + "number": node.get("number"), + "title": node.get("title"), + "url": node.get("url"), + "headRefOid": node.get("headRefOid"), + "headRefName": node.get("headRefName"), + "baseRefName": node.get("baseRefName"), + "isDraft": node.get("isDraft"), + "author": node.get("author"), + "reviewDecision": node.get("reviewDecision"), + "mergeStateStatus": node.get("mergeStateStatus"), + "mergeable": node.get("mergeable"), + "updatedAt": node.get("updatedAt"), + "reviews": reviews, + "statusCheckRollup": rollup, + } + + def observe_github_repository_prs( repo_slug: str, *, runner: Callable[..., Any] = subprocess.run ) -> dict[str, Any]: """Read bounded public/authorized open PR signal metadata without side effects.""" + normalized = normalize_github_repo_slug(repo_slug) + if not normalized or "/" not in normalized: + return {"status": "unavailable", "repository": repo_slug} + owner, name = normalized.split("/", 1) + by_number: dict[int, dict[str, Any]] = {} + cursor: str | None = None try: - completed = runner( - [ - "gh", "pr", "list", "--repo", repo_slug, "--state", "open", - "--limit", "100", "--json", GITHUB_PR_LIST_FIELDS, - ], - capture_output=True, text=True, check=False, timeout=15, - ) - if completed.returncode != 0: - return {"status": "unavailable", "repository": repo_slug} - rows = json.loads(completed.stdout or "[]") - if not isinstance(rows, list): - return {"status": "unavailable", "repository": repo_slug} - items = [ - summarize_github_pr_signal(row) - for row in sorted( - (r for r in rows if isinstance(r, Mapping)), - key=lambda r: int(r.get("number") or 0), + for page_index in range(_GITHUB_PR_LIST_MAX_PAGES): + cmd = [ + "gh", + "api", + "graphql", + "-f", + f"query={_GITHUB_OPEN_PRS_GRAPHQL_QUERY}", + "-f", + f"owner={owner}", + "-f", + f"name={name}", + ] + if cursor is not None: + cmd.extend(["-f", f"cursor={cursor}"]) + completed = runner( + cmd, + capture_output=True, + text=True, + check=False, + timeout=20, ) - ] + if completed.returncode != 0: + return {"status": "unavailable", "repository": repo_slug} + payload = json.loads(completed.stdout or "{}") + if not isinstance(payload, Mapping) or payload.get("errors"): + return {"status": "unavailable", "repository": repo_slug} + data = payload.get("data") + repo = data.get("repository") if isinstance(data, Mapping) else None + pull_requests = repo.get("pullRequests") if isinstance(repo, Mapping) else None + nodes = pull_requests.get("nodes") if isinstance(pull_requests, Mapping) else None + if not isinstance(nodes, list): + return {"status": "unavailable", "repository": repo_slug} + for node in nodes: + if not isinstance(node, Mapping): + continue + number = int(node.get("number") or 0) + if number > 0: + by_number[number] = summarize_github_pr_signal(_normalize_graphql_pr_node(node)) + page_info = pull_requests.get("pageInfo") + if not isinstance(page_info, Mapping) or page_info.get("hasNextPage") is not True: + break + next_cursor = str(page_info.get("endCursor") or "").strip() + if not next_cursor or next_cursor == cursor: + if page_index < _GITHUB_PR_LIST_MAX_PAGES - 1: + return {"status": "unavailable", "repository": repo_slug} + break + cursor = next_cursor + items = [by_number[num] for num in sorted(by_number)] return {"status": "ok", "repository": repo_slug, "items": items} except (OSError, ValueError, TypeError, subprocess.TimeoutExpired): return {"status": "unavailable", "repository": repo_slug} diff --git a/loopx/capabilities/native_chat/steward_monitor.py b/loopx/capabilities/native_chat/steward_monitor.py index 18372f55b6..4bfb11b612 100644 --- a/loopx/capabilities/native_chat/steward_monitor.py +++ b/loopx/capabilities/native_chat/steward_monitor.py @@ -445,13 +445,30 @@ def pump_steward_monitors(controller: Any, *, cancelled=lambda: False) -> list[d _LOG.warning("An original steward report is unavailable; its window and other work are retained") due = service.due_monitors(actor) if due: - todo = due[0] - accepted, created = StewardTurnHost(controller).submit(binding["binding_id"], - turn_instance_id=monitor_turn_id(todo), todo_id=todo["todo_id"], work_kind="monitor", - monitor_deadline=todo["next_due_at"]) - if created: - changed.append(accepted) - continue + dispatched = False + for todo in due: + try: + accepted, created = StewardTurnHost(controller).submit( + binding["binding_id"], + turn_instance_id=monitor_turn_id(todo), + todo_id=todo["todo_id"], + work_kind="monitor", + monitor_deadline=todo["next_due_at"], + ) + except (OSError, ValueError, KeyError, TypeError, RuntimeError): + _LOG.warning( + "A native steward monitor is unavailable; original work retained" + ) + continue + if created: + changed.append(accepted) + dispatched = True + break + if isinstance(accepted, dict) and accepted.get("status") in {"queued", "running"}: + dispatched = True + break + if dispatched: + continue for adv in service.runnable_advancement_todos(actor): from .steward_turn import native_steward_client_id turn_id_token = advancement_turn_id(adv) @@ -473,8 +490,9 @@ def pump_steward_monitors(controller: Any, *, cancelled=lambda: False) -> list[d if all_monitors: soonest = min(all_monitors, key=lambda row: datetime.fromisoformat(row["next_due_at"].replace("Z", "+00:00"))) due_dt = datetime.fromisoformat(soonest["next_due_at"].replace("Z", "+00:00")) - remaining = max(0, (due_dt - datetime.now(timezone.utc)).total_seconds()) - cache[binding["binding_id"]] = time.monotonic() + min(60, remaining) + remaining = (due_dt - datetime.now(timezone.utc)).total_seconds() + if remaining > 0: + cache[binding["binding_id"]] = time.monotonic() + min(60, remaining) except (OSError, ValueError, KeyError, TypeError, RuntimeError): # One unreadable conversation or unavailable provider must not starve # other native monitors; retain its original data and deadline. diff --git a/loopx/control_plane/collaboration/steward_managed_mode.ts b/loopx/control_plane/collaboration/steward_managed_mode.ts index 391e689f88..05ad062b08 100644 --- a/loopx/control_plane/collaboration/steward_managed_mode.ts +++ b/loopx/control_plane/collaboration/steward_managed_mode.ts @@ -62,10 +62,26 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se && Array.isArray(basis.phases) && basis.phases.length === 1 && requireJsonObject(basis.phases[0], "original watch phase").phase === "manager_context" && Array.isArray(events) && events.filter(kind => kind === "agent.phase").length === 1; + // A custom signal monitor may fail inside the read-only external signal + // check after manager_context completes and before any poll receipt is written. + // Allow at most one bounded recovery when zero-effect proofs hold. + let signalUnavailable = false; + if (original.error_code === "runtime_error" + && original.error === "steward monitor external signal source temporarily unavailable" + && requireJsonObject(prior.actor, "original watch actor").schema_version === "loopx_steward_managed_actor_v0" + && Array.isArray(basis.contexts) && basis.contexts.length === 1 + && Array.isArray(basis.phases) && basis.phases.length === 1 + && requireJsonObject(basis.phases[0], "original watch phase").phase === "manager_context" + && Array.isArray(events) && events.filter(kind => kind === "agent.phase").length === 1 + && Array.isArray(events) && events.filter(kind => kind === "manager.context").length === 1) { + const context = requireJsonObject(basis.contexts[0], "signal unavailable watch context"); + signalUnavailable = context.schema_version === "manager_turn_context_v1"; + } if (Object.keys(recovery).join() !== "original_turn_id" || recovery.original_turn_id !== original.turn_id || typeof original.turn_id !== "string" || !/^[a-f0-9]{32}$/.test(original.turn_id) || original.session_id !== session.session_id - || original.status !== "failed" || !(original.error_code === "server_restarted" || scopeRefused || identityRefused) + || original.status !== "failed" + || !(original.error_code === "server_restarted" || scopeRefused || identityRefused || signalUnavailable) || original.origin !== "native_loopx" || original.loopx_execution !== true || original.response !== null || original.upstream_turn_id !== null || original.delta_count !== 0 || prior.work_kind !== "monitor" || request.work_kind !== "monitor" || request.validation_command !== null diff --git a/loopx/control_plane/quota/monitor_poll_admission.ts b/loopx/control_plane/quota/monitor_poll_admission.ts index 0e05fa9e4d..7ce319ae3f 100644 --- a/loopx/control_plane/quota/monitor_poll_admission.ts +++ b/loopx/control_plane/quota/monitor_poll_admission.ts @@ -69,11 +69,14 @@ function dueMonitorAllowed( ): boolean { if (!facts.must_attempt_work) return false; const candidate = decision.due_monitor_candidates.find((item) => candidateMatches(item, observation)); - if (facts.obligation === "attempt_due_monitor") return Boolean(candidate); - if (facts.reason_codes.has("due_monitor_context") && candidate) return true; const registry = decision.registry_due_monitor; - return facts.reason_codes.has("due_monitor_context") && registry.due === true && + const registryAllowed = registry.due === true && registry.claimed_by === decision.agent_id && candidateMatches(registry, observation); + if (facts.obligation === "attempt_due_monitor") { + return Boolean(candidate) || (!decision.requires_user_action && registryAllowed); + } + if (facts.reason_codes.has("due_monitor_context") && candidate) return true; + return facts.reason_codes.has("due_monitor_context") && registryAllowed; } function monitorGateAllowsObservation(decision: MonitorDecision): boolean { diff --git a/loopx/extensions/lark/private_conversations.py b/loopx/extensions/lark/private_conversations.py index 516f826685..8798c9e3e8 100644 --- a/loopx/extensions/lark/private_conversations.py +++ b/loopx/extensions/lark/private_conversations.py @@ -12,6 +12,7 @@ import re import subprocess import time +import zlib from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone @@ -123,6 +124,8 @@ def __init__(self, *, controller: Any, runtime_root: Path, runner: Any, cli_bin: self.root = controller.store.root / "lark-private-deliveries" self._context_recovery_probe_at: dict[Path, float] = {} self._historical_image_probe_at: dict[Path, float] = {} + self._planned_repairs_stamp: tuple[bool, int | None] | None = None + self._planned_repairs_names: frozenset[str] = frozenset() self._delivery_discovery = PrivateDeliveryDiscovery(self.root) self._body_discovery = PrivateDeliveryDiscovery(self.root) self._web_projection_scan_at = 0.0 @@ -1525,6 +1528,27 @@ def authorized() -> bool: _atomic_write_json(path, record) return 1 + def _planned_historical_repairs(self) -> frozenset[str]: + repairs_dir = self.root / "historical-answer-images" + try: + stamp: tuple[bool, int | None] = (True, repairs_dir.stat().st_mtime_ns) + except OSError: + stamp = (False, None) + if stamp == getattr(self, "_planned_repairs_stamp", None): + return getattr(self, "_planned_repairs_names", frozenset()) + if not stamp[0]: + names: frozenset[str] = frozenset() + else: + names = frozenset(item.name for item in repairs_dir.glob("*.json")) + try: + stamp = (True, repairs_dir.stat().st_mtime_ns) + except OSError: + stamp = (False, None) + names = frozenset() + self._planned_repairs_stamp = stamp + self._planned_repairs_names = names + return names + def pending_body_paths(self) -> list[Path]: """Find unsent bodies without waiting for historical recovery probes.""" paths = [path for path, row, _ in self._body_discovery.rows() @@ -1537,7 +1561,7 @@ def pending_delivery_paths(self) -> list[Path]: paths = [] recovery_candidate = self.core.context_recovery_observer() now = datetime.now(timezone.utc).timestamp() - planned_repairs = {item.name for item in (self.root / "historical-answer-images").glob("*.json")} + planned_repairs = self._planned_historical_repairs() for path, row, repairs in self._delivery_discovery.rows(): if row["status"] != "delivered": if triage.pending(row) and not _answer_image_retry_waits(row, now): @@ -1552,8 +1576,14 @@ def pending_delivery_paths(self) -> list[Path]: # Only a discovery/backoff hint, never execution authority. # The canonical request, provider effect inventory and source # are independently re-read by recover_failed_request. - self._context_recovery_probe_at[path] = time.monotonic() + 60 + # Stagger steady-state ineligible re-probes across [60, 120)s by filename + # hash so cold-start bursts do not stay synchronized on the same tick, + # while keeping eligible candidates on the exact 60s retry cadence. + now_mono = time.monotonic() + stagger = zlib.crc32(path.name.encode("utf-8")) % 60 + self._context_recovery_probe_at[path] = now_mono + 60 + stagger if repairs or recovery_candidate(row["request_ref"]): + self._context_recovery_probe_at[path] = now_mono + 60 paths.append(path) return [*paths, *self._web_projection_paths()] diff --git a/tests/control_plane_ts/quota_monitor_poll_commit.test.ts b/tests/control_plane_ts/quota_monitor_poll_commit.test.ts index 69d7c418ed..62e85380e7 100644 --- a/tests/control_plane_ts/quota_monitor_poll_commit.test.ts +++ b/tests/control_plane_ts/quota_monitor_poll_commit.test.ts @@ -291,6 +291,67 @@ test("admission revalidates due, external, and exact blocked-wait modes", async ); }); +test("attempt_due_monitor admits secondary claimed due monitors via registry_due_monitor unless user gate applies", async () => { + const secondaryCandidate = { + todo_id: "todo_secondary_monitor", + target_key: "secondary-watch", + task_class: "continuous_monitor", + claimed_by: "codex-main-control", + due: true, + }; + const baseDecision = (overrides: Record = {}) => decision({ + should_run: true, + agent_id: "codex-main-control", + effective_action: "normal_run", + requires_user_action: false, + work_lane_contract: { + must_attempt_work: true, + obligation: "attempt_due_monitor", + }, + due_monitor_candidates: [{ + todo_id: "todo_primary_monitor", + target_key: "primary-watch", + task_class: "continuous_monitor", + }], + registry_due_monitor: secondaryCandidate, + ...overrides, + }); + const admitted = await evaluateQuotaMonitorPollCommit(request({ + decision: baseDecision(), + observation: observation({ + todo_id: "todo_secondary_monitor", + target_key: "secondary-watch", + result_hash: "unchanged-secondary", + }), + })); + assert.equal( + (admitted.record?.monitor_event as Record).monitor_mode, + "due_monitor_observed_without_material_transition", + ); + + for (const invalid of [ + { requires_user_action: true }, + { registry_due_monitor: { ...secondaryCandidate, due: false } }, + { registry_due_monitor: { ...secondaryCandidate, claimed_by: "other-agent" } }, + ]) { + await assert.rejects( + () => evaluateQuotaMonitorPollCommit(request({ + decision: baseDecision(invalid), + observation: observation({ + todo_id: "todo_secondary_monitor", + target_key: "secondary-watch", + result_hash: "unchanged-secondary", + }), + })), + (error: unknown) => { + assert.ok(error instanceof EffectRuntimeRequestError); + assert.equal(error.code, "monitor_poll_admission_rejected"); + return true; + }, + ); + } +}); + test("Todo preflight rejects invalid admission before journaling provider intent", async (t) => { const runtimeRoot = await tempRuntime(t); const effectId = "quota-monitor-poll:rejected-provider"; diff --git a/tests/control_plane_ts/steward_managed_mode.test.ts b/tests/control_plane_ts/steward_managed_mode.test.ts index fea6e2422e..2ee64fe2a8 100644 --- a/tests/control_plane_ts/steward_managed_mode.test.ts +++ b/tests/control_plane_ts/steward_managed_mode.test.ts @@ -82,6 +82,45 @@ test("identity-refused watches need independent collector evidence and retain cu } }); +test("signal-unavailable custom monitors admit exactly one zero-effect recovery after manager_context", () => { + const {params, goal} = fixture(); + const actor = bindStewardManagedActor({...params, goal}); + const prior = {operation: "native_turn", host: "codex-app-server", actor, + work_kind: "monitor", turn_instance_id: "monitor.signal", todo_id: "todo_signal_watch", validation_command: null, + monitor_deadline: "2026-11-19T13:21:54Z", monitor_lease_version: 0}; + const original = {turn_id: "2".repeat(32), session_id: params.session.session_id, + status: "failed", error_code: "runtime_error", + error: "steward monitor external signal source temporarily unavailable", + origin: "native_loopx", loopx_execution: true, upstream_turn_id: null, response: null, + delta_count: 0, loopx_request: prior}; + const basis = {original, contexts: [{schema_version: "manager_turn_context_v1", goals: [goal]}], + phases: [{phase: "manager_context"}], + event_kinds: ["turn.queued", "agent.phase", "manager.context", "turn.failed"], + poll_absent: true, old_worker_excluded: true, lease_available: true}; + const monitor = {todo_id: "todo_signal_watch", task_class: "continuous_monitor", + target_key: "steward-stalled-pr-monitor", claimed_by: "steward", status: "open", + next_due_at: prior.monitor_deadline}; + const input = {...params, goal, goal_active: true, binding: {...params.binding, managed_actor: actor}, + request: {...prior, monitor_restart_recovery: {original_turn_id: original.turn_id}}, + monitor_todo: monitor, monitor_restart_basis: basis}; + assert.equal(admitStewardManagedTurn(input).observation_only, true); + + for (const changed of [ + {contexts: []}, + {contexts: [{schema_version: "wrong_schema"}]}, + {contexts: [...basis.contexts, ...basis.contexts]}, + {phases: [{phase: "model_start"}]}, + {event_kinds: ["turn.queued", "agent.phase", "turn.failed"]}, + {poll_absent: false}, + {lease_available: false}, + {original: {...original, error: "other transient failure"}}, + {original: {...original, upstream_turn_id: "model-started"}}, + {original: {...original, loopx_request: {...prior, monitor_restart_recovery: {original_turn_id: "3".repeat(32)}}}}, + ]) { + assert.throws(() => admitStewardManagedTurn({...input, monitor_restart_basis: {...basis, ...changed}})); + } +}); + test("monitor configuration retains the original actor and native Todo authority", () => { const {params, goal} = fixture(); const actor = bindStewardManagedActor({...params, goal}); diff --git a/tests/test_lark_steward_directory_scan_perf.py b/tests/test_lark_steward_directory_scan_perf.py index aee1ff8b8f..645f5e470d 100644 --- a/tests/test_lark_steward_directory_scan_perf.py +++ b/tests/test_lark_steward_directory_scan_perf.py @@ -298,3 +298,63 @@ def test_external_conversations_recover_does_not_retain_delivered_image_payloads retained_bytes = sum(stat.size_diff for stat in after.compare_to(before, "lineno") if stat.size_diff > 0) assert retained_bytes < 512 * 1024, f"recover() retained {retained_bytes} bytes of delivered image payloads" + + +def test_private_conversations_caches_historical_repairs_and_staggers_recovery_probes(tmp_path, monkeypatch): + from loopx.extensions.lark import private_conversations as private + from loopx.extensions.lark.private_delivery_discovery import PrivateDeliveryDiscovery + + repairs_dir = tmp_path / "historical-answer-images" + repairs_dir.mkdir() + _atomic_write_json(repairs_dir / f"{_hex_ref(1)}.json", {"turn_id": "t1"}) + + clock = [100.0] + monkeypatch.setattr(private, "time", SimpleNamespace(monotonic=lambda: clock[0])) + + transport = private.LarkPrivateConversations.__new__(private.LarkPrivateConversations) + transport.root = tmp_path + transport._context_recovery_probe_at = {} + transport._historical_image_probe_at = {} + transport._delivery_discovery = PrivateDeliveryDiscovery(tmp_path) + transport._web_projection_paths = lambda: [] + transport.core = SimpleNamespace(context_recovery_observer=lambda: lambda _ref: False) + + for idx in range(2, 62): + ref = _hex_ref(idx) + _atomic_write_json( + tmp_path / f"{ref}.json", + { + "status": "delivered", + "request_ref": ref, + "session_id": f"sess_{idx}", + "turn_id": f"turn_{idx}", + "profile": "fixture", + "event": {"message_id": f"om_{idx}"}, + }, + ) + + glob_calls = 0 + original_glob = Path.glob + + def counted_glob(self: Path, pattern: str): + nonlocal glob_calls + if self == repairs_dir: + glob_calls += 1 + return original_glob(self, pattern) + + monkeypatch.setattr(Path, "glob", counted_glob) + + assert transport.pending_delivery_paths() == [] + assert glob_calls == 1 + assert transport.pending_delivery_paths() == [] + assert glob_calls == 1, "unchanged historical-answer-images dir must hit mtime cache without globbing" + + # Adding a new repair manifest updates dir mtime_ns and refreshes the cached set once. + _atomic_write_json(repairs_dir / f"{_hex_ref(2)}.json", {"turn_id": "t2"}) + assert transport.pending_delivery_paths() == [tmp_path / f"{_hex_ref(2)}.json"] + assert glob_calls == 2 + + # Delivered probe deadlines must stagger across [60, 120)s rather than locking every file onto T+60. + deadlines = set(transport._context_recovery_probe_at.values()) + assert len(deadlines) > 10 + assert all(160.0 <= value < 220.0 for value in deadlines) diff --git a/tests/test_native_steward_monitor.py b/tests/test_native_steward_monitor.py index 6051b0dffb..ba51e97a76 100644 --- a/tests/test_native_steward_monitor.py +++ b/tests/test_native_steward_monitor.py @@ -1247,3 +1247,128 @@ def test_fresh_scope_includes_a_new_stopped_goal_without_reusing_or_rewriting_mo assert not result["response"]["native_loopx"]["monitor"]["material_change"] assert capture.read_bytes() == calls and store.load_session(sid)["manager_authorization_scope_id"] == original_scope assert monitor.current(actor)["result_hash"] == first_hash + + +def test_signal_unavailable_monitor_recovers_once_and_does_not_starve_other_due_monitors( + steward, tmp_path, monkeypatch, +): + from test_steward_generic_monitor_todo import _set_signal_source_policy + + store, runtime, _, transport, binding, _, _ = steward + sid, actor, monitor = prepared(steward, tmp_path, monkeypatch, "sqlite") + session = store.load_session(sid) + _set_signal_source_policy(runtime, session, binding, goal_id=actor["goal_id"], agent_id=actor["agent_id"]) + + # Push the default portfolio watch into the future so our two custom monitors control due ordering. + default_watch = monitor.current(actor) + edit_local_watch( + runtime, + actor, + default_watch["todo_id"], + "--next-due-at", + (datetime.now(timezone.utc) + timedelta(hours=2)).isoformat(), + ) + + from loopx.capabilities.native_chat.steward_commands import native_command + + first_due = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat() + second_due = (datetime.now(timezone.utc) - timedelta(seconds=5)).isoformat() + added_first = native_command( + runtime, + [ + "todo", "add", "--goal-id", actor["goal_id"], "--role", "agent", + "--text", "Monitor open PR signals", + "--claimed-by", actor["agent_id"], + "--task-class", "continuous_monitor", + "--action-kind", "local_runtime_observe", + "--continuation-policy", "same_agent_non_delivery", + "--required-capability", "filesystem", + "--target-key", "steward-stalled-pr-monitor", + "--task-repository", "git:github.com/org/repo", + "--watch-only", + "--cadence", "30m", + "--next-due-at", first_due, + "--execute", + ], + ) + added_second = native_command( + runtime, + [ + "todo", "add", "--goal-id", actor["goal_id"], "--role", "agent", + "--text", "Monitor secondary inbox state", + "--claimed-by", actor["agent_id"], + "--task-class", "continuous_monitor", + "--action-kind", "local_runtime_observe", + "--continuation-policy", "same_agent_non_delivery", + "--required-capability", "filesystem", + "--target-key", "steward-secondary-watch", + "--watch-only", + "--cadence", "30m", + "--next-due-at", second_due, + "--execute", + ], + ) + pr_todo_id = added_first["todo_id"] + secondary_todo_id = added_second["todo_id"] + transport.reconcile() + + # 1. Fail due[0] inside _observe_github_repository_prs before any poll receipt. + calls = {"count": 0} + + def flaky_repo_observe(repo_slug, **_kwargs): + calls["count"] += 1 + if calls["count"] == 1: + return {"status": "unavailable", "repository": repo_slug} + return {"status": "ok", "repository": repo_slug, "items": []} + + monkeypatch.setattr( + "loopx.capabilities.native_chat.steward_monitor._observe_github_repository_prs", + flaky_repo_observe, + ) + runtime._steward_monitor_next_check.clear() + initial = pump_steward_monitors(runtime) + assert len(initial) == 1 and initial[0]["loopx_request"]["todo_id"] == pr_todo_id + failed = finish(runtime, initial[0]) + assert failed["status"] == "failed" + assert failed["error_code"] == "runtime_error" + assert failed["error"] == "steward monitor external signal source temporarily unavailable" + + # 2. If due[0] stays unavailable on recovery, it executes its ONE bounded retry and stops failing over itself. + # Replace flaky mock to fail a second time so we prove 1-retry cap AND non-starvation of due[1]! + monkeypatch.setattr( + "loopx.capabilities.native_chat.steward_monitor._observe_github_repository_prs", + lambda repo_slug, **_kw: {"status": "unavailable", "repository": repo_slug}, + ) + runtime._steward_monitor_next_check.clear() + recovery = pump_steward_monitors(runtime) + assert len(recovery) == 1 + assert recovery[0]["loopx_request"]["todo_id"] == pr_todo_id + assert recovery[0]["loopx_request"]["monitor_restart_recovery"]["original_turn_id"] == failed["turn_id"] + failed_again = finish(runtime, recovery[0]) + assert failed_again["status"] == "failed" + + # 3. Next pump skips the unrecoverable due[0] and dispatches due[1] instead of starving it! + runtime._steward_monitor_next_check.clear() + next_due = pump_steward_monitors(runtime) + assert len(next_due) == 1 and next_due[0]["loopx_request"]["todo_id"] == secondary_todo_id + done_secondary = finish(runtime, next_due[0]) + assert done_secondary["status"] == "completed", done_secondary + assert local_watch_row(runtime, actor, secondary_todo_id)["next_due_at"] != second_due + + # 4. Also verify recovery succeeds when GitHub recovers after the first failure. + third_due = (datetime.now(timezone.utc) - timedelta(seconds=3)).isoformat() + edit_local_watch(runtime, actor, pr_todo_id, "--next-due-at", third_due) + monkeypatch.setattr( + "loopx.capabilities.native_chat.steward_monitor._observe_github_repository_prs", + flaky_repo_observe, + ) + calls["count"] = 0 + runtime._steward_monitor_next_check.clear() + first_try = pump_steward_monitors(runtime) + assert len(first_try) == 1 and finish(runtime, first_try[0])["status"] == "failed" + runtime._steward_monitor_next_check.clear() + recovered_try = pump_steward_monitors(runtime) + assert len(recovered_try) == 1 + recovered_done = finish(runtime, recovered_try[0]) + assert recovered_done["status"] == "completed", recovered_done + assert local_watch_row(runtime, actor, pr_todo_id)["next_due_at"] != third_due diff --git a/tests/test_steward_pr_merge_and_review_signal.py b/tests/test_steward_pr_merge_and_review_signal.py index e8e96ba746..0d216334da 100644 --- a/tests/test_steward_pr_merge_and_review_signal.py +++ b/tests/test_steward_pr_merge_and_review_signal.py @@ -202,36 +202,120 @@ def test_summarize_pr_distinguishes_gpt_gemini_fallback_stale_casual_and_conflic assert conflicted["review_summary"]["effective_verdict"] == "REQUEST_CHANGES" +def _graphql_page( + nodes: list[dict[str, Any]], *, has_next_page: bool = False, end_cursor: str | None = None +) -> dict[str, Any]: + return { + "data": { + "repository": { + "pullRequests": { + "pageInfo": {"hasNextPage": has_next_page, "endCursor": end_cursor}, + "nodes": nodes, + } + } + } + } + + +def _graphql_node(row: dict[str, Any], *, contexts: list[dict[str, Any]] | None = None) -> dict[str, Any]: + copied = dict(row) + default_contexts = copied.pop("statusCheckRollup", []) + rollup_contexts = contexts if contexts is not None else default_contexts + copied["reviews"] = {"nodes": copied.get("reviews") or []} + copied["commits"] = { + "nodes": [ + { + "commit": { + "statusCheckRollup": { + "contexts": { + "nodes": rollup_contexts, + } + } + } + } + ] + } + return copied + + def test_observe_github_repository_prs_enriches_signal_summary(monkeypatch) -> None: - head = "c" * 40 + head_99 = "c" * 40 + head_101 = "1" * 40 gpt_decl = "model_agent · gpt-6.2-sol · OpenAI (runtime_reported, max)" - payload = [ + pr_99 = _graphql_node( _sample_pr( number=99, - head_oid=head, + head_oid=head_99, reviews=[ { "id": "PRR_99", "author": {"login": "loopx-agent"}, "state": "COMMENTED", - "commit": {"oid": head}, + "commit": {"oid": head_99}, "submittedAt": "2030-01-01T08:30:00Z", "body": _valid_bilingual_body( - head_oid=head, verdict="REQUEST_CHANGES", reviewer=gpt_decl, author_owned=True + head_oid=head_99, verdict="REQUEST_CHANGES", reviewer=gpt_decl, author_owned=True ), } ], ) - ] + ) + pr_101 = _graphql_node( + _sample_pr(number=101, head_oid=head_101), + contexts=[ + {"__typename": "CheckRun", "name": "ci", "status": "COMPLETED", "conclusion": "SUCCESS"}, + {"__typename": "StatusContext", "context": "gate", "state": "FAILURE"}, + ], + ) + seen_cursors: list[str | None] = [] def fake_run(cmd, **_kwargs): - assert cmd[:3] == ["gh", "pr", "list"] - return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + assert cmd[:3] == ["gh", "api", "graphql"] + cursor_args = [arg for arg in cmd if arg.startswith("cursor=")] + cursor = cursor_args[0].split("=", 1)[1] if cursor_args else None + seen_cursors.append(cursor) + if cursor is None: + body = _graphql_page([pr_101], has_next_page=True, end_cursor="cursor-page-2") + else: + assert cursor == "cursor-page-2" + # Include duplicate PR 101 across page shift + new PR 99 to test deduplication & sorting. + body = _graphql_page([pr_101, pr_99], has_next_page=False, end_cursor="cursor-end") + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(body), stderr="") monkeypatch.setattr("loopx.capabilities.native_chat.steward_monitor.subprocess.run", fake_run) observed = _observe_github_repository_prs("org/repo") - assert observed["status"] == "ok" and len(observed["items"]) == 1 + assert seen_cursors == [None, "cursor-page-2"] + assert observed["status"] == "ok" and [item["number"] for item in observed["items"]] == [99, 101] assert observed["items"][0]["review_summary"]["effective_verdict"] == "REQUEST_CHANGES" + assert observed["items"][1]["checks_summary"] == { + "total": 2, + "pass_count": 1, + "fail_count": 1, + "pending_count": 0, + } + + +def test_observe_github_repository_prs_fails_closed_on_page_error(monkeypatch) -> None: + calls = 0 + + def fake_run(cmd, **_kwargs): + nonlocal calls + calls += 1 + if calls == 1: + body = _graphql_page([_graphql_node(_sample_pr(number=10))], has_next_page=True, end_cursor="p2") + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(body), stderr="") + return subprocess.CompletedProcess(cmd, 1, stdout="", stderr="502 Bad Gateway") + + monkeypatch.setattr("loopx.capabilities.native_chat.steward_monitor.subprocess.run", fake_run) + observed = _observe_github_repository_prs("org/repo") + assert observed == {"status": "unavailable", "repository": "org/repo"} + assert calls == 2 + + def graphql_errors_run(cmd, **_kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps({"errors": [{"message": "timeout"}]}), stderr="") + + monkeypatch.setattr("loopx.capabilities.native_chat.steward_monitor.subprocess.run", graphql_errors_run) + assert _observe_github_repository_prs("org/repo") == {"status": "unavailable", "repository": "org/repo"} def test_inspect_github_pull_request_attaches_rest_html_url() -> None: