From efdc510330fc04ea0de0a26558c4b07e3c98bd36 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:16:35 +0800 Subject: [PATCH] fix(runtime): bound pytest deadlocks and Antigravity Stop-hook trajectory scan Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/agy_goal_mode/host_runtime.py | 83 ++++++++++++++++++-------- pyproject.toml | 3 + tests/test_antigravity_host_runtime.py | 71 ++++++++++++++++++++++ tests/test_python_ci_workflow.py | 34 +++++++++++ 4 files changed, 165 insertions(+), 26 deletions(-) diff --git a/loopx/agy_goal_mode/host_runtime.py b/loopx/agy_goal_mode/host_runtime.py index c9ccd9953d..dc41e2e10e 100644 --- a/loopx/agy_goal_mode/host_runtime.py +++ b/loopx/agy_goal_mode/host_runtime.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +from collections.abc import Iterator, Mapping from contextlib import contextmanager import hashlib import json @@ -15,20 +16,22 @@ import subprocess import sys import time -from typing import Any from ..command_invocation import loopx_cli_argv from ..entrypoint import pin_utf8_stdio from ..file_lock import exclusive_file_lock +_TRAJECTORY_TAIL_WINDOW = 200 +_TRAJECTORY_SCAN_DEADLINE_SECONDS = 8.0 -def retryable_native_error(error: Any) -> bool: + +def retryable_native_error(error: object) -> bool: """Conservative mapping of the native provider's unstructured error field. Only known Google model transport errors qualify. User/model prose, quota, auth, cancellation and permission errors do not establish retry authority. """ - if isinstance(error, dict): + if isinstance(error, Mapping): error = error.get("shortError") if not isinstance(error, str): return False @@ -92,13 +95,14 @@ def bootstrap_prompt(args: argparse.Namespace) -> str: @contextmanager -def _state(args: argparse.Namespace): +def _state(args: argparse.Namespace) -> Iterator[dict[str, object]]: # Native conversation ids never become path components. key = hashlib.sha256(args.conversation_id.encode()).hexdigest() path = Path(args.state_dir).expanduser() / (key + ".json") path.parent.mkdir(parents=True, exist_ok=True) with exclusive_file_lock(path, timeout_seconds=2): - data = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + loaded = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + data: dict[str, object] = dict(loaded) if isinstance(loaded, Mapping) else {} yield data temporary = path.with_suffix(".tmp") with temporary.open("w", encoding="utf-8") as stream: @@ -109,12 +113,20 @@ def _state(args: argparse.Namespace): temporary.replace(path) -def _reserve_retry(state: dict[str, Any], input_key: str) -> bool: +def _reserve_retry(state: dict[str, object], input_key: str) -> bool: if state.get("input_key") != input_key: state.update(input_key=input_key, retries=0) - if state.get("retries", 0) >= 2 or time.time() - state.get("retry_at", 0) < 60: + retries_raw = state.get("retries") + retries = retries_raw if type(retries_raw) is int else 0 + retry_at_raw = state.get("retry_at") + retry_at = ( + float(retry_at_raw) + if isinstance(retry_at_raw, (int, float)) and not isinstance(retry_at_raw, bool) + else 0.0 + ) + if retries >= 2 or time.time() - retry_at < 60: return False - state.update(retries=state.get("retries", 0) + 1, retry_at=time.time()) + state.update(retries=retries + 1, retry_at=time.time()) return True @@ -153,29 +165,48 @@ def _native_input(args: argparse.Namespace, text: str) -> tuple[str, str, int] | or trajectory.get("cascadeId") != args.conversation_id ): return None - raw = _rpc_post( - url, - token, - "GetCascadeTrajectorySteps", - {**payload, "startIndex": 0, "endIndex": total}, - timeout=2, - ).get("steps") - if not isinstance(raw, list) or len(raw) != total: + deadline = time.monotonic() + _TRAJECTORY_SCAN_DEADLINE_SECONDS + inputs: list[tuple[int, dict[str, object]]] = [] + end = total + while end > 0: + if time.monotonic() >= deadline: + return None + start = max(0, end - _TRAJECTORY_TAIL_WINDOW) + raw = _rpc_post( + url, + token, + "GetCascadeTrajectorySteps", + {**payload, "startIndex": start, "endIndex": end}, + timeout=2, + ).get("steps") + if not isinstance(raw, list) or not raw: + return None + if start > 0 and len(raw) == total: + # Older servers ignore range bounds and return the whole prefix. + indexed, covered = _observed_turn_steps(raw, 0, total) + if covered != total: + return None + start = 0 + else: + indexed, covered = _observed_turn_steps(raw, start, end) + if covered < end: + return None + inputs = [ + (step_index, dict(step)) + for step_index, step in indexed + if isinstance(step, Mapping) and step.get("type") == "CORTEX_STEP_TYPE_USER_INPUT" + ] + if inputs: + break + end = start + if not inputs: return None - indexed, covered = _observed_turn_steps(raw, 0, total) - inputs = [ - (index, step) - for index, step in indexed - if step.get("type") == "CORTEX_STEP_TYPE_USER_INPUT" - ] after = _rpc_post(url, token, "GetCascadeTrajectory", payload, timeout=2) except (ValueError, CodexChatAgentError, OSError): return None trajectory = after.get("trajectory") if ( - not inputs - or covered != total - or after.get("status") != status + after.get("status") != status or after.get("numTotalSteps") != total or not isinstance(trajectory, dict) or trajectory.get("cascadeId") != args.conversation_id @@ -196,7 +227,7 @@ def _native_input(args: argparse.Namespace, text: str) -> tuple[str, str, int] | return hashlib.sha256(anchor.encode()).hexdigest(), status, total -def stop_hook(args: argparse.Namespace, event: dict[str, Any]) -> dict[str, Any]: +def stop_hook(args: argparse.Namespace, event: Mapping[str, object]) -> dict[str, object]: if ( event.get("conversationId") != args.conversation_id or event.get("terminationReason") != "ERROR" diff --git a/pyproject.toml b/pyproject.toml index 623d084858..752de07d1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -201,6 +201,9 @@ include = ["loopx*"] [tool.pytest.ini_options] markers = ["stage2c_e2e: real process Stage 2C correctness and recovery acceptance"] +# Bound deadlocked tests above CI's --ci-stall-after=120s cooperative snapshot so local/subagent runs never hang forever. +faulthandler_timeout = 180 +faulthandler_exit_on_timeout = true # Private snapshots and packaging copies are not source tests; optional provider suites have dedicated workflows. norecursedirs = [".local", "build", "dist", "deprecate", "packages/loopx-ark-turn", "packages/loopx-memt-turn"] diff --git a/tests/test_antigravity_host_runtime.py b/tests/test_antigravity_host_runtime.py index 97f5fa336a..6e8e7eb928 100644 --- a/tests/test_antigravity_host_runtime.py +++ b/tests/test_antigravity_host_runtime.py @@ -365,3 +365,74 @@ def test_hooks_migrate_only_an_unchanged_owned_legacy_entry(tmp_path): == "retired" ) assert json.loads(path.read_text()) == {"personal-hook": unrelated} + + +def test_native_input_reads_bounded_tail_windows_on_long_trajectories( + tmp_path, monkeypatch +): + import hashlib + from loopx.capabilities.native_chat import antigravity_adapter as provider + + monkeypatch.setenv("ANTIGRAVITY_LS_ADDRESS", "127.0.0.1:1234") + monkeypatch.setenv("ANTIGRAVITY_CSRF_TOKEN", "synthetic-token") + text = "Continue" + + def make_steps(total_steps: int, user_index: int | None) -> list[dict[str, object]]: + steps = [ + { + "type": "CORTEX_STEP_TYPE_PLANNER_RESPONSE", + "status": "CORTEX_STEP_STATUS_DONE", + "metadata": {"sourceTrajectoryStepInfo": {"stepIndex": idx}}, + "plannerResponse": {"response": f"step-{idx}"}, + } + for idx in range(total_steps) + ] + if user_index is not None: + steps[user_index] = { + "type": "CORTEX_STEP_TYPE_USER_INPUT", + "status": "CORTEX_STEP_STATUS_DONE", + "metadata": { + "createdAt": "2026-01-01T00:00:00Z", + "sourceTrajectoryStepInfo": {"stepIndex": user_index}, + }, + "userInput": {"userResponse": text}, + } + return steps + + def run_case(total_steps: int, user_index: int | None): + steps = make_steps(total_steps, user_index) + ranges: list[tuple[int, int]] = [] + + def read(url, token, method, payload, **_): + if method == "GetCascadeTrajectorySteps": + start = payload["startIndex"] + end = payload["endIndex"] + assert end - start <= 200, f"unbounded full trajectory read: {(start, end)}" + ranges.append((start, end)) + return {"steps": steps[start:end]} + return { + "trajectory": {"cascadeId": "native-chat"}, + "status": "CASCADE_RUN_STATUS_RUNNING", + "numTotalSteps": len(steps), + } + + monkeypatch.setattr(provider, "_rpc_post", read) + return runtime._native_input(bound(tmp_path), text), ranges + + tail_hit, tail_ranges = run_case(5000, 4985) + assert tail_ranges == [(4800, 5000)] + expected_tail = hashlib.sha256( + json.dumps(["native-chat", 4985, "2026-01-01T00:00:00Z", text]).encode() + ).hexdigest() + assert tail_hit == (expected_tail, "CASCADE_RUN_STATUS_RUNNING", 5000) + + back_hit, back_ranges = run_case(5000, 4550) + assert back_ranges == [(4800, 5000), (4600, 4800), (4400, 4600)] + expected_back = hashlib.sha256( + json.dumps(["native-chat", 4550, "2026-01-01T00:00:00Z", text]).encode() + ).hexdigest() + assert back_hit == (expected_back, "CASCADE_RUN_STATUS_RUNNING", 5000) + + empty_hit, empty_ranges = run_case(500, None) + assert empty_hit is None + assert empty_ranges == [(300, 500), (100, 300), (0, 100)] diff --git a/tests/test_python_ci_workflow.py b/tests/test_python_ci_workflow.py index ab82c69360..f703296e01 100644 --- a/tests/test_python_ci_workflow.py +++ b/tests/test_python_ci_workflow.py @@ -609,3 +609,37 @@ def test_merge_queue_candidates_run_full_qualification() -> None: assert "github.event_name != 'merge_group'" in WORKFLOW.split(" sonar:\n", 1)[1].split(" uses:", 1)[0] # The required check list is unchanged: merge-gate always reports. assert "if: always()" in WORKFLOW.split(" merge-gate:\n", 1)[1] + + +def test_pytest_config_enforces_bounded_deadlock_exit_above_ci_stall_snapshot( + tmp_path: Path, +) -> None: + import tomllib + + config = tomllib.loads((WORKFLOW_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + ini = config["tool"]["pytest"]["ini_options"] + assert ini["faulthandler_timeout"] == 180 + assert ini["faulthandler_timeout"] > 120 + assert ini["faulthandler_exit_on_timeout"] is True + + (tmp_path / "pyproject.toml").write_text( + (WORKFLOW_ROOT / "pyproject.toml").read_text(encoding="utf-8"), + encoding="utf-8", + ) + (tmp_path / "test_hang.py").write_text( + "import threading\n\ndef test_deadlock():\n threading.Event().wait(30)\n", + encoding="utf-8", + ) + started = time.monotonic() + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-o", "faulthandler_timeout=1"], + cwd=tmp_path, + env={k: v for k, v in os.environ.items() if not k.startswith("PYTEST")}, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert time.monotonic() - started < 10 + assert result.returncode != 0 + assert "Timeout" in result.stdout + result.stderr