From e93faf34aa87c241cd9f4b7530db4e7d8576114d Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:53:44 +0800 Subject: [PATCH] fix(steward): tolerate report validator python drift and allow cancel of mid-turn failed reports Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../native_chat/steward_reports.py | 40 +++- .../capabilities/native_chat/steward_turn.py | 24 ++- ...t_steward_report_recovery_and_validator.py | 173 ++++++++++++++++++ 3 files changed, 232 insertions(+), 5 deletions(-) create mode 100644 tests/test_steward_report_recovery_and_validator.py diff --git a/loopx/capabilities/native_chat/steward_reports.py b/loopx/capabilities/native_chat/steward_reports.py index 2f09ca47ea..da82967192 100644 --- a/loopx/capabilities/native_chat/steward_reports.py +++ b/loopx/capabilities/native_chat/steward_reports.py @@ -91,6 +91,38 @@ def _known_generation_failure(self, actor: dict[str, Any], window: dict[str, Any return {"turn_id": turn["turn_id"], "todo_id": request["todo_id"]} return None + def _cancellable_failed_window(self, actor: dict[str, Any], window: dict[str, Any]) -> dict[str, str] | None: + """Identify a terminal failed report attempt that owner cancel may retire.""" + known = self._known_generation_failure(actor, window) + if known is not None: + return known + from .steward_turn import report_turn + folder = attempt_directory(self.root, window) + if any(path.is_symlink() or not path.is_file() + for path in (folder / "source.json", folder / "editorial-request.json")): + return None + for name in ("editorial-response.json", "generation-bundle.json", "report.md", + "publication-candidate.json", "private-generation.json", "private-delivery.json"): + try: + (folder / name).lstat() + except FileNotFoundError: + continue + return None + turn = report_turn(self.controller.store, actor, window) + request = (turn or {}).get("loopx_request") or {} + native = ((turn or {}).get("response") or {}).get("native_loopx") or {} + failure = native.get("host_failure") or {} + if (turn and turn.get("status") == "completed" and native.get("status") == "failed" + and native.get("result_kind") == "host_failure" and failure.get("retryable") is False + and native.get("effects") == {"host_invoked": True, "state_written": False, + "quota_spent": False, "scheduler_acknowledged": False} + and request.get("actor") == actor and request.get("work_kind") == "report" + and request.get("report_window_id") == window["window_id"] + and request.get("report_subscription_revision") == window["subscription_revision"] + and isinstance(request.get("todo_id"), str) and bool(request["todo_id"])): + return {"turn_id": turn["turn_id"], "todo_id": request["todo_id"]} + return None + def _execution_failed(self, actor: dict[str, Any], window: dict[str, Any]) -> bool: """Observe the current attempt; this grants no cancellation or retry.""" from .steward_turn import report_turn @@ -183,7 +215,7 @@ def _cancel_failed_window(self, actor: dict[str, Any], window: dict[str, Any], from ..periodic_report.cadence_journal import ( _retire_failed_cadence_window_locked, retire_failed_cadence_window, ) - if self._known_generation_failure(actor, window) != failure: + if self._cancellable_failed_window(actor, window) != failure: raise ValueError("the original failed report changed before cancellation") native_command(self.controller, ["todo", "update", "--goal-id", actor["goal_id"], "--todo-id", failure["todo_id"], "--agent-id", actor["agent_id"], @@ -222,7 +254,7 @@ def configure(self, binding_id: str, *, operation: str, schedule: dict[str, Any] pending = None failure = None if journal: - failure = self._known_generation_failure(actor, journal["window"]) + failure = self._cancellable_failed_window(actor, journal["window"]) pending = {"subscription_revision": journal["window"]["subscription_revision"], "prepared": (attempt_directory(self.root, journal["window"]) / "editorial-request.json").exists(), "verified_publication": self._published_window(journal["window"]), @@ -484,7 +516,9 @@ def session_report_configuration_guard(controller: Any, goal_id: str, proposed: after = {k: v for k, v in (proposed or {}).items() if k != "enabled"} if before != after or not current and proposed is None: actor = reports._frozen_actor(window) - failure = reports._known_generation_failure(actor, window) if actor else None + failure = (reports._cancellable_failed_window(actor, window) + if hasattr(reports, "_cancellable_failed_window") + else reports._known_generation_failure(actor, window)) if actor else None if (proposed is not None or not failure) and (current or proposed is not None): if window["profile_ref"]["profile_id"] == "session_task": raise PendingStewardReportError("This calendar task is unfinished or its result is unconfirmed. Pause or restore its original plan; only a known failed independent validation can be cleared.") diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index 6cfc3bc961..b5908a7cb7 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -7,7 +7,7 @@ """ from __future__ import annotations -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from contextlib import contextmanager, ExitStack, nullcontext import hashlib import json @@ -21,6 +21,20 @@ from .steward_managed_mode import ManagedSessionTurn, ManagedTaskValidationUnavailable, StewardManagedMode _USER_ACTION_DEDUPE_KEY = re.compile(r"^[a-z0-9][a-z0-9._:@/-]{0,119}$") +_REPORT_VALIDATOR_PREFIX = ( + "from pathlib import Path; from loopx.capabilities.periodic_report.private_generation import validate_private_generation; " + "validate_private_generation(Path(" +) + + +def _same_report_validator_command(frozen_cmd: Any, request_cmd: Any) -> bool: + """Allow report validation commands whose only drift is the host Python path.""" + return (isinstance(frozen_cmd, list) and isinstance(request_cmd, list) + and len(frozen_cmd) == 3 and len(request_cmd) == 3 + and all(isinstance(part, str) and bool(part) for part in [*frozen_cmd, *request_cmd]) + and frozen_cmd[1] == "-c" and request_cmd[1] == "-c" + and frozen_cmd[2] == request_cmd[2] and frozen_cmd[2].startswith(_REPORT_VALIDATOR_PREFIX) + and frozen_cmd[2].endswith("))")) def native_steward_client_id(actor: Mapping[str, Any], turn_instance_id: str) -> str: @@ -547,7 +561,13 @@ def _prepare_submission(self, binding: dict[str, Any] | None, *, turn_instance_i frozen = {key: value for key, value in (existing.get("loopx_request") or {}).items() if key not in {"monitor_lease_version", "monitor_deadline"}} frozen.setdefault("work_kind", "native_turn") - if frozen != request or work_kind == "monitor" and existing["loopx_request"].get("monitor_deadline") != monitor_deadline: + comparable_frozen = ( + {**frozen, "validation_command": request["validation_command"]} + if work_kind == "report" and _same_report_validator_command( + frozen.get("validation_command"), request.get("validation_command")) + else frozen + ) + if comparable_frozen != request or work_kind == "monitor" and existing["loopx_request"].get("monitor_deadline") != monitor_deadline: raise ValueError("native steward request identity conflict") if work_kind == "report": retained = report_turn(self.store, actor, report_window) diff --git a/tests/test_steward_report_recovery_and_validator.py b/tests/test_steward_report_recovery_and_validator.py new file mode 100644 index 0000000000..d602e38bb9 --- /dev/null +++ b/tests/test_steward_report_recovery_and_validator.py @@ -0,0 +1,173 @@ +"""Focused regressions for report validator path portability and cancel recovery.""" +from datetime import datetime, timedelta, timezone +import json +import logging + +import pytest + +from loopx.capabilities.native_chat.steward_monitor import pump_steward_monitors +from loopx.capabilities.native_chat.steward_turn import StewardTurnHost, _same_report_validator_command +from loopx.capabilities.periodic_report import cadence_runtime +from loopx.capabilities.periodic_report.cadence_journal import read_cadence_journal +from loopx.capabilities.periodic_report.incremental import read_periodic_report_publication_cursor +from loopx.capabilities.periodic_report.private_generation import attempt_directory +import loopx.capabilities.native_chat.steward_reports as report_host +from test_native_steward_private import finish, steward as steward_fixture +from test_native_steward_reports import configured, install_model + +steward = steward_fixture + + +def test_same_report_validator_command_tolerates_only_python_executable_drift(): + folder = "/tmp/example-goal/periodic_reports/win-1" + snippet = ( + "from pathlib import Path; " + "from loopx.capabilities.periodic_report.private_generation import validate_private_generation; " + f"validate_private_generation(Path({folder!r}))" + ) + old_cmd = ["/opt/old-release/python3", "-c", snippet] + new_cmd = ["/opt/new-release/python3.12", "-c", snippet] + other_folder_cmd = [ + "/opt/new-release/python3.12", + "-c", + snippet.replace(folder, "/tmp/other-goal/periodic_reports/win-1"), + ] + arbitrary_cmd = ["/opt/new-release/python3.12", "-c", "print('hi')"] + + assert _same_report_validator_command(old_cmd, new_cmd) is True + assert _same_report_validator_command(old_cmd, other_folder_cmd) is False + assert _same_report_validator_command(arbitrary_cmd, arbitrary_cmd) is False + assert _same_report_validator_command(["/opt/old/python", "-m", snippet], new_cmd) is False + + +def test_pump_tolerates_report_validator_python_upgrade_and_still_rejects_target_drift( + steward, tmp_path, monkeypatch, caplog, +): + from loopx.chat_agent import CodexChatAgentError + from loopx.capabilities.native_chat.codex_adapter import CodexAppServerAdapter + + store, runtime, *_ = steward + sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, "sqlite") + point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2) + + class CalendarClock(datetime): + @classmethod + def now(cls, tz=None): + return point.astimezone(tz) if tz else point.replace(tzinfo=None) + + monkeypatch.setattr(report_host, "datetime", CalendarClock) + monkeypatch.setattr(cadence_runtime, "datetime", CalendarClock) + + def fail_mid_turn(adapter, _prompt, event_sink): + event_sink("turn.started", {"upstream_turn_id": "mid-turn-failure-1"}) + raise CodexChatAgentError("Synthetic mid-turn upstream failure.") + + monkeypatch.setattr(CodexAppServerAdapter, "start_turn", fail_mid_turn) + first = finish(runtime, pump_steward_monitors(runtime)[0]) + assert first["response"]["native_loopx"]["status"] == "failed" + + # Rewrite only the recorded Python executable path as happens across local release upgrades. + turn = store.load_turn(sid, first["turn_id"]) + old_req = json.loads(json.dumps(turn["loopx_request"])) + old_req["validation_command"][0] = "/tmp/previous-release/bin/python" + turn["loopx_request"] = old_req + store._turn_path(sid, turn["turn_id"]).write_text(json.dumps(turn)) + + # Avoid app-server stdio probing in this unit-level identity check; keep focus on _prepare_submission. + monkeypatch.setattr(StewardTurnHost, "report_failure_observation", lambda self, original: None) + runtime._steward_monitor_next_check.clear() + with caplog.at_level(logging.WARNING): + assert pump_steward_monitors(runtime) == [] + assert "An original steward report is unavailable" not in caplog.text + + # Mutating the validated folder path must still fail closed as an identity conflict. + _, window = reports.current_window(actor["goal_id"].removeprefix("steward-agent-")) + host = StewardTurnHost(runtime) + binding = host._submission_binding(actor["goal_id"].removeprefix("steward-agent-")) + with pytest.raises(ValueError, match="native steward request identity conflict"): + host._prepare_submission( + binding, + turn_instance_id=old_req["turn_instance_id"], + todo_id=old_req["todo_id"], + validation_command=["/opt/new/python", "-c", old_req["validation_command"][2].replace("'", "x'")], + work_kind="report", + monitor_deadline=None, + report_window=window, + ) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_owner_cancel_retires_mid_turn_failed_report_without_editorial_response_and_advances( + steward, tmp_path, monkeypatch, provider, +): + from loopx.chat_agent import CodexChatAgentError + from loopx.capabilities.native_chat.codex_adapter import CodexAppServerAdapter + from loopx.capabilities.native_chat.steward_monitor import StewardMonitor + + store, runtime, bot, transport, binding, capture, _ = steward + sid, actor, reports, schedule = configured(steward, tmp_path, monkeypatch, provider) + point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2) + + class CalendarClock(datetime): + @classmethod + def now(cls, tz=None): + return point.astimezone(tz) if tz else point.replace(tzinfo=None) + + monkeypatch.setattr(report_host, "datetime", CalendarClock) + monkeypatch.setattr(cadence_runtime, "datetime", CalendarClock) + + def mid_turn_failure(adapter, _prompt, event_sink): + event_sink("turn.started", {"upstream_turn_id": "mid-turn-limit-hit"}) + raise CodexChatAgentError("You've hit your usage limit mid-turn.") + + with monkeypatch.context() as failure: + failure.setattr(CodexAppServerAdapter, "start_turn", mid_turn_failure) + first = finish(runtime, pump_steward_monitors(runtime)[0]) + native = first["response"]["native_loopx"] + assert native["status"] == "failed" + assert native["result_kind"] == "host_failure" + assert native["host_failure"]["retryable"] is False + + old = read_cadence_journal(runtime_root=reports.root, goal_id=actor["goal_id"]) + folder = attempt_directory(reports.root, old["window"]) + assert not (folder / "editorial-response.json").exists() + assert not (folder / "private-delivery.json").exists() + + # Keep _known_generation_failure and health().delivery_status unchanged when no editorial-response exists. + assert reports._known_generation_failure(actor, old["window"]) is None + assert reports.health(binding["binding_id"])["delivery_status"] == "not_generated" + + # Negative check: an unconfirmed private delivery record must still block failure retirement. + delivery_path = folder / "private-delivery.json" + delivery_path.write_text(json.dumps({"delivery_status": "unconfirmed"})) + blocked_cancel = reports.configure(binding["binding_id"], operation="cancel")["plan"] + assert blocked_cancel["retires_failed_window"] is False + delivery_path.unlink() + + # Without delivery or generation artifacts, owner cancel may retire the stuck window cleanly. + cancel = reports.configure(binding["binding_id"], operation="cancel")["plan"] + assert cancel["retires_failed_window"] is True + reports.configure( + binding["binding_id"], + operation="cancel", + execute=True, + expected_plan_id=cancel["plan_id"], + ) + retired = read_cadence_journal(runtime_root=reports.root, goal_id=actor["goal_id"]) + assert retired["retirement"]["source_receipt_id"] == first["turn_id"] + task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == first["loopx_request"]["todo_id"]) + assert task["status"] == "blocked" and not task.get("done") + + # Re-enabling advances cleanly to the next due window and delivers a real report. + install_model(steward, tmp_path, sid) + resume = reports.configure(binding["binding_id"], operation="enable", schedule=schedule)["plan"] + reports.configure(binding["binding_id"], operation="enable", schedule=schedule, execute=True, expected_plan_id=resume["plan_id"]) + point += timedelta(days=1) + runtime._steward_monitor_next_check.clear() + next_turns = pump_steward_monitors(runtime) + assert len(next_turns) == 1 + completed = finish(runtime, next_turns[0]) + assert completed["response"]["native_loopx"]["status"] == "committed", completed + assert transport.steward_reports.reconcile() == 1 + cursor = read_periodic_report_publication_cursor(runtime_root=reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"]) + assert cursor["covered_until"] == old["window"]["next_due_at"]