From f5e5af1d6c788624141535cc844c9f6ea6d5c4fd Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:19:29 +0800 Subject: [PATCH] fix(reports): retain verified terminal failure evidence Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/product/periodic-reports.md | 10 +++ .../native_chat/steward_managed_mode.py | 2 +- .../native_chat/steward_reports.py | 4 +- .../capabilities/native_chat/steward_turn.py | 87 ++++++++++++++++++- tests/test_native_steward_reports.py | 29 ++++++- tests/test_native_steward_turn_host.py | 4 +- 6 files changed, 127 insertions(+), 9 deletions(-) diff --git a/docs/product/periodic-reports.md b/docs/product/periodic-reports.md index 344fc8276e..401bd4a3e5 100644 --- a/docs/product/periodic-reports.md +++ b/docs/product/periodic-reports.md @@ -740,6 +740,16 @@ deadline is not a promise that the held window can run. This observation grants no cancellation, retirement or retry. A correlated quota continuation that succeeds clears the current failure projection while retaining its failed predecessor. +When the original quota check can still verify a terminal Codex failure with +only the original user input, the report attempt records that observation +under its generation lock. The record binds the original Chat and Codex Turns, +the report window and subscription revision, and the original native result +digest. A later independent period can use the record if Codex history is +unavailable, after the existing journal, generation and delivery checks pass. +Quota continuation for the original period still requires a live observation. +Missing or mismatched evidence still leaves a window held when the original +failure cannot be verified. + | Cancelled failure on the original settings page | Narrow retained-failure state | | --- | --- | | ![Cancelled generation is not completed or delivered](../assets/personal-workspace/steward-reports/failed-window-cancelled.png) | ![Retained cancelled window in a narrow viewport](../assets/personal-workspace/steward-reports/cancelled-window-narrow.png) | diff --git a/loopx/capabilities/native_chat/steward_managed_mode.py b/loopx/capabilities/native_chat/steward_managed_mode.py index 34f3098778..0ff867cc78 100644 --- a/loopx/capabilities/native_chat/steward_managed_mode.py +++ b/loopx/capabilities/native_chat/steward_managed_mode.py @@ -449,7 +449,7 @@ def submit(self, session_id: str, *, agent_id: str, operation_id: str, retained = report_turn(self.store, request["actor"], report_window) if retained is not None and retained["turn_id"] != existing["turn_id"]: return retained, False - quota_request = host.report_quota_request(existing) + quota_request = host.report_quota_request(existing, report_window=report_window) if quota_request is not None: request = quota_request client_id += ".quota-recovery" diff --git a/loopx/capabilities/native_chat/steward_reports.py b/loopx/capabilities/native_chat/steward_reports.py index 2f09ca47ea..6d1450d1af 100644 --- a/loopx/capabilities/native_chat/steward_reports.py +++ b/loopx/capabilities/native_chat/steward_reports.py @@ -174,9 +174,11 @@ def _failed_without_generation(self, actor: dict[str, Any], window: dict[str, An or identity.get("turn_instance_id") != request["turn_instance_id"]): return False inspection = interpret_turn_journal_projection(journal, goal_id=goal_id, agent_id=agent_id, turn_key=key) + host = StewardTurnHost(self.controller) return (inspection.get("journal_consistent") is True and inspection.get("completed_phases") == [] - and StewardTurnHost(self.controller).report_failure_observation(turn) is not None) + and (host.retained_report_failure_observation(turn, window) + or host.report_failure_observation(turn) is not None)) def _cancel_failed_window(self, actor: dict[str, Any], window: dict[str, Any], failure: dict[str, str], *, cadence_locked: bool = False) -> None: diff --git a/loopx/capabilities/native_chat/steward_turn.py b/loopx/capabilities/native_chat/steward_turn.py index ccd4c83603..96fbd2c61d 100644 --- a/loopx/capabilities/native_chat/steward_turn.py +++ b/loopx/capabilities/native_chat/steward_turn.py @@ -11,15 +11,22 @@ from contextlib import contextmanager, ExitStack, nullcontext import hashlib import json +import logging from pathlib import Path from typing import Any from ...control_plane.goals.activation import goal_is_stopped from ...control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected -from ...file_lock import exclusive_file_lock +from ...file_lock import exclusive_file_lock, try_exclusive_file_lock +from ...registry import atomic_write_json +from ..periodic_report.private_generation import attempt_directory from .steward_managed_mode import ManagedSessionTurn, ManagedTaskValidationUnavailable, StewardManagedMode +_LOG = logging.getLogger(__name__) +_REPORT_FAILURE_PROOF = "native-terminal-failure.json" + + def native_steward_client_id(actor: Mapping[str, Any], turn_instance_id: str) -> str: """The original native request identity, shared by dispatch and readback.""" if actor.get("schema_version") == "loopx_managed_session_actor_v0": @@ -268,12 +275,85 @@ def report_failure_observation(self, original: Mapping[str, Any]) -> dict[str, A return None return proof - def report_quota_request(self, original: Mapping[str, Any]) -> dict[str, Any] | None: + @staticmethod + def _report_failure_binding(original: Mapping[str, Any], window: dict[str, Any]) -> dict[str, str] | None: + request = original.get("loopx_request") or {} + actor = request.get("actor") or {} + result = (original.get("response") or {}).get("native_loopx") or {} + failure = result.get("host_failure") or {} + if (original.get("origin") != "native_loopx" or original.get("loopx_execution") is not True + or original.get("status") != "completed" or original.get("session_id") != actor.get("session_id") + or request.get("work_kind") != "report" or request.get("report_window_id") != window["window_id"] + or request.get("report_subscription_revision") != window["subscription_revision"] + or actor.get("goal_id") != window["goal_id"] or actor.get("agent_id") != window["agent_id"] + or result.get("status") != "failed" or result.get("result_kind") != "host_failure" + or failure.get("retryable") is not False): + return None + identity = { + "session_id": original.get("session_id"), + "chat_turn_id": original.get("turn_id"), + "provider_turn_id": original.get("upstream_turn_id"), + "turn_instance_id": request.get("turn_instance_id"), + "window_id": window["window_id"], + "subscription_revision": window["subscription_revision"], + } + if any(not isinstance(value, str) or not value for value in identity.values()): + return None + encoded = json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + return {**identity, "native_result_sha256": hashlib.sha256(encoded).hexdigest()} + + def retained_report_failure_observation(self, original: Mapping[str, Any], window: dict[str, Any]) -> bool: + """Scope retained provider evidence to the same failure and cadence window.""" + binding = self._report_failure_binding(original, window) + if binding is None: + return False + path = attempt_directory(self.controller.coordination_runtime_root, window) / _REPORT_FAILURE_PROOF + try: + if path.parent.is_symlink() or path.is_symlink() or not path.is_file(): + return False + proof = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return False + return (isinstance(proof, dict) + and set(proof) == {"binding", "provider_thread_id", "only_original_user_item"} + and proof["binding"] == binding and proof["only_original_user_item"] is True + and isinstance(proof["provider_thread_id"], str) and bool(proof["provider_thread_id"])) + + def _retain_report_failure_observation(self, original: Mapping[str, Any], window: dict[str, Any], + proof: Mapping[str, Any]) -> None: + binding = self._report_failure_binding(original, window) + thread_id = proof.get("thread_id") + if (binding is None or proof.get("turn_id") != binding["provider_turn_id"] + or proof.get("only_original_user_item") is not True + or not isinstance(thread_id, str) or not thread_id): + return + folder = attempt_directory(self.controller.coordination_runtime_root, window) + if folder.is_symlink() or not folder.is_dir(): + return + path = folder / _REPORT_FAILURE_PROOF + with try_exclusive_file_lock(folder / "generation.lock", preserve_holder=True, + operation="retain_report_failure_observation") as ownership: + if ownership is None or path.exists() or path.is_symlink(): + return + # Read the original again after metadata I/O. Never overwrite a + # prior receipt or copy the provider's live quota observation. + current = self.store.load_turn(original["session_id"], original["turn_id"]) + if current is not None and self._report_failure_binding(current, window) == binding: + atomic_write_json(path, {"binding": binding, + "provider_thread_id": thread_id, "only_original_user_item": True}) + + def report_quota_request(self, original: Mapping[str, Any], *, + report_window: dict[str, Any] | None = None) -> dict[str, Any] | None: """Observe quota recovery, never resume or rewrite the failed native Turn.""" request = original.get("loopx_request") or {} if request.get("report_quota_recovery") is not None: return None proof = self.report_failure_observation(original) + if proof is not None and report_window is not None: + try: + self._retain_report_failure_observation(original, report_window, proof) + except (OSError, ValueError, TypeError, KeyError): + _LOG.warning("An original report's terminal failure proof could not be retained") if (not proof or proof.get("only_original_user_item") is not True or proof.get("codex_error_info") != "usageLimitExceeded" or proof.get("quota_available") is not True): return None @@ -617,7 +697,8 @@ def inspect_submission(quota_observation: dict[str, Any] | None = None) -> dict[ quota_observation = None if "quota_probe" in prepared: quota_observation = prepared["quota_probe"] - quota_observation["request"] = self.report_quota_request(quota_observation["original"]) + quota_observation["request"] = self.report_quota_request( + quota_observation["original"], report_window=report_window) prepared = inspect_submission(quota_observation) if "turn" in prepared: return prepared["turn"], False diff --git a/tests/test_native_steward_reports.py b/tests/test_native_steward_reports.py index 64e6cfb099..85ac7eaf73 100644 --- a/tests/test_native_steward_reports.py +++ b/tests/test_native_steward_reports.py @@ -1635,6 +1635,7 @@ def now(cls, tz=None): fake = Path(runtime.codex_bin) source = fake.read_text() fail_once = tmp_path / "fail-next-report" + history_unavailable = tmp_path / "report-history-unavailable" fail_once.touch() probe = f""" if method == "turn/start" and __import__("pathlib").Path({str(fail_once)!r}).exists(): @@ -1643,6 +1644,9 @@ def now(cls, tz=None): print(json.dumps({{"method": "turn/completed", "params": {{"threadId": "durable-thread", "turn": {{"id": "report-failed-after-launch", "status": "failed", "error": {{"codexErrorInfo": "other", "message": "Synthetic terminal failure"}}}}}}}}), flush=True) continue sys.exit(0) + if method == "thread/turns/list" and __import__("pathlib").Path({str(history_unavailable)!r}).exists(): + print(json.dumps({{"id": request["id"], "result": {{"data": []}}}}), flush=True) + continue if method == "thread/turns/list": print(json.dumps({{"id": request["id"], "result": {{"data": [{{"id": "report-failed-after-launch", "status": {native_status!r}, "error": {{"codexErrorInfo": "other"}}}}]}}}}), flush=True) continue @@ -1660,7 +1664,8 @@ def now(cls, tz=None): cadence = cadence_journal_path(reports.root, actor["goal_id"]) previous = json.loads(cadence.read_text()) native_path = turn_journal_path(reports.root, goal_id=actor["goal_id"], turn_key=native["resume_turn_key"]) - old_artifacts = (store._turn_path(sid, original["turn_id"]), native_path, + old_artifacts = (store._turn_path(sid, original["turn_id"]), + store._event_path(sid, original["turn_id"]), native_path, old_folder / "source.json", old_folder / "editorial-request.json") original_bytes = [path.read_bytes() for path in old_artifacts] sent = list(bot.writes) @@ -1674,6 +1679,8 @@ def wake(): return pump_steward_monitors(runtime) assert wake() == [] + failure_proof = old_folder / "native-terminal-failure.json" + assert failure_proof.is_file() is (native_status == "failed") point += timedelta(days=1) assert wake() == [] # Keep the normal recovery opportunity at the next boundary. point += timedelta(days=2) @@ -1699,9 +1706,26 @@ def starts(): assert starts() == original_starts from loopx.capabilities.native_chat.steward_turn import StewardTurnHost - native_proof = StewardTurnHost(runtime).report_failure_observation(original) + host = StewardTurnHost(runtime) + native_proof = host.report_failure_observation(original) assert bool(native_proof) is (native_status == "failed"), native_proof assert reports._failed_without_generation(actor, old) is (native_status == "failed") + if native_status == "failed": + retained_bytes = failure_proof.read_bytes() + retained = json.loads(retained_bytes) + assert set(retained) == {"binding", "provider_thread_id", "only_original_user_item"} + assert retained["provider_thread_id"] == native_proof["thread_id"] + assert retained["binding"]["provider_turn_id"] == native_proof["turn_id"] + history_unavailable.touch() + assert host.report_failure_observation(original) is None + assert host.report_quota_request(original, report_window=old) is None + retained["binding"]["native_result_sha256"] = "0" * 64 + failure_proof.write_text(json.dumps(retained), encoding="utf-8") + try: + assert wake() == [] and starts() == original_starts + assert json.loads(cadence.read_text()) == previous + finally: + failure_proof.write_bytes(retained_bytes) pending = wake() if native_status != "failed": assert pending == [] and starts() == original_starts @@ -1722,6 +1746,7 @@ def starts(): assert "report_quota_recovery" not in done["loopx_request"] assert json.loads((cadence.parent / "superseded-cadence" / (old["window_id"] + ".json")).read_text()) == previous assert [path.read_bytes() for path in old_artifacts] == original_bytes + assert failure_proof.read_bytes() == retained_bytes assert reports.delivery_current(owner, old) is False with pytest.raises(ValueError, match="no longer the active window"): freeze_private_report(root=reports.root, window=old, response={}, require_current_window=True) diff --git a/tests/test_native_steward_turn_host.py b/tests/test_native_steward_turn_host.py index 3c11caab85..15cc029a46 100644 --- a/tests/test_native_steward_turn_host.py +++ b/tests/test_native_steward_turn_host.py @@ -832,8 +832,8 @@ def test_native_submission_history_read_allows_input_and_rejects_changed_binding store.update_session(sid, status="ready", active_turn_id=None) observe_quota = host.report_quota_request - def checked_quota(turn): - result = observe_quota(turn) + def checked_quota(turn, *, report_window=None): + result = observe_quota(turn, report_window=report_window) quota_checked.set() return result