Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 37 additions & 3 deletions loopx/capabilities/native_chat/steward_reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,38 @@ def _known_generation_failure(self, actor: dict[str, Any], window: dict[str, Any
return {"turn_id": turn["turn_id"], "todo_id": request["todo_id"]}
return None

def _cancellable_failed_window(self, actor: dict[str, Any], window: dict[str, Any]) -> dict[str, str] | None:
"""Identify a terminal failed report attempt that owner cancel may retire."""
known = self._known_generation_failure(actor, window)
if known is not None:
return known
from .steward_turn import report_turn
folder = attempt_directory(self.root, window)
if any(path.is_symlink() or not path.is_file()
for path in (folder / "source.json", folder / "editorial-request.json")):
return None
for name in ("editorial-response.json", "generation-bundle.json", "report.md",
"publication-candidate.json", "private-generation.json", "private-delivery.json"):
try:
(folder / name).lstat()
except FileNotFoundError:
continue
return None
turn = report_turn(self.controller.store, actor, window)
request = (turn or {}).get("loopx_request") or {}
native = ((turn or {}).get("response") or {}).get("native_loopx") or {}
failure = native.get("host_failure") or {}
if (turn and turn.get("status") == "completed" and native.get("status") == "failed"
and native.get("result_kind") == "host_failure" and failure.get("retryable") is False
and native.get("effects") == {"host_invoked": True, "state_written": False,
"quota_spent": False, "scheduler_acknowledged": False}
and request.get("actor") == actor and request.get("work_kind") == "report"
and request.get("report_window_id") == window["window_id"]
and request.get("report_subscription_revision") == window["subscription_revision"]
and isinstance(request.get("todo_id"), str) and bool(request["todo_id"])):
return {"turn_id": turn["turn_id"], "todo_id": request["todo_id"]}
return None

def _execution_failed(self, actor: dict[str, Any], window: dict[str, Any]) -> bool:
"""Observe the current attempt; this grants no cancellation or retry."""
from .steward_turn import report_turn
Expand Down Expand Up @@ -183,7 +215,7 @@ def _cancel_failed_window(self, actor: dict[str, Any], window: dict[str, Any],
from ..periodic_report.cadence_journal import (
_retire_failed_cadence_window_locked, retire_failed_cadence_window,
)
if self._known_generation_failure(actor, window) != failure:
if self._cancellable_failed_window(actor, window) != failure:
raise ValueError("the original failed report changed before cancellation")
native_command(self.controller, ["todo", "update", "--goal-id", actor["goal_id"],
"--todo-id", failure["todo_id"], "--agent-id", actor["agent_id"],
Expand Down Expand Up @@ -222,7 +254,7 @@ def configure(self, binding_id: str, *, operation: str, schedule: dict[str, Any]
pending = None
failure = None
if journal:
failure = self._known_generation_failure(actor, journal["window"])
failure = self._cancellable_failed_window(actor, journal["window"])
pending = {"subscription_revision": journal["window"]["subscription_revision"],
"prepared": (attempt_directory(self.root, journal["window"]) / "editorial-request.json").exists(),
"verified_publication": self._published_window(journal["window"]),
Expand Down Expand Up @@ -484,7 +516,9 @@ def session_report_configuration_guard(controller: Any, goal_id: str, proposed:
after = {k: v for k, v in (proposed or {}).items() if k != "enabled"}
if before != after or not current and proposed is None:
actor = reports._frozen_actor(window)
failure = reports._known_generation_failure(actor, window) if actor else None
failure = (reports._cancellable_failed_window(actor, window)
if hasattr(reports, "_cancellable_failed_window")
else reports._known_generation_failure(actor, window)) if actor else None
if (proposed is not None or not failure) and (current or proposed is not None):
if window["profile_ref"]["profile_id"] == "session_task":
raise PendingStewardReportError("This calendar task is unfinished or its result is unconfirmed. Pause or restore its original plan; only a known failed independent validation can be cleared.")
Expand Down
24 changes: 22 additions & 2 deletions loopx/capabilities/native_chat/steward_turn.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"""
from __future__ import annotations

from collections.abc import Callable, Mapping, Sequence
from collections.abc import Mapping, Sequence
from contextlib import contextmanager, ExitStack, nullcontext
import hashlib
import json
Expand All @@ -21,6 +21,20 @@
from .steward_managed_mode import ManagedSessionTurn, ManagedTaskValidationUnavailable, StewardManagedMode

_USER_ACTION_DEDUPE_KEY = re.compile(r"^[a-z0-9][a-z0-9._:@/-]{0,119}$")
_REPORT_VALIDATOR_PREFIX = (
"from pathlib import Path; from loopx.capabilities.periodic_report.private_generation import validate_private_generation; "
"validate_private_generation(Path("
)


def _same_report_validator_command(frozen_cmd: Any, request_cmd: Any) -> bool:
"""Allow report validation commands whose only drift is the host Python path."""
return (isinstance(frozen_cmd, list) and isinstance(request_cmd, list)
and len(frozen_cmd) == 3 and len(request_cmd) == 3
and all(isinstance(part, str) and bool(part) for part in [*frozen_cmd, *request_cmd])
and frozen_cmd[1] == "-c" and request_cmd[1] == "-c"
and frozen_cmd[2] == request_cmd[2] and frozen_cmd[2].startswith(_REPORT_VALIDATOR_PREFIX)
and frozen_cmd[2].endswith("))"))


def native_steward_client_id(actor: Mapping[str, Any], turn_instance_id: str) -> str:
Expand Down Expand Up @@ -547,7 +561,13 @@ def _prepare_submission(self, binding: dict[str, Any] | None, *, turn_instance_i
frozen = {key: value for key, value in (existing.get("loopx_request") or {}).items()
if key not in {"monitor_lease_version", "monitor_deadline"}}
frozen.setdefault("work_kind", "native_turn")
if frozen != request or work_kind == "monitor" and existing["loopx_request"].get("monitor_deadline") != monitor_deadline:
comparable_frozen = (
{**frozen, "validation_command": request["validation_command"]}
if work_kind == "report" and _same_report_validator_command(
frozen.get("validation_command"), request.get("validation_command"))
else frozen
)
if comparable_frozen != request or work_kind == "monitor" and existing["loopx_request"].get("monitor_deadline") != monitor_deadline:
raise ValueError("native steward request identity conflict")
if work_kind == "report":
retained = report_turn(self.store, actor, report_window)
Expand Down
173 changes: 173 additions & 0 deletions tests/test_steward_report_recovery_and_validator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
"""Focused regressions for report validator path portability and cancel recovery."""
from datetime import datetime, timedelta, timezone
import json
import logging

import pytest

from loopx.capabilities.native_chat.steward_monitor import pump_steward_monitors
from loopx.capabilities.native_chat.steward_turn import StewardTurnHost, _same_report_validator_command
from loopx.capabilities.periodic_report import cadence_runtime
from loopx.capabilities.periodic_report.cadence_journal import read_cadence_journal
from loopx.capabilities.periodic_report.incremental import read_periodic_report_publication_cursor
from loopx.capabilities.periodic_report.private_generation import attempt_directory
import loopx.capabilities.native_chat.steward_reports as report_host
from test_native_steward_private import finish, steward as steward_fixture
from test_native_steward_reports import configured, install_model

steward = steward_fixture


def test_same_report_validator_command_tolerates_only_python_executable_drift():
folder = "/tmp/example-goal/periodic_reports/win-1"
snippet = (
"from pathlib import Path; "
"from loopx.capabilities.periodic_report.private_generation import validate_private_generation; "
f"validate_private_generation(Path({folder!r}))"
)
old_cmd = ["/opt/old-release/python3", "-c", snippet]
new_cmd = ["/opt/new-release/python3.12", "-c", snippet]
other_folder_cmd = [
"/opt/new-release/python3.12",
"-c",
snippet.replace(folder, "/tmp/other-goal/periodic_reports/win-1"),
]
arbitrary_cmd = ["/opt/new-release/python3.12", "-c", "print('hi')"]

assert _same_report_validator_command(old_cmd, new_cmd) is True
assert _same_report_validator_command(old_cmd, other_folder_cmd) is False
assert _same_report_validator_command(arbitrary_cmd, arbitrary_cmd) is False
assert _same_report_validator_command(["/opt/old/python", "-m", snippet], new_cmd) is False


def test_pump_tolerates_report_validator_python_upgrade_and_still_rejects_target_drift(
steward, tmp_path, monkeypatch, caplog,
):
from loopx.chat_agent import CodexChatAgentError
from loopx.capabilities.native_chat.codex_adapter import CodexAppServerAdapter

store, runtime, *_ = steward
sid, actor, reports, _ = configured(steward, tmp_path, monkeypatch, "sqlite")
point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2)

class CalendarClock(datetime):
@classmethod
def now(cls, tz=None):
return point.astimezone(tz) if tz else point.replace(tzinfo=None)

monkeypatch.setattr(report_host, "datetime", CalendarClock)
monkeypatch.setattr(cadence_runtime, "datetime", CalendarClock)

def fail_mid_turn(adapter, _prompt, event_sink):
event_sink("turn.started", {"upstream_turn_id": "mid-turn-failure-1"})
raise CodexChatAgentError("Synthetic mid-turn upstream failure.")

monkeypatch.setattr(CodexAppServerAdapter, "start_turn", fail_mid_turn)
first = finish(runtime, pump_steward_monitors(runtime)[0])
assert first["response"]["native_loopx"]["status"] == "failed"

# Rewrite only the recorded Python executable path as happens across local release upgrades.
turn = store.load_turn(sid, first["turn_id"])
old_req = json.loads(json.dumps(turn["loopx_request"]))
old_req["validation_command"][0] = "/tmp/previous-release/bin/python"
turn["loopx_request"] = old_req
store._turn_path(sid, turn["turn_id"]).write_text(json.dumps(turn))

# Avoid app-server stdio probing in this unit-level identity check; keep focus on _prepare_submission.
monkeypatch.setattr(StewardTurnHost, "report_failure_observation", lambda self, original: None)
runtime._steward_monitor_next_check.clear()
with caplog.at_level(logging.WARNING):
assert pump_steward_monitors(runtime) == []
assert "An original steward report is unavailable" not in caplog.text

# Mutating the validated folder path must still fail closed as an identity conflict.
_, window = reports.current_window(actor["goal_id"].removeprefix("steward-agent-"))
host = StewardTurnHost(runtime)
binding = host._submission_binding(actor["goal_id"].removeprefix("steward-agent-"))
with pytest.raises(ValueError, match="native steward request identity conflict"):
host._prepare_submission(
binding,
turn_instance_id=old_req["turn_instance_id"],
todo_id=old_req["todo_id"],
validation_command=["/opt/new/python", "-c", old_req["validation_command"][2].replace("'", "x'")],
work_kind="report",
monitor_deadline=None,
report_window=window,
)


@pytest.mark.parametrize("provider", ["file", "sqlite"])
def test_owner_cancel_retires_mid_turn_failed_report_without_editorial_response_and_advances(
steward, tmp_path, monkeypatch, provider,
):
from loopx.chat_agent import CodexChatAgentError
from loopx.capabilities.native_chat.codex_adapter import CodexAppServerAdapter
from loopx.capabilities.native_chat.steward_monitor import StewardMonitor

store, runtime, bot, transport, binding, capture, _ = steward
sid, actor, reports, schedule = configured(steward, tmp_path, monkeypatch, provider)
point = datetime.now(timezone.utc).replace(hour=9, minute=1, second=0, microsecond=0) - timedelta(days=2)

class CalendarClock(datetime):
@classmethod
def now(cls, tz=None):
return point.astimezone(tz) if tz else point.replace(tzinfo=None)

monkeypatch.setattr(report_host, "datetime", CalendarClock)
monkeypatch.setattr(cadence_runtime, "datetime", CalendarClock)

def mid_turn_failure(adapter, _prompt, event_sink):
event_sink("turn.started", {"upstream_turn_id": "mid-turn-limit-hit"})
raise CodexChatAgentError("You've hit your usage limit mid-turn.")

with monkeypatch.context() as failure:
failure.setattr(CodexAppServerAdapter, "start_turn", mid_turn_failure)
first = finish(runtime, pump_steward_monitors(runtime)[0])
native = first["response"]["native_loopx"]
assert native["status"] == "failed"
assert native["result_kind"] == "host_failure"
assert native["host_failure"]["retryable"] is False

old = read_cadence_journal(runtime_root=reports.root, goal_id=actor["goal_id"])
folder = attempt_directory(reports.root, old["window"])
assert not (folder / "editorial-response.json").exists()
assert not (folder / "private-delivery.json").exists()

# Keep _known_generation_failure and health().delivery_status unchanged when no editorial-response exists.
assert reports._known_generation_failure(actor, old["window"]) is None
assert reports.health(binding["binding_id"])["delivery_status"] == "not_generated"

# Negative check: an unconfirmed private delivery record must still block failure retirement.
delivery_path = folder / "private-delivery.json"
delivery_path.write_text(json.dumps({"delivery_status": "unconfirmed"}))
blocked_cancel = reports.configure(binding["binding_id"], operation="cancel")["plan"]
assert blocked_cancel["retires_failed_window"] is False
delivery_path.unlink()

# Without delivery or generation artifacts, owner cancel may retire the stuck window cleanly.
cancel = reports.configure(binding["binding_id"], operation="cancel")["plan"]
assert cancel["retires_failed_window"] is True
reports.configure(
binding["binding_id"],
operation="cancel",
execute=True,
expected_plan_id=cancel["plan_id"],
)
retired = read_cadence_journal(runtime_root=reports.root, goal_id=actor["goal_id"])
assert retired["retirement"]["source_receipt_id"] == first["turn_id"]
task = next(row for row in StewardMonitor(runtime).todos(actor) if row["todo_id"] == first["loopx_request"]["todo_id"])
assert task["status"] == "blocked" and not task.get("done")

# Re-enabling advances cleanly to the next due window and delivers a real report.
install_model(steward, tmp_path, sid)
resume = reports.configure(binding["binding_id"], operation="enable", schedule=schedule)["plan"]
reports.configure(binding["binding_id"], operation="enable", schedule=schedule, execute=True, expected_plan_id=resume["plan_id"])
point += timedelta(days=1)
runtime._steward_monitor_next_check.clear()
next_turns = pump_steward_monitors(runtime)
assert len(next_turns) == 1
completed = finish(runtime, next_turns[0])
assert completed["response"]["native_loopx"]["status"] == "committed", completed
assert transport.steward_reports.reconcile() == 1
cursor = read_periodic_report_publication_cursor(runtime_root=reports.root, goal_id=actor["goal_id"], agent_id=actor["agent_id"])
assert cursor["covered_until"] == old["window"]["next_due_at"]
Loading