From 2ac82cad082f5d55fa0cd687cde26a1e7edc1210 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 25 Oct 2026 12:36:33 -0700 Subject: [PATCH 1/8] fix(adapters): pause a Claude session blocked by a usage limit instead of charging it (#610) A Claude Code session stopped behind a subscription limit idled out its clock and was charged a dev attempt, eventually deferring a story whose spec was fine. Seeding the captured lines as env-fault patterns alone would not have helped: #764's token veto overrules any post-mortem match on a session that billed tokens before it hit the wall, which is every such session. Add the usage-limit pattern class: profile fields usage_limit_patterns (blocked, no auto-continue: credit exhaustion) and usage_wait_patterns (blocked until a reset the CLI continues from), disjoint by contract, seeded in claude.toml from the lines captured in #610's runs. The wait loop matches them against the VISIBLE pane at stall-grace expiry (one capture shared with the parked gate; the nudge is withheld and the session ends stalled + parked) and once when the session timeout fires. A dead window is never usage-limited. A live sighting sets SessionResult.usage_limited, and only that is exempt from the token veto: the line is on screen at the verdict, so tokens spent before the limit cannot disprove it. The same patterns also scan the log tail as ordinary env-fault families, where the veto still applies, so a session that recovered from a limit and later stalled for a real reason is still charged. Both pause reasons name a usage limit when the field is set; session-end and workflow-end carry it present-only. The corpus gates now cover all three pane fields. Co-authored-by: sirendhead --- CHANGELOG.md | 5 + src/bmad_loop/adapters/base.py | 9 + src/bmad_loop/adapters/env_fault.py | 63 ++++- src/bmad_loop/adapters/generic.py | 160 +++++++----- src/bmad_loop/adapters/profile.py | 29 +++ src/bmad_loop/data/profiles/claude.toml | 46 +++- src/bmad_loop/engine.py | 7 + src/bmad_loop/escalation.py | 29 ++- tests/test_engine.py | 49 ++++ tests/test_env_fault.py | 2 + tests/test_env_fault_patterns.py | 312 ++++++++++++++++++------ tests/test_escalation.py | 52 ++++ tests/test_generic_tmux.py | 284 ++++++++++++++++++++- tests/test_portability_guard.py | 5 + tests/test_profile.py | 138 ++++++++++- 15 files changed, 1038 insertions(+), 152 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67ffb8016..7b0294f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -175,6 +175,11 @@ breaking changes may land in a minor release. ### Fixed +- Pause a Claude session blocked by a usage limit (credit exhaustion or a session-window + limit seen on the live pane) with a `usage limit:` reason instead of charging it a dev + attempt; a live sighting is exempt from the token veto. New profile fields + `usage_limit_patterns` and `usage_wait_patterns`, seeded from lines captured in + community reports (#610). - Read an epic the initiative declares but has not sliced yet: `status` no longer faults the board on its folderless row, and it triggers no retrospective or closure. - Name the T1 ticket plan an interrupted commit window left `done` or `blocked` when diff --git a/src/bmad_loop/adapters/base.py b/src/bmad_loop/adapters/base.py index 9bdfa3fba..ac4eba07d 100644 --- a/src/bmad_loop/adapters/base.py +++ b/src/bmad_loop/adapters/base.py @@ -343,6 +343,15 @@ class SessionResult: # in `session_failure_reason`. Diagnostic only: never read by routing. # APPENDED. env_fault_overruled: str | None = None + # Usage-limit evidence (#610): the visible-pane line a `usage_limit_patterns` + # / `usage_wait_patterns` match found LIVE — at stall-grace expiry (ending + # the session stalled + parked) or the instant the session timeout fired. + # Set only from a live pane sighting, never from the post-mortem log tail and + # never on a `crashed` or completed result. A set value makes + # `_classify_env_fault` stamp `env_fault` WITHOUT the #764 token veto (the + # limit is on screen now, not a stale match), and both pause reasons name it + # as a usage limit. APPENDED. + usage_limited: str | None = None class CodingCLIAdapter(ABC): diff --git a/src/bmad_loop/adapters/env_fault.py b/src/bmad_loop/adapters/env_fault.py index 455acd375..25011a06d 100644 --- a/src/bmad_loop/adapters/env_fault.py +++ b/src/bmad_loop/adapters/env_fault.py @@ -47,6 +47,14 @@ rationale that already excludes ``over_budget``. Untracked usage (``None``), a tracked zero, or a fault reading usage keep today's verdict: only a measurement can overrule. + +A usage-limit line the wait loop saw LIVE on the visible pane (#610, +``SessionResult.usage_limited``) is the one verdict the veto does not touch: a +session that billed tokens and then hit its subscription limit is exactly the +case the limit pause exists for, and the sighting is of the screen as it stood, +not a stale line from earlier in the log. The same usage patterns also scan the +log TAIL as ordinary families, and there the veto applies — a limit the session +recovered from must not excuse a later real stall. """ from __future__ import annotations @@ -112,7 +120,8 @@ class EnvFaultMixin: Mixed into every adapter that writes a per-task diagnostic log; which files are scanned is named by ``ENV_FAULT_LOG_SUFFIXES`` (or, when that is unset, the single ``ENV_FAULT_LOG_SUFFIX``). Inert for a profile with no - ``env_fault_patterns``, so mixing it in is always safe. + ``env_fault_patterns`` and no usage-limit patterns (and no live + ``usage_limited`` sighting), so mixing it in is always safe. Several files are scanned independently — each its own tail, straddle and match-timeout handling, each read fault its own crumb — and the last match @@ -190,6 +199,26 @@ def _env_fault_patterns(self) -> tuple[regex.Pattern[str], ...]: ``_env_fault_patterns`` directly instead.""" return tuple(regex.compile(p) for p in self.profile.env_fault_patterns) + @cached_property + def _usage_limit_patterns(self) -> tuple[regex.Pattern[str], ...]: + """``profile.usage_limit_patterns``, compiled once per adapter (#610). + Validated at parse time with this engine, so compiling cannot raise. Same + caching CONSTRAINT as ``_env_fault_patterns``. Read by the live pane gates + (generic.py) and, as ordinary families, by the log-tail scan here.""" + return tuple(regex.compile(p) for p in self.profile.usage_limit_patterns) + + @cached_property + def _usage_wait_patterns(self) -> tuple[regex.Pattern[str], ...]: + """``profile.usage_wait_patterns``, compiled once per adapter (#610); as + ``_usage_limit_patterns``.""" + return tuple(regex.compile(p) for p in self.profile.usage_wait_patterns) + + @property + def _tail_scan_patterns(self) -> tuple[regex.Pattern[str], ...]: + """Every family the post-mortem tail scan matches: the env-fault patterns, + then both usage-limit lists as ordinary families (subject to the veto).""" + return self._env_fault_patterns + self._usage_limit_patterns + self._usage_wait_patterns + def _classify_env_fault( self, handle: SessionHandle, spec: SessionSpec, result: SessionResult ) -> SessionResult: @@ -208,12 +237,30 @@ def _classify_env_fault( ``env_fault`` stays False, ``env_fault_overruled`` carries the evidence, and an ``env-fault-overruled`` breadcrumb (with ``tokens``) replaces the classified one. Usage is read only after a match, so a healthy log costs - no extra read.""" - if ( - not self._env_fault_patterns - or result.status not in ENV_FAULT_STATUSES - or result.result_json is not None - ): + no extra read. The tail scan covers ``env_fault_patterns`` and both usage + lists (``_tail_scan_patterns``); a tail-only usage match is overruled + like any other. + + A LIVE usage-limit sighting (``result.usage_limited``, #610) short-cuts + all of that: ``env_fault`` is stamped with that evidence and no veto and + no tail scan, crumbed ``env-fault-classified`` with + ``usage_limited=True``.""" + if result.status not in ENV_FAULT_STATUSES or result.result_json is not None: + return result + if result.usage_limited is not None: + # Veto-exempt by design: the line was on the visible pane at the + # verdict, so tokens spent BEFORE the limit cannot disprove it. + self._note_lifecycle( + handle.task_id, + "env-fault-classified", + status=result.status, + evidence=result.usage_limited, + usage_limited=True, + ) + return dataclasses.replace( + result, env_fault=True, env_fault_evidence=result.usage_limited + ) + if not self._tail_scan_patterns: return result evidence = self._env_fault_evidence(handle.task_id) if evidence is None: @@ -287,7 +334,7 @@ def _env_fault_evidence(self, task_id: str) -> str | None: if lines is None: continue # read fault, crumbed: the other files still count for line in lines: - for pat in self._env_fault_patterns: + for pat in self._tail_scan_patterns: try: hit = pat.search(line, timeout=ENV_FAULT_MATCH_TIMEOUT_S) except TimeoutError as exc: diff --git a/src/bmad_loop/adapters/generic.py b/src/bmad_loop/adapters/generic.py index 0893e273c..1c991f8b5 100644 --- a/src/bmad_loop/adapters/generic.py +++ b/src/bmad_loop/adapters/generic.py @@ -1430,6 +1430,44 @@ def produced_work() -> bool: handle, stop_seen, activity_seen or transcript_work_seen or usage_seen ) + def stall_exit(parked_now: str | None, usage_now: str | None) -> SessionResult: + """The non-completion exit of a stalled session (DW-348, #610): + resolve, then re-probe liveness. A now-dead window is ``crashed`` + (never parked or usage-limited: nothing waits on a human once the CLI + is gone). Otherwise ``stalled`` with ``accept_result=False``: an + artifact on disk cannot upgrade the stall to completed (it may be + stale or mid-write; only a Stop or window death vouches for it), and + ``parked_now`` / ``usage_now`` label it. Both exits carry the + rollout.""" + try_resolve() + if self._probe_liveness(handle, "stall") is False: + return self._final( + handle, + spec, + "crashed", + session_id, + transcript_path, + budget_weighted=budget_weighted, + stop_seen=stop_seen, + produced_work=produced_work(), + ) + stalled = self._final( + handle, + spec, + "stalled", + session_id, + transcript_path, + accept_result=False, + budget_weighted=budget_weighted, + stop_seen=stop_seen, + produced_work=produced_work(), + ) + if parked_now is not None: + return dataclasses.replace( + stalled, parked=True, parked_evidence=parked_now, usage_limited=usage_now + ) + return stalled + while True: # Top-of-tick pane-frame sample for the no-work verdict (#727). Before # the timeout check so growth on the final tick still counts; before the @@ -1463,6 +1501,15 @@ def produced_work() -> bool: # rollout existed, then no further hook event): the exit's # `transcript_path` argument is read before `produced_work()`. try_resolve() + # One look at the visible pane for a usage limit (#610): a + # session whose clock ran out while it sat behind one is not a + # failed attempt. A dead window is never usage-limited (the CLI + # is gone, though a remain-on-exit pane still captures), so a + # match is confirmed alive first; a probe fault reads "not dead". + (usage_hit,) = self._pane_evidence(handle, self._usage_pane_patterns) + usage_now = None + if usage_hit is not None and self._probe_liveness(handle, "timeout") is not False: + usage_now = usage_hit[0] return SessionResult( status="timeout", session_id=session_id, @@ -1472,6 +1519,7 @@ def produced_work() -> bool: budget_weighted=budget_weighted, stop_seen=stop_seen, produced_work=produced_work(), + usage_limited=usage_now, ) # Hard-stop poll (#319), per-iteration and deliberately NOT inside # the heartbeat throttle below: the loop's own wait is capped at 5s @@ -1787,15 +1835,30 @@ def produced_work() -> bool: # (nudges spent, or none configured) is labelled parked too # and pauses instead of retrying (DW-433). Either way no # nudge is typed and the session ends through the tail below. + # The same single capture is matched against the usage-limit + # lists (#610): a CLI stopped behind a usage limit is parked + # too (a nudge can only reproduce the limit line), and the + # match is recorded as `usage_limited` even when the hook + # latch already supplied the parked evidence, since claude's + # idle_prompt latches on exactly this kind of idle. parked_now = parked_evidence + parked_hit, usage_hit = self._pane_evidence( + handle, + self._parked_prompt_patterns if parked_now is None else (), + self._usage_pane_patterns, + ) + if parked_now is None and parked_hit is not None: + parked_now = parked_hit[0] + usage_now = usage_hit[0] if usage_hit is not None else None if parked_now is None: - parked_now = self._pane_parked_evidence(handle) + parked_now = usage_now if parked_now is not None: self._note_lifecycle( handle.task_id, "stall-nudge-withheld", evidence=parked_now, nudge_due=nudge_due, + **({"usage_limited": usage_now} if usage_now is not None else {}), ) elif nudge_due: # The wake nudge IS the re-invocation bmad-loop otherwise @@ -1830,36 +1893,7 @@ def produced_work() -> bool: # persistent failure. A dead window is never labelled parked: # nothing is waiting on a human once the CLI is gone. The # probe fault is crumbed (`site="stall"`, DW-454). - # Both exits below (crashed or stalled) carry the rollout. - try_resolve() - if self._probe_liveness(handle, "stall") is False: - return self._final( - handle, - spec, - "crashed", - session_id, - transcript_path, - budget_weighted=budget_weighted, - stop_seen=stop_seen, - produced_work=produced_work(), - ) - # Still alive: an artifact on disk cannot upgrade the stall to - # completed — it may be stale or mid-write; only a Stop or - # window death vouches for it. - stalled = self._final( - handle, - spec, - "stalled", - session_id, - transcript_path, - accept_result=False, - budget_weighted=budget_weighted, - stop_seen=stop_seen, - produced_work=produced_work(), - ) - if parked_now is not None: - return dataclasses.replace(stalled, parked=True, parked_evidence=parked_now) - return stalled + return stall_exit(parked_now, usage_now) continue # Drop a nested CLI's events before they can re-point the identity # or transcript, set stop_seen, spend nudges or re-arm the stall @@ -2401,28 +2435,32 @@ def _parked_prompt_patterns(self) -> tuple[regex.Pattern[str], ...]: first stall-expiry, or assign this attribute directly.""" return tuple(regex.compile(p) for p in self.profile.parked_prompt_patterns) - def _pane_parked_evidence(self, handle: SessionHandle) -> str | None: - """Whether the VISIBLE pane shows a prompt only a human should answer - (DW-350): the evidence (pattern plus quoted line) of the first - ``parked_prompt_patterns`` match, else None. - - Read only at a stall-grace expiry with no hook latch — whether a wake - nudge is due or the stall is final (DW-433). Observation that - degrades: no patterns, a backend that cannot capture (the seam default - raises ``MultiplexerError``; a duck-typed backend may lack the method), - a capture fault, or a search that blows ``PARKED_PROMPT_MATCH_TIMEOUT_S`` - all read as "no match": a due nudge goes out, or the final stall ends - unparked, exactly as before. The two faults are not silent (DW-448): a + def _pane_evidence( + self, handle: SessionHandle, *pattern_sets: tuple[regex.Pattern[str], ...] + ) -> tuple[tuple[str, str] | None, ...]: + """One look at the VISIBLE pane, matched against each pattern set: per + set, ``(evidence, matched_line)`` for its first match (lines outer, + patterns inner; evidence = pattern plus quoted line), else None. + + The shared matcher behind the parked gate (DW-350: + ``parked_prompt_patterns``) and the usage-limit gates (#610: + ``usage_limit_patterns`` + ``usage_wait_patterns``), so a decision point + that consults several sets captures the pane ONCE. Observation that + degrades: every set empty, or a backend that cannot capture (the seam + default raises ``MultiplexerError``; a duck-typed backend may lack the + method), reads as "no match" for every set; a capture fault does too; + a search that blows ``PARKED_PROMPT_MATCH_TIMEOUT_S`` reads as "no + match" for its own set. The two faults are not silent (DW-448): a ``parked-probe-failed`` crumb (``reason`` = ``capture-failed`` or - ``match-timeout`` + its ``pattern``; ``error``) records that the look - at the pane failed. No patterns and a backend without ``capture_pane`` - are configuration, not faults, and stay silent.""" - patterns = self._parked_prompt_patterns - if not patterns: - return None + ``match-timeout`` + its ``pattern``; ``error``) records that the look at + the pane failed. Empty sets and a backend without ``capture_pane`` are + configuration, not faults: no capture, no crumb.""" + none: tuple[tuple[str, str] | None, ...] = tuple(None for _ in pattern_sets) + if not any(pattern_sets): + return none capture = getattr(self.mux, "capture_pane", None) if capture is None: - return None + return none try: screen = capture(handle.pane_target or handle.native_id) except MultiplexerError as e: @@ -2432,11 +2470,17 @@ def _pane_parked_evidence(self, handle: SessionHandle) -> str | None: reason="capture-failed", error=f"{type(e).__name__}: {e}", ) - return None + return none if not isinstance(screen, str): - return None - text = _ANSI_RE.sub("", screen.replace("\r", "\n")) - for line in text.split("\n"): + return none + lines = _ANSI_RE.sub("", screen.replace("\r", "\n")).split("\n") + return tuple(self._first_pane_match(handle, lines, pats) for pats in pattern_sets) + + def _first_pane_match( + self, handle: SessionHandle, lines: list[str], patterns: tuple[regex.Pattern[str], ...] + ) -> tuple[str, str] | None: + """``_pane_evidence``'s search of one pattern set over captured lines.""" + for line in lines: for pat in patterns: try: hit = pat.search(line, timeout=PARKED_PROMPT_MATCH_TIMEOUT_S) @@ -2450,9 +2494,15 @@ def _pane_parked_evidence(self, handle: SessionHandle) -> str | None: ) return None if hit is not None: - return f"pane matched {pat.pattern!r}: {_excerpt(line, hit.start())}" + return (f"pane matched {pat.pattern!r}: {_excerpt(line, hit.start())}", line) return None + @property + def _usage_pane_patterns(self) -> tuple[regex.Pattern[str], ...]: + """Both usage-limit lists (#610) as the live gates consult them: disjoint + by contract, so either one matching means usage-limited.""" + return self._usage_limit_patterns + self._usage_wait_patterns + def _session_vanished(self, task_id: str) -> bool: # The disambiguating probe (#489): `list_window_ids` returns [] for a # dead window AND for a session that no longer exists, so a plain diff --git a/src/bmad_loop/adapters/profile.py b/src/bmad_loop/adapters/profile.py index 0024b1e6d..b6a92af53 100644 --- a/src/bmad_loop/adapters/profile.py +++ b/src/bmad_loop/adapters/profile.py @@ -265,6 +265,26 @@ class CLIProfile: # derive from usage_parser (SUBAGENT_TRANSCRIPT_DIR_DEFAULTS); "" = off. # APPENDED after session_id_flag for the same positional-compatibility reason. subagent_transcript_dir: str | None = None + # Patterns naming a subscription/usage limit the CLI is stopped behind (#610), + # matched line-by-line against the ANSI-stripped VISIBLE pane at the same live + # decision points as parked_prompt_patterns (stall-grace expiry, and once when + # the session timeout fires) and, as ordinary env-fault families, against the + # post-mortem log tail. Two lists, disjoint by contract — "usage-limited" means + # either one matched: + # usage_limit_patterns: the CLI is BLOCKED and will not continue by itself + # (credit exhaustion); pause now, an operator must act. + # usage_wait_patterns: the CLI is blocked but WAITING for a reset it then + # continues from on its own (a session-window limit). + # A LIVE pane sighting exempts the verdict from the #764 token veto (the line is + # on screen now, so it is not a stale match from earlier in the session); a + # tail-only match is still subject to it. Same evidentiary bar as + # env_fault_patterns: seed only lines captured from the CLI's own screen, with + # the run cited, and single-character classes so the profile line does not + # match itself. Compiled and validated at parse time; empty = inert. + # APPENDED after subagent_transcript_dir for the same positional-compatibility + # reason. + usage_limit_patterns: tuple[str, ...] = () + usage_wait_patterns: tuple[str, ...] = () @property def hookless(self) -> bool: @@ -583,6 +603,13 @@ def fail(msg: str) -> ProfileError: f"parked_prompt_patterns entry is not a valid regex: {pattern!r} ({e})" ) from e + for field_name in ("usage_limit_patterns", "usage_wait_patterns"): + for pattern in getattr(profile, field_name): + try: + regex.compile(pattern) # same engine the usage gate matches with (timeout-guarded) + except (regex.error, RecursionError) as e: + raise fail(f"{field_name} entry is not a valid regex: {pattern!r} ({e})") from e + def _legacy_adapter_default(dialect: str) -> str: """The adapter kind a TOML profile that predates the ``adapter`` field meant. @@ -713,6 +740,8 @@ def str_list(key: str) -> tuple[str, ...]: seed_files=str_list("seed_files"), env_fault_patterns=str_list("env_fault_patterns"), parked_prompt_patterns=str_list("parked_prompt_patterns"), + usage_limit_patterns=str_list("usage_limit_patterns"), + usage_wait_patterns=str_list("usage_wait_patterns"), workspace_trust=workspace_trust, subagent_transcript_dir=raw_subagent_dir, ) diff --git a/src/bmad_loop/data/profiles/claude.toml b/src/bmad_loop/data/profiles/claude.toml index 5396967f5..12570eb1c 100644 --- a/src/bmad_loop/data/profiles/claude.toml +++ b/src/bmad_loop/data/profiles/claude.toml @@ -55,10 +55,13 @@ seed_files = [ # configuration as a provider outage (#507). Pinned by # test_shipped_patterns_do_not_match_their_own_profile_line. # -# No usage-limit / quota cause is seeded: no captured Claude Code quota line -# exists, and on a pane capture that vocabulary is what a story IMPLEMENTING -# rate limiting prints all day. Seed one only from a captured line, citing the -# run it came from — the bar test_unseeded_profiles_stay_inert states. +# No usage-limit / quota cause is seeded HERE: on a pane capture that vocabulary +# is what a story IMPLEMENTING rate limiting prints all day. The two captured +# Claude Code usage-limit sentences live in their own class below +# (usage_limit_patterns / usage_wait_patterns), which is looked at on the live +# pane and is exempt from the token veto only there. Seed any further one only +# from a captured line, citing the run it came from — the bar +# test_unseeded_profiles_stay_inert states. # # To disable or extend, copy this profile into .bmad-loop/profiles/ and edit. env_fault_patterns = [ @@ -88,6 +91,41 @@ parked_prompt_patterns = [ "Enter to confirm · Esc t[o] cancel", ] +# Usage-limit patterns (#610): regexes matched line-by-line against the +# ANSI-stripped VISIBLE pane at stall-grace expiry and when the session timeout +# fires. A live match withholds the stall nudge and ends the session parked with +# a usage-limit pause reason, exempt from the token veto (the line is on screen +# now); the same patterns also scan the post-mortem log tail as ordinary +# env-fault families, where the veto still applies (a limit the session +# recovered from must not excuse a later real stall). +# +# usage_limit_patterns — BLOCKED, no auto-continue: credit exhaustion, the pane +# line `You're out of usage credit` + `s. Run /usage-credits to keep using +# or /model to switch models.`, captured on psmux/Windows 11 (bmad-loop +# 0.11.1) in runs 20260907-084618-824f (dev-1 and review-1) and +# 20260907-090132-98e3 (dev-1 and dev-2). The model name is deliberately left +# out of the anchor: it is whatever model the operator runs, so pinning it would +# miss the next capture. +# +# usage_wait_patterns — blocked but WAITING for a reset Claude Code then +# continues from on its own (autoContinueAtUsageLimit): the session-window +# limit, the pane line `You've hit your session limi` + `t · resets