From 89148015cdd741a86519cfd4adda40c3db20d844 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:28:47 +0100 Subject: [PATCH 01/88] refactor: cli reaches the ledger through named domain methods TDD-Run: 1 TDD-Cycle: 1 TDD-Phase: refactor --- src/tddcli/cli.py | 212 ++++++++++--------------------- src/tddcli/ledger.py | 289 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 328 insertions(+), 173 deletions(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index ebcb75b..3ff7d2b 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -12,7 +12,6 @@ import os import pwd import socket -import sqlite3 import sys import time from collections.abc import Callable @@ -44,7 +43,7 @@ from .adapters.base import FAILED, NOT_COLLECTED from .advance import advance as do_advance from .envelope import Envelope, NextAction, Verb, failure, heartbeat -from .ledger import Ledger, LedgerVersionError, ledger_path, now +from .ledger import Ledger, LedgerVersionError, ledger_path, now, open_ledger from .machine import CLOSED, SKIPPED, Engine BASELINE_MAX_FAILURE_RATIO_DEFAULT = 0.5 @@ -73,7 +72,7 @@ def _worktree() -> Path: def _context(require_run: bool = True): worktree = _worktree() cfg = config_mod.load(worktree) - ledger = Ledger(gitutil.repo_identity(worktree)) + ledger = open_ledger(gitutil.repo_identity(worktree)) run = ledger.active_run(str(worktree)) if require_run and run is None: raise SystemExit( @@ -369,7 +368,7 @@ def cmd_doctor(args) -> Envelope: return Envelope(ok=False, error="configuration invalid", result={"checks": checks}) repo = gitutil.repo_identity(worktree) - ledger = Ledger(repo) + ledger = open_ledger(repo) check("ledger reachable", True, str(ledger.path)) check( "ledger outside worktree", not str(ledger.path).startswith(str(worktree)), str(ledger.path) @@ -537,7 +536,7 @@ def cmd_doctor(args) -> Envelope: def cmd_plan_register(args) -> Envelope: worktree = _worktree() cfg = config_mod.load(worktree) - ledger = Ledger(gitutil.repo_identity(worktree)) + ledger = open_ledger(gitutil.repo_identity(worktree)) rel = str(Path(args.plan)) try: parsed = contract_mod.register(worktree, rel, cfg) @@ -561,23 +560,18 @@ def cmd_plan_register(args) -> Envelope: findings=lint_findings, ) - existing = ledger.one( - "SELECT * FROM plan_contract WHERE plan_path = ? AND git_blob_sha IS ?", - (rel, parsed.blob_sha), - ) + existing = ledger.contract_by_blob(rel, parsed.blob_sha) contract_id = ( existing["id"] if existing - else ledger.insert( - "plan_contract", + else ledger.register_contract( plan_path=rel, - git_blob_sha=parsed.blob_sha, - git_commit=parsed.commit_sha, + blob_sha=parsed.blob_sha, + commit_sha=parsed.commit_sha, status=parsed.status, declared_cycles=contract_mod.cycles_to_json(parsed.cycles), annotation_keys=json.dumps(parsed.annotation_keys), ancillary_files=json.dumps(parsed.ancillary_files), - registered_at=now(), ) ) return Envelope( @@ -749,7 +743,7 @@ def _worker(name, project): def cmd_run_start(args) -> Envelope: worktree = _worktree() cfg = config_mod.load(worktree) - ledger = Ledger(gitutil.repo_identity(worktree)) + ledger = open_ledger(gitutil.repo_identity(worktree)) active = ledger.active_run(str(worktree)) if active is not None: @@ -761,9 +755,7 @@ def cmd_run_start(args) -> Envelope: ) rel = str(Path(args.plan)) - contract_row = ledger.one( - "SELECT * FROM plan_contract WHERE plan_path = ? ORDER BY id DESC LIMIT 1", (rel,) - ) + contract_row = ledger.latest_contract(rel) if contract_row is None: return failure(f"{rel} is not registered; run `tdd plan register {rel}` first") @@ -838,14 +830,13 @@ def cmd_run_start(args) -> Envelope: if existing is not None and existing["stale"]: ledger.release_claim(str(worktree)) - try: - ledger.claim( - str(worktree), - hostname=socket.gethostname(), - pid=os.getpid(), - projects_total=len(probe_projects), - ) - except sqlite3.IntegrityError: + claimed = ledger.claim( + str(worktree), + hostname=socket.gethostname(), + pid=os.getpid(), + projects_total=len(probe_projects), + ) + if claimed is None: return failure( "a baseline is already being collected in this worktree; do not re-run" " `run start` — poll `tdd progress` instead, which reports" @@ -929,20 +920,18 @@ def cmd_run_start(args) -> Envelope: ) executor = identity.resolve(worktree, args.executor) - run_id = ledger.insert( - "run", - plan_contract_id=contract_row["id"], + run_id = ledger.start_run( + contract_row["id"], executor_model=executor.model, executor_session=executor.session, executor_source=executor.source, - worktree_path=str(worktree), - started_at=now(), - allow_dirty=int(bool(args.allow_dirty)), - preexisting_dirty=json.dumps(dirty), + worktree=str(worktree), + allow_dirty=bool(args.allow_dirty), + preexisting_dirty=dirty, config_sha=config_mod.config_sha(worktree), start_sha=start_sha, ) - run = ledger.one("SELECT * FROM run WHERE id = ?", (run_id,)) + run = ledger.run(run_id) if blob_changed: ledger.event(run_id, None, "plan_blob_changed", rel) skipped = sorted(set(cfg.projects) - set(probe_projects)) @@ -958,21 +947,14 @@ def cmd_run_start(args) -> Envelope: # Baselines and the collection snapshot, per project (R9.5, R8.9) — from the # probe above, so the suite is not run twice. for name, (verdict, collection) in probes.items(): - ledger.insert( - "baseline", - run_id=run_id, - project=name, - failing=json.dumps(sorted(verdict.failed)), - captured_at=now(), + ledger.record_baseline( + run_id, + name, + sorted(verdict.failed), source="reused" if name in reused else "probed", ) - ledger.insert( - "collection_snapshot", - run_id=run_id, - project=name, - tests=json.dumps(sorted(collection.tests)), - failed_files=json.dumps(collection.failed_files), - captured_at=now(), + ledger.record_collection( + run_id, name, sorted(collection.tests), collection.failed_files ) for name, (verdict, _collection) in probes.items(): @@ -1002,7 +984,7 @@ def cmd_run_start(args) -> Envelope: if artifact_outcome.failed: f = artifact_outcome.failed[0] stderr_tail = (f.get("stderr") or "").strip()[-200:] - ledger.update("run", run_id, ended_at=now(), outcome="refused") + ledger.end_run(run_id, "refused") return failure( f"artifact {f['artifact']}: regenerate hook exited {f['code']}" + (f" — {stderr_tail}" if stderr_tail else "") @@ -1079,10 +1061,9 @@ def cmd_advance(args) -> Envelope: if existing_advance is not None and existing_advance["stale"]: ledger.release_advance_claim(str(worktree)) - try: - ledger.claim_advance(str(worktree), hostname=socket.gethostname(), pid=os.getpid()) - except sqlite3.IntegrityError: - held = ledger.one("SELECT * FROM advance_claim WHERE worktree_path = ?", (str(worktree),)) + claimed = ledger.claim_advance(str(worktree), hostname=socket.gethostname(), pid=os.getpid()) + if claimed is None: + held = ledger.advance_claim_row(str(worktree)) started = datetime.fromisoformat(held["started_at"]) if started.tzinfo is None: started = started.replace(tzinfo=timezone.utc) @@ -1108,17 +1089,10 @@ def cmd_cycle_skip(args) -> Envelope: cycle = ledger.open_cycle(run["id"]) if cycle is None: return failure("no open cycle") - ledger.update("cycle", cycle["id"], phase=SKIPPED, closed_at=now(), skip_reason=args.reason) - ledger.insert( - "transition", - cycle_id=cycle["id"], - from_phase=cycle["phase"], - to_phase=SKIPPED, - at=now(), - ) + ledger.skip_cycle(cycle["id"], from_phase=cycle["phase"], to_phase=SKIPPED, reason=args.reason) nxt_declared = next((c for c in engine.declared if c.ordinal > cycle["ordinal"]), None) if nxt_declared is None: - ledger.update("run", run["id"], ended_at=now(), outcome="complete") + ledger.end_run(run["id"], "complete") return Envelope( run={"id": run["id"], "cycle": cycle["ordinal"], "phase": SKIPPED}, next_action=NextAction( @@ -1140,14 +1114,7 @@ def cmd_cycle_skip(args) -> Envelope: def cmd_annotate(args) -> Envelope: worktree, cfg, ledger, run = _context() cycle = ledger.open_cycle(run["id"]) - ledger.insert( - "annotation", - run_id=run["id"], - cycle_id=cycle["id"] if cycle else None, - key=args.key, - value=args.value, - at=now(), - ) + ledger.add_annotation(run["id"], cycle["id"] if cycle else None, args.key, args.value) return Envelope( run={"id": run["id"], "cycle": cycle["ordinal"] if cycle else None}, result={"key": args.key}, @@ -1158,20 +1125,15 @@ def cmd_annotate(args) -> Envelope: def cmd_note(args) -> Envelope: worktree, cfg, ledger, run = _context(require_run=False) if run is None: - run = ledger.one( - "SELECT * FROM run WHERE worktree_path = ? ORDER BY id DESC LIMIT 1", - (str(worktree),), - ) + run = ledger.latest_run(str(worktree)) if run is None: return failure("no runs recorded for this worktree; `tdd run start --plan `") cycle = ledger.open_cycle(run["id"]) - ledger.insert( - "note", - run_id=run["id"], - cycle_id=cycle["id"] if cycle else None, - phase=cycle["phase"] if cycle else None, - text=args.text, - at=now(), + ledger.add_note( + run["id"], + cycle["id"] if cycle else None, + cycle["phase"] if cycle else None, + args.text, ) if cycle is not None: next_action = NextAction( @@ -1191,16 +1153,9 @@ def cmd_blocker(args) -> Envelope: if args.kind not in BLOCKER_KINDS: return failure(f"unknown blocker kind {args.kind!r}; use one of {sorted(BLOCKER_KINDS)}") cycle = ledger.open_cycle(run["id"]) - ledger.insert( - "blocker", - run_id=run["id"], - cycle_id=cycle["id"] if cycle else None, - kind=args.kind, - detail=args.detail, - at=now(), - ) + ledger.add_blocker(run["id"], cycle["id"] if cycle else None, args.kind, args.detail) # R8.7 — a blocked run is not live, so the stop hook must release. - ledger.update("run", run["id"], ended_at=now(), outcome="blocked") + ledger.end_run(run["id"], "blocked") return Envelope( run={"id": run["id"], "cycle": cycle["ordinal"] if cycle else None, "phase": "BLOCKED"}, result={"kind": args.kind, "detail": args.detail}, @@ -1221,15 +1176,8 @@ def _accept_failures_into_baseline( A test that passes at start_sha is refused — it is a run-introduced regression. Returns (accepted, refused). """ - latest = ledger.all( - "SELECT project, other_failures FROM invocation WHERE id IN (" - " SELECT MAX(id) FROM invocation WHERE run_id = ? AND phase_at = 'CLOSE_SWEEP'" - " GROUP BY project)", - (run_id,), - ) - rows = { - r["project"]: r for r in ledger.all("SELECT * FROM baseline WHERE run_id = ?", (run_id,)) - } + latest = ledger.latest_close_sweeps(run_id) + rows = {r["project"]: r for r in ledger.baseline_rows(run_id)} accepted: dict[str, list[str]] = {} refused: dict[str, list[str]] = {} for sweep in latest: @@ -1254,11 +1202,11 @@ def _accept_failures_into_baseline( acc = [] ref = list(new) if acc: - ledger.update("baseline", row["id"], failing=json.dumps(sorted(known | set(acc)))) + ledger.amend_baseline(run_id, row["id"], sorted(known | set(acc))) accepted[project] = acc if ref: refused[project] = ref - run_row = ledger.one("SELECT start_sha FROM run WHERE id = ?", (run_id,)) + run_row = ledger.run(run_id) start_sha = run_row["start_sha"] if run_row else None verdicts: dict[str, dict] = {} for project in set(list(accepted.keys()) + list(refused.keys())): @@ -1274,18 +1222,10 @@ def _accept_failures_into_baseline( return accepted, refused -def _latest_blocked_run(ledger: Ledger, worktree: Path) -> sqlite3.Row | None: - return ledger.one( - "SELECT * FROM run WHERE worktree_path = ? AND outcome = 'blocked'" - " ORDER BY id DESC LIMIT 1", - (str(worktree),), - ) - - def cmd_resume(args) -> Envelope: worktree = _worktree() cfg = config_mod.load(worktree) - ledger = Ledger(gitutil.repo_identity(worktree)) + ledger = open_ledger(gitutil.repo_identity(worktree)) run = ledger.active_run(str(worktree)) accepted: dict[str, list[str]] = {} refused_from_baseline: dict[str, list[str]] = {} @@ -1296,14 +1236,13 @@ def cmd_resume(args) -> Envelope: if args.unblock: if run is not None: return failure("run is already live; --unblock applies to a blocked run") - blocked = _latest_blocked_run(ledger, worktree) + blocked = ledger.latest_blocked_run(str(worktree)) if blocked is None: return failure("no blocked run to unblock in this worktree") if not args.note: return failure("--unblock requires --note describing the intervention") - ledger.update("run", blocked["id"], ended_at=None, outcome=None) - ledger.insert("human_intervention", run_id=blocked["id"], note=args.note, at=now()) - run = ledger.one("SELECT * FROM run WHERE id = ?", (blocked["id"],)) + ledger.reopen_run(blocked["id"], args.note) + run = ledger.run(blocked["id"]) if args.accept_failures: _eng = _engine(worktree, cfg, ledger, run) accepted, refused_from_baseline = _accept_failures_into_baseline( @@ -1349,9 +1288,9 @@ def cmd_run_abandon(args) -> Envelope: if not args.reason.strip(): return failure("--reason must say why the run is abandoned", reason="reason_required") worktree = _worktree() - ledger = Ledger(gitutil.repo_identity(worktree)) + ledger = open_ledger(gitutil.repo_identity(worktree)) if args.run is not None: - run = ledger.one("SELECT * FROM run WHERE id = ?", (args.run,)) + run = ledger.run(args.run) if run is None: return failure(f"no run {args.run} in this ledger", reason="run_not_found") if run["ended_at"] is not None and run["outcome"] != "blocked": @@ -1365,7 +1304,7 @@ def cmd_run_abandon(args) -> Envelope: reason="worktree_exists", ) else: - run = ledger.active_run(str(worktree)) or _latest_blocked_run(ledger, worktree) + run = ledger.active_run(str(worktree)) or ledger.latest_blocked_run(str(worktree)) if run is None: return failure("no live or blocked run in this worktree", reason="no_run") # A stale claim is released below, not obeyed; a live one means an advance is mid-flight. @@ -1377,20 +1316,14 @@ def cmd_run_abandon(args) -> Envelope: pid=claim["pid"], ) executor = identity.resolve(worktree) - at = now() - ledger.update("run", run["id"], ended_at=at, outcome="abandoned") - ledger.insert( - "abandonment", - run_id=run["id"], + ledger.abandon_run( + run["id"], reason=args.reason, account=_calling_account(), executor_model=executor.model, executor_source=executor.source, - at=at, + at=now(), ) - ledger.insert("human_intervention", run_id=run["id"], note=f"abandoned: {args.reason}", at=at) - ledger.release_claim(run["worktree_path"]) - ledger.release_advance_claim(run["worktree_path"]) return Envelope( result={"run_id": run["id"]}, next_action=NextAction(Verb.COMPLETE, f"Run {run['id']} abandoned."), @@ -1406,12 +1339,10 @@ def cmd_sensitivity(args) -> Envelope: if args.step == "begin": if ledger.open_sensitivity(cycle["id"]) is not None: return failure("a sensitivity check is already open") - check_id = ledger.insert( - "sensitivity_check", - cycle_id=cycle["id"], + check_id = ledger.open_sensitivity_check( + cycle["id"], reference_diff=snapshot.capture(worktree, cfg), reference_untracked=snapshot.fingerprint(worktree, cfg), - opened_at=now(), ) return Envelope( run={"id": run["id"], "cycle": cycle["ordinal"]}, @@ -1437,8 +1368,8 @@ def cmd_sensitivity(args) -> Envelope: # A mutation that breaks collection also proves the test depends on the code. bites = bool(outcomes) and all(o in (FAILED, NOT_COLLECTED) for o in outcomes.values()) evidence = next((v.target_evidence for v in verdicts if v.target_evidence), "") - ledger.update( - "sensitivity_check", + ledger.record_sensitivity_mutation( + cycle["id"], open_check["id"], mutation_diff=gitutil.diff_text(worktree)[:20000], observed_failure=failure_text[:4000], @@ -1467,12 +1398,7 @@ def cmd_sensitivity(args) -> Envelope: # end — restore and verify byte-identical (R8.5) to_restore = snapshot.restore(worktree, cfg, open_check["reference_diff"]) restored_ok = snapshot.fingerprint(worktree, cfg) == open_check["reference_untracked"] - ledger.update( - "sensitivity_check", - open_check["id"], - restored_ok=int(restored_ok), - closed_at=now(), - ) + ledger.close_sensitivity_check(cycle["id"], open_check["id"], restored_ok=restored_ok) if not restored_ok: ledger.event(run["id"], cycle["id"], "restore_mismatch", json.dumps(to_restore)) return Envelope( @@ -1527,7 +1453,7 @@ def cmd_target(args) -> Envelope: f" the target was not changed.{hint}" ) - ledger.update("cycle", cycle["id"], target_tests=json.dumps([target])) + ledger.set_targets(cycle["id"], [target]) ledger.event(run["id"], cycle["id"], "target_named_by_agent", target) return Envelope( run={"id": run["id"], "cycle": cycle["ordinal"]}, @@ -1539,10 +1465,7 @@ def cmd_target(args) -> Envelope: def cmd_log_render(args) -> Envelope: worktree, cfg, ledger, run = _context(require_run=False) if run is None: - run = ledger.one( - "SELECT * FROM run WHERE worktree_path = ? ORDER BY id DESC LIMIT 1", - (str(worktree),), - ) + run = ledger.latest_run(str(worktree)) if run is None: return failure("no runs recorded for this worktree") text = render.friction_log(ledger, run) @@ -1571,10 +1494,7 @@ def cmd_progress(args) -> Envelope: """Human-readable progress. `status` remains the agent's machine view.""" worktree, cfg, ledger, run = _context(require_run=False) if run is None: - run = ledger.one( - "SELECT * FROM run WHERE worktree_path = ? ORDER BY id DESC LIMIT 1", - (str(worktree),), - ) + run = ledger.latest_run(str(worktree)) if run is None: # A baseline can take minutes; a claim with no run row yet is in-flight, not # "never started". `ok: true` — a polling agent must not see diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 5d95c05..12ca430 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -625,20 +625,26 @@ def cached_baseline( # -- baseline claim ---------------------------------------------------- - def claim(self, worktree: str, hostname: str, pid: int, projects_total: int) -> int: - """Insert the claim row. The insert is the lock: `worktree_path` - carries `UNIQUE`, so a second claim on the same worktree raises - `sqlite3.IntegrityError` rather than racing a read-then-write check.""" - return self.insert( - "baseline_claim", - worktree_path=worktree, - hostname=hostname, - pid=pid, - projects_total=projects_total, - projects_done=0, - current_project=None, - started_at=now(), - ) + def claim(self, worktree: str, hostname: str, pid: int, projects_total: int) -> int | None: + """Insert the claim row, or return None when the worktree is already claimed. + + The insert is the lock: `worktree_path` carries `UNIQUE`, so a second claim + fails at the insert rather than racing a read-then-write check. The conflict + is answered as None, not as `sqlite3.IntegrityError`, because no sqlite + exception can cross a socket to a remote caller.""" + try: + return self.insert( + "baseline_claim", + worktree_path=worktree, + hostname=hostname, + pid=pid, + projects_total=projects_total, + projects_done=0, + current_project=None, + started_at=now(), + ) + except sqlite3.IntegrityError: + return None def release_claim(self, worktree: str) -> None: self.db.execute("DELETE FROM baseline_claim WHERE worktree_path = ?", (worktree,)) @@ -654,17 +660,19 @@ def update_claim(self, worktree: str, projects_done: int, current_project: str) ) self.db.commit() - def claim_advance(self, worktree: str, hostname: str, pid: int) -> int: - """Insert the advance claim row. The insert is the lock: `worktree_path` - carries `UNIQUE`, so a second claim on the same worktree raises - `sqlite3.IntegrityError` rather than racing a read-then-write check.""" - return self.insert( - "advance_claim", - worktree_path=worktree, - hostname=hostname, - pid=pid, - started_at=now(), - ) + def claim_advance(self, worktree: str, hostname: str, pid: int) -> int | None: + """Insert the advance claim row, or return None when the worktree is already + claimed. The insert is the lock, as in `claim`.""" + try: + return self.insert( + "advance_claim", + worktree_path=worktree, + hostname=hostname, + pid=pid, + started_at=now(), + ) + except sqlite3.IntegrityError: + return None def release_advance_claim(self, worktree: str) -> None: self.db.execute("DELETE FROM advance_claim WHERE worktree_path = ?", (worktree,)) @@ -672,7 +680,7 @@ def release_advance_claim(self, worktree: str) -> None: def active_advance_claim(self, worktree: str) -> dict | None: """Read-only observer — staleness computed but no row deleted.""" - row = self.one("SELECT * FROM advance_claim WHERE worktree_path = ?", (worktree,)) + row = self.advance_claim_row(worktree) if row is None: return None claim = dict(row) @@ -683,7 +691,7 @@ def active_claim(self, worktree: str) -> dict | None: """Read-only, per the store's append-only contract — `cmd_progress` and `cmd_status` call this as pure observers. Only `cmd_run_start` acts on the computed `stale` flag (release + reclaim); nothing here deletes a row.""" - row = self.one("SELECT * FROM baseline_claim WHERE worktree_path = ?", (worktree,)) + row = self.claim_row(worktree) if row is None: return None claim = dict(row) @@ -692,3 +700,230 @@ def active_claim(self, worktree: str) -> dict | None: # "alive" bricks the worktree, a false "dead" reopens the bug (Decisions). claim["stale"] = self._claim_is_stale(claim["hostname"], claim["pid"], claim["started_at"]) return claim + + def claim_row(self, worktree: str) -> sqlite3.Row | None: + """The raw baseline claim, with no judgement of whether its owner lives.""" + return self.one("SELECT * FROM baseline_claim WHERE worktree_path = ?", (worktree,)) + + def advance_claim_row(self, worktree: str) -> sqlite3.Row | None: + """The raw advance claim, with no judgement of whether its owner lives.""" + return self.one("SELECT * FROM advance_claim WHERE worktree_path = ?", (worktree,)) + + # -- plan contracts ---------------------------------------------------- + + def contract_by_blob(self, plan_path: str, blob_sha: str | None) -> sqlite3.Row | None: + return self.one( + "SELECT * FROM plan_contract WHERE plan_path = ? AND git_blob_sha IS ?", + (plan_path, blob_sha), + ) + + def latest_contract(self, plan_path: str) -> sqlite3.Row | None: + return self.one( + "SELECT * FROM plan_contract WHERE plan_path = ? ORDER BY id DESC LIMIT 1", + (plan_path,), + ) + + def register_contract( + self, + *, + plan_path: str, + blob_sha: str | None, + commit_sha: str | None, + status: str, + declared_cycles: str, + annotation_keys: str, + ancillary_files: str, + ) -> int: + return self.insert( + "plan_contract", + plan_path=plan_path, + git_blob_sha=blob_sha, + git_commit=commit_sha, + status=status, + declared_cycles=declared_cycles, + annotation_keys=annotation_keys, + ancillary_files=ancillary_files, + registered_at=now(), + ) + + # -- runs ---------------------------------------------------------------- + + def run(self, run_id: int) -> sqlite3.Row | None: + return self.one("SELECT * FROM run WHERE id = ?", (run_id,)) + + def latest_run(self, worktree: str) -> sqlite3.Row | None: + return self.one( + "SELECT * FROM run WHERE worktree_path = ? ORDER BY id DESC LIMIT 1", (worktree,) + ) + + def latest_blocked_run(self, worktree: str) -> sqlite3.Row | None: + return self.one( + "SELECT * FROM run WHERE worktree_path = ? AND outcome = 'blocked'" + " ORDER BY id DESC LIMIT 1", + (worktree,), + ) + + def start_run( + self, + contract_id: int, + *, + executor_model: str, + executor_session: str | None, + executor_source: str, + worktree: str, + allow_dirty: bool, + preexisting_dirty: list[str], + config_sha: str | None, + start_sha: str | None, + ) -> int: + return self.insert( + "run", + plan_contract_id=contract_id, + executor_model=executor_model, + executor_session=executor_session, + executor_source=executor_source, + worktree_path=worktree, + started_at=now(), + allow_dirty=int(bool(allow_dirty)), + preexisting_dirty=json.dumps(preexisting_dirty), + config_sha=config_sha, + start_sha=start_sha, + ) + + def end_run(self, run_id: int, outcome: str) -> None: + self.update("run", run_id, ended_at=now(), outcome=outcome) + + def reopen_run(self, run_id: int, note: str) -> None: + """`resume --unblock`: the run is live again, and a human said why.""" + self.update("run", run_id, ended_at=None, outcome=None) + self.insert("human_intervention", run_id=run_id, note=note, at=now()) + + def abandon_run( + self, + run_id: int, + *, + reason: str, + account: str, + executor_model: str, + executor_source: str, + at: str, + ) -> None: + """End the run as abandoned, record who did it, and free its worktree's claims.""" + run = self.run(run_id) + self.update("run", run_id, ended_at=at, outcome="abandoned") + self.insert( + "abandonment", + run_id=run_id, + reason=reason, + account=account, + executor_model=executor_model, + executor_source=executor_source, + at=at, + ) + self.insert("human_intervention", run_id=run_id, note=f"abandoned: {reason}", at=at) + self.release_claim(run["worktree_path"]) + self.release_advance_claim(run["worktree_path"]) + + def record_baseline(self, run_id: int, project: str, failing: list[str], source: str) -> None: + self.insert( + "baseline", + run_id=run_id, + project=project, + failing=json.dumps(sorted(failing)), + captured_at=now(), + source=source, + ) + + def record_collection( + self, run_id: int, project: str, tests: list[str], failed_files: dict + ) -> None: + self.insert( + "collection_snapshot", + run_id=run_id, + project=project, + tests=json.dumps(sorted(tests)), + failed_files=json.dumps(failed_files), + captured_at=now(), + ) + + def baseline_rows(self, run_id: int) -> list[sqlite3.Row]: + return self.all("SELECT * FROM baseline WHERE run_id = ?", (run_id,)) + + def amend_baseline(self, run_id: int, baseline_id: int, failing: list[str]) -> None: + self._write( + "UPDATE baseline SET failing = ? WHERE id = ? AND run_id = ?", + (json.dumps(sorted(failing)), baseline_id, run_id), + ) + + def latest_close_sweeps(self, run_id: int) -> list[sqlite3.Row]: + """Each project's most recent close-sweep invocation in the run.""" + return self.all( + "SELECT project, other_failures FROM invocation WHERE id IN (" + " SELECT MAX(id) FROM invocation WHERE run_id = ? AND phase_at = 'CLOSE_SWEEP'" + " GROUP BY project)", + (run_id,), + ) + + # -- cycles and what is said about them --------------------------------- + + def skip_cycle(self, cycle_id: int, *, from_phase: str, to_phase: str, reason: str) -> None: + at = now() + self._write( + "UPDATE cycle SET phase = ?, closed_at = ?, skip_reason = ? WHERE id = ?", + (to_phase, at, reason, cycle_id), + ) + self.insert( + "transition", cycle_id=cycle_id, from_phase=from_phase, to_phase=to_phase, at=at + ) + + def set_targets(self, cycle_id: int, targets: list[str]) -> None: + self.update("cycle", cycle_id, target_tests=json.dumps(targets)) + + def add_annotation(self, run_id: int, cycle_id: int | None, key: str, value: str) -> None: + self.insert("annotation", run_id=run_id, cycle_id=cycle_id, key=key, value=value, at=now()) + + def add_note(self, run_id: int, cycle_id: int | None, phase: str | None, text: str) -> None: + self.insert("note", run_id=run_id, cycle_id=cycle_id, phase=phase, text=text, at=now()) + + def add_blocker(self, run_id: int, cycle_id: int | None, kind: str, detail: str) -> None: + self.insert("blocker", run_id=run_id, cycle_id=cycle_id, kind=kind, detail=detail, at=now()) + + # -- sensitivity checks -------------------------------------------------- + + def open_sensitivity_check( + self, cycle_id: int, *, reference_diff: str, reference_untracked: str + ) -> int: + return self.insert( + "sensitivity_check", + cycle_id=cycle_id, + reference_diff=reference_diff, + reference_untracked=reference_untracked, + opened_at=now(), + ) + + def record_sensitivity_mutation( + self, + cycle_id: int, + check_id: int, + *, + mutation_diff: str, + observed_failure: str, + evidence_line: str, + ) -> None: + self._write( + "UPDATE sensitivity_check SET mutation_diff = ?, observed_failure = ?," + " evidence_line = ? WHERE id = ? AND cycle_id = ?", + (mutation_diff, observed_failure, evidence_line, check_id, cycle_id), + ) + + def close_sensitivity_check(self, cycle_id: int, check_id: int, *, restored_ok: bool) -> None: + self._write( + "UPDATE sensitivity_check SET restored_ok = ?, closed_at = ?" + " WHERE id = ? AND cycle_id = ?", + (int(restored_ok), now(), check_id, cycle_id), + ) + + +def open_ledger(repo_path: Path) -> Ledger: + """The ledger of a repository. Every command opens its ledger through here.""" + return Ledger(repo_path) From 75160e8e6c74c1060164d16c2107773b4fd0f4b8 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:33:57 +0100 Subject: [PATCH 02/88] refactor: the state machine reaches the ledger through named domain methods TDD-Run: 1 TDD-Cycle: 2 TDD-Phase: refactor --- src/tddcli/advance.py | 58 +++-------- src/tddcli/ledger.py | 237 ++++++++++++++++++++++++++++++++++++++++++ src/tddcli/machine.py | 165 +++++++++-------------------- 3 files changed, 301 insertions(+), 159 deletions(-) diff --git a/src/tddcli/advance.py b/src/tddcli/advance.py index 3a2b9b6..f5b4eb9 100644 --- a/src/tddcli/advance.py +++ b/src/tddcli/advance.py @@ -12,7 +12,6 @@ from . import config as config_mod from .adapters.base import FAILED, NOT_COLLECTED, NOT_FOUND, PASSED from .envelope import Envelope, NextAction, Verb -from .ledger import now from .machine import ( AWAITING_IMPL, AWAITING_PIN, @@ -28,22 +27,15 @@ def _note_nudge(engine: Engine, cycle) -> str: if cycle is None: return "" - events = engine.ledger.all( - "SELECT kind FROM integrity_event WHERE cycle_id = ? AND kind IN ({})".format( - ",".join("?" * len(NUDGE_KINDS)) - ), - (cycle["id"], *NUDGE_KINDS), - ) - if not events: + if not engine.ledger.cycle_event_kinds(cycle["id"], sorted(NUDGE_KINDS)): return "" - existing_note = engine.ledger.one("SELECT id FROM note WHERE cycle_id = ?", (cycle["id"],)) - if existing_note is not None: + if engine.ledger.cycle_has_note(cycle["id"]): return "" return ' An integrity event was recorded on this cycle — consider `tdd note ""` while the reason is fresh.' def _reply(engine: Engine, cycle, verb: Verb, detail: str, **result) -> Envelope: - fresh = engine.ledger.one("SELECT * FROM cycle WHERE id = ?", (cycle["id"],)) if cycle else None + fresh = engine.ledger.cycle(cycle["id"]) if cycle else None nudge = _note_nudge(engine, fresh or cycle) return Envelope( run=engine.run_state(fresh or cycle), @@ -67,11 +59,7 @@ def _adopt_target(engine: Engine, cycle, missing: list[str]) -> tuple[list[str], at_start = engine.ledger.collection(engine.run["id"]) known = {t for tests in at_start.values() for t in tests} run_targets = { - t - for row in engine.ledger.all( - "SELECT target_tests FROM cycle WHERE run_id = ?", (engine.run["id"],) - ) - for t in json.loads(row["target_tests"]) + t for targets in engine.ledger.run_target_tests(engine.run["id"]) for t in targets } new_tests: list[str] = [] @@ -86,23 +74,11 @@ def _adopt_target(engine: Engine, cycle, missing: list[str]) -> tuple[list[str], def _stub_directive_issued(engine: Engine, cycle) -> bool: - return ( - engine.ledger.one( - "SELECT id FROM integrity_event WHERE cycle_id = ? AND kind = 'stub_directive_issued'", - (cycle["id"],), - ) - is not None - ) + return engine.ledger.cycle_has_event(cycle["id"], "stub_directive_issued") def _last_outside_emitted(engine: Engine, cycle) -> str | None: - row = engine.ledger.one( - "SELECT detail FROM integrity_event" - " WHERE cycle_id = ? AND kind = 'undeclared_file_touched'" - " ORDER BY id DESC LIMIT 1", - (cycle["id"],), - ) - return row["detail"] if row else None + return engine.ledger.last_cycle_event_detail(cycle["id"], "undeclared_file_touched") def _sanctioned_stubs(engine: Engine, cycle, implementation: list[str]) -> list[str]: @@ -253,8 +229,8 @@ def _adopt( declared=missing, ) engine.ledger.event(engine.run["id"], cycle["id"], "declared_test_mismatch", json.dumps(detail)) - engine.ledger.update("cycle", cycle["id"], target_tests=json.dumps(kept + [adopted])) - cycle = engine.ledger.one("SELECT * FROM cycle WHERE id = ?", (cycle["id"],)) + engine.ledger.set_targets(cycle["id"], kept + [adopted]) + cycle = engine.ledger.cycle(cycle["id"]) if outcome is None: return _reply( engine, @@ -603,15 +579,10 @@ def _handle_refactor(engine: Engine, cycle, retried: bool) -> Envelope: if nxt is None: blocking = engine.close_undeclared_gate(cycle) if blocking: - engine.ledger.insert( - "blocker", - run_id=engine.run["id"], - cycle_id=cycle["id"], - kind="undeclared_file_uncommitted", - detail=json.dumps(blocking), - at=now(), + engine.ledger.add_blocker( + engine.run["id"], cycle["id"], "undeclared_file_uncommitted", json.dumps(blocking) ) - engine.ledger.update("run", engine.run["id"], outcome="blocked") + engine.ledger.set_run_outcome(engine.run["id"], "blocked") return Envelope( run={ "id": engine.run["id"], @@ -684,12 +655,7 @@ def _check_config_drift(engine: Engine, cycle) -> None: current = config_mod.config_sha(engine.worktree) if current == pinned: return - already = engine.ledger.one( - "SELECT id FROM integrity_event WHERE run_id = ? AND kind = 'config_changed'" - " AND detail = ?", - (engine.run["id"], current), - ) - if already is None: + if not engine.ledger.run_has_event(engine.run["id"], "config_changed", current): engine.ledger.event( engine.run["id"], cycle["id"], diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 12ca430..8580168 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -923,6 +923,243 @@ def close_sensitivity_check(self, cycle_id: int, check_id: int, *, restored_ok: (int(restored_ok), now(), check_id, cycle_id), ) + # -- the state machine's records ------------------------------------------ + + def contract(self, contract_id: int) -> sqlite3.Row | None: + return self.one("SELECT * FROM plan_contract WHERE id = ?", (contract_id,)) + + def cycle(self, cycle_id: int) -> sqlite3.Row | None: + return self.one("SELECT * FROM cycle WHERE id = ?", (cycle_id,)) + + def unclosed_cycle(self, run_id: int, ordinal: int) -> sqlite3.Row | None: + return self.one( + "SELECT * FROM cycle WHERE run_id = ? AND ordinal = ? AND closed_at IS NULL", + (run_id, ordinal), + ) + + def add_cycle( + self, + run_id: int, + *, + ordinal: int, + kind: str, + projects: list[str], + declared_tests: list[str], + target_tests: list[str], + phase: str, + head_at_open: str, + title: str | None, + ) -> int: + return self.insert( + "cycle", + run_id=run_id, + ordinal=ordinal, + kind=kind, + projects=json.dumps(projects), + declared_tests=json.dumps(declared_tests), + target_tests=json.dumps(target_tests), + phase=phase, + head_at_open=head_at_open, + title=title, + opened_at=now(), + ) + + def move_cycle(self, cycle_id: int, *, from_phase: str, to_phase: str) -> None: + """Record a phase transition and put the cycle in its new phase.""" + self.insert( + "transition", cycle_id=cycle_id, from_phase=from_phase, to_phase=to_phase, at=now() + ) + self.update("cycle", cycle_id, phase=to_phase) + + def mark_cycle_closed(self, cycle_id: int) -> None: + self.update("cycle", cycle_id, closed_at=now()) + + def set_run_outcome(self, run_id: int, outcome: str) -> None: + self.update("run", run_id, outcome=outcome) + + def run_target_tests(self, run_id: int) -> list[list[str]]: + """Every cycle's targets in the run, one list per cycle.""" + rows = self.all("SELECT target_tests FROM cycle WHERE run_id = ?", (run_id,)) + return [json.loads(r["target_tests"]) for r in rows] + + def record_invocation( + self, + run_id: int, + cycle_id: int | None, + *, + phase_at: str, + project: str, + adapter: str, + target_test: str | None, + target_outcome: str | None, + target_failure: str, + total_passed: int, + total_failed: int, + other_failures: list[str], + others_observed: bool = True, + duration_ms: int, + retried: bool = False, + tree_hash: str | None, + ) -> int: + return self.insert( + "invocation", + run_id=run_id, + cycle_id=cycle_id, + phase_at=phase_at, + project=project, + adapter=adapter, + target_test=target_test, + target_outcome=target_outcome, + target_failure=target_failure, + total_passed=total_passed, + total_failed=total_failed, + other_failures=json.dumps(other_failures), + others_observed=int(others_observed), + duration_ms=duration_ms, + retried=int(retried), + tree_hash=tree_hash, + started_at=now(), + ) + + def last_gate(self, run_id: int, project: str, kind: str) -> sqlite3.Row | None: + return self.one( + "SELECT ok, tree_hash FROM gate_result" + " WHERE run_id = ? AND project = ? AND kind = ?" + " ORDER BY id DESC LIMIT 1", + (run_id, project, kind), + ) + + def record_gate( + self, + run_id: int, + cycle_id: int | None, + *, + project: str, + kind: str, + ok: bool, + output: str, + tree_hash: str | None, + skipped: bool = False, + ) -> None: + self.insert( + "gate_result", + run_id=run_id, + cycle_id=cycle_id, + project=project, + kind=kind, + ok=int(ok), + output=output, + tree_hash=tree_hash, + skipped=int(skipped), + at=now(), + ) + + def record_artifact_check( + self, + run_id: int, + cycle_id: int | None, + *, + artifact: str, + stale: bool, + regenerate_failed: bool, + ) -> int: + return self.insert( + "artifact_check", + run_id=run_id, + cycle_id=cycle_id, + artifact=artifact, + stale=int(stale), + regenerated=0, + regenerate_failed=int(regenerate_failed), + at=now(), + ) + + def mark_artifact_check( + self, run_id: int, check_id: int, *, regenerated: bool = False, failed: bool = False + ) -> None: + """Flag an artifact check regenerated, or its regenerate hook failed.""" + cols = {} + if regenerated: + cols["regenerated"] = 1 + if failed: + cols["regenerate_failed"] = 1 + sets = ", ".join(f"{k} = ?" for k in cols) + self._write( + f"UPDATE artifact_check SET {sets} WHERE id = ? AND run_id = ?", + (*cols.values(), check_id, run_id), + ) + + def record_commit( + self, + run_id: int, + cycle_id: int | None, + *, + phase: str, + sha: str, + message: str, + files: list[str], + ) -> None: + self.insert( + "commit_record", + run_id=run_id, + cycle_id=cycle_id, + phase=phase, + sha=sha, + message=message, + files=json.dumps(files), + at=now(), + ) + + def event_details(self, run_id: int, kind: str) -> list[str]: + """The detail of every integrity event of one kind in the run, oldest first.""" + rows = self.all( + "SELECT detail FROM integrity_event WHERE run_id = ? AND kind = ? ORDER BY id", + (run_id, kind), + ) + return [r["detail"] for r in rows] + + def cycle_event_kinds(self, cycle_id: int, kinds: list[str]) -> list[str]: + """Which of `kinds` were recorded against the cycle, one entry per event.""" + rows = self.all( + "SELECT kind FROM integrity_event WHERE cycle_id = ? AND kind IN ({})".format( + ",".join("?" * len(kinds)) + ), + (cycle_id, *kinds), + ) + return [r["kind"] for r in rows] + + def last_cycle_event_detail(self, cycle_id: int, kind: str) -> str | None: + row = self.one( + "SELECT detail FROM integrity_event WHERE cycle_id = ? AND kind = ?" + " ORDER BY id DESC LIMIT 1", + (cycle_id, kind), + ) + return row["detail"] if row else None + + def cycle_has_event(self, cycle_id: int, kind: str) -> bool: + return ( + self.one( + "SELECT id FROM integrity_event WHERE cycle_id = ? AND kind = ?", (cycle_id, kind) + ) + is not None + ) + + def run_has_event(self, run_id: int, kind: str, detail: str) -> bool: + return ( + self.one( + "SELECT id FROM integrity_event WHERE run_id = ? AND kind = ? AND detail = ?", + (run_id, kind, detail), + ) + is not None + ) + + def cycle_has_note(self, cycle_id: int) -> bool: + return self.one("SELECT id FROM note WHERE cycle_id = ?", (cycle_id,)) is not None + + def annotation_keys_of(self, cycle_id: int) -> set[str]: + rows = self.all("SELECT key FROM annotation WHERE cycle_id = ?", (cycle_id,)) + return {r["key"] for r in rows} + def open_ledger(repo_path: Path) -> Ledger: """The ledger of a repository. Every command opens its ledger through here.""" diff --git a/src/tddcli/machine.py b/src/tddcli/machine.py index 58184cc..b42614d 100644 --- a/src/tddcli/machine.py +++ b/src/tddcli/machine.py @@ -23,7 +23,7 @@ from .config import Config from .contract import PIN, REFACTOR, DeclaredCycle from .envelope import Verb, heartbeat -from .ledger import Ledger, now +from .ledger import Ledger AWAITING_TEST = "AWAITING_TEST" AWAITING_IMPL = "AWAITING_IMPL" @@ -71,9 +71,7 @@ def __init__(self, ledger: Ledger, config: Config, worktree: Path, run_row): self.config = config self.worktree = worktree self.run = run_row - self.contract_row = ledger.one( - "SELECT * FROM plan_contract WHERE id = ?", (run_row["plan_contract_id"],) - ) + self.contract_row = ledger.contract(run_row["plan_contract_id"]) self.declared = contract_mod.cycles_from_json(self.contract_row["declared_cycles"]) self.annotation_keys = json.loads(self.contract_row["annotation_keys"]) self.ancillary_files = json.loads(self.contract_row["ancillary_files"]) @@ -238,10 +236,9 @@ def run_projects( if verdict.target_failure: failure_text = verdict.target_failure - self.ledger.insert( - "invocation", - run_id=self.run["id"], - cycle_id=cycle_row["id"], + self.ledger.record_invocation( + self.run["id"], + cycle_row["id"], phase_at=phase, project=name, adapter=adapter.name, @@ -250,12 +247,11 @@ def run_projects( target_failure=verdict.target_failure[:2000], total_passed=len(verdict.passed), total_failed=len(verdict.failed), - other_failures=json.dumps(other), - others_observed=int(observed), + other_failures=other, + others_observed=bool(observed), duration_ms=verdict.duration_ms, - retried=int(retried), + retried=bool(retried), tree_hash=tree, - started_at=now(), ) if verdict.error: self.ledger.event(self.run["id"], cycle_row["id"], "tooling_defect", verdict.error) @@ -287,37 +283,28 @@ def sweep(self, cycle_row, touched: set[str], skip_own: bool = False) -> SweepOu current_hash = self.tree_hash([name]) gate_failed = False for kind, gate_fn in (("lint", adapter.lint), ("typecheck", adapter.typecheck)): - last = self.ledger.one( - "SELECT ok, tree_hash FROM gate_result" - " WHERE run_id = ? AND project = ? AND kind = ?" - " ORDER BY id DESC LIMIT 1", - (self.run["id"], name, kind), - ) + last = self.ledger.last_gate(self.run["id"], name, kind) if last and last["ok"] == 1 and last["tree_hash"] == current_hash: - self.ledger.insert( - "gate_result", - run_id=self.run["id"], - cycle_id=cycle_row["id"], + self.ledger.record_gate( + self.run["id"], + cycle_row["id"], project=name, kind=kind, - ok=1, + ok=True, output="", tree_hash=current_hash, - skipped=1, - at=now(), + skipped=True, ) continue gate = gate_fn() - self.ledger.insert( - "gate_result", - run_id=self.run["id"], - cycle_id=cycle_row["id"], + self.ledger.record_gate( + self.run["id"], + cycle_row["id"], project=name, kind=kind, - ok=int(gate.ok), + ok=gate.ok, output=gate.output, tree_hash=current_hash, - at=now(), ) if not gate.ok: gates.append((name, kind, gate.output)) @@ -343,22 +330,10 @@ def sweep(self, cycle_row, touched: set[str], skip_own: bool = False) -> SweepOu if name not in baselines: probe = self.probe_at_start_sha(name) if probe.observed: - self.ledger.insert( - "baseline", - run_id=self.run["id"], - project=name, - failing=json.dumps(sorted(probe.failing)), - captured_at=now(), - source="late_probe", - ) - self.ledger.insert( - "collection_snapshot", - run_id=self.run["id"], - project=name, - tests=json.dumps([]), - failed_files=json.dumps([]), - captured_at=now(), + self.ledger.record_baseline( + self.run["id"], name, sorted(probe.failing), source="late_probe" ) + self.ledger.record_collection(self.run["id"], name, [], []) self.ledger.event( self.run["id"], cycle_row["id"], @@ -392,10 +367,9 @@ def sweep(self, cycle_row, touched: set[str], skip_own: bool = False) -> SweepOu else: base = baselines[name] failures.extend(f for f in verdict.failed if f not in base) - self.ledger.insert( - "invocation", - run_id=self.run["id"], - cycle_id=cycle_row["id"], + self.ledger.record_invocation( + self.run["id"], + cycle_row["id"], phase_at="CLOSE_SWEEP", project=name, adapter=adapter.name, @@ -404,10 +378,9 @@ def sweep(self, cycle_row, touched: set[str], skip_own: bool = False) -> SweepOu target_failure="", total_passed=len(verdict.passed), total_failed=len(verdict.failed), - other_failures=json.dumps(verdict.failed), + other_failures=verdict.failed, duration_ms=verdict.duration_ms, tree_hash=self.tree_hash([name]), - started_at=now(), ) return SweepOutcome( failures=failures, gates=gates, unbaselined=unbaselined, unobserved=unobserved @@ -423,15 +396,12 @@ def check_artifacts(self, cycle_row) -> ArtifactOutcome: if not art.check and not art.regenerate: continue stale, failure = self._artifact_stale(art) - check_id = self.ledger.insert( - "artifact_check", - run_id=self.run["id"], - cycle_id=cycle_row["id"] if cycle_row else None, + check_id = self.ledger.record_artifact_check( + self.run["id"], + cycle_row["id"] if cycle_row else None, artifact=art.name, - stale=int(stale), - regenerated=0, - regenerate_failed=int(bool(failure)), - at=now(), + stale=stale, + regenerate_failed=bool(failure), ) if failure: self.ledger.event( @@ -449,7 +419,7 @@ def check_artifacts(self, cycle_row) -> ArtifactOutcome: rcode, _rout, rerr = adapters.base.run_command(art.regenerate, self.worktree) if rcode != 0: hook_failure = {"artifact": art.name, "code": rcode, "stderr": rerr[-2000:]} - self.ledger.update("artifact_check", check_id, regenerate_failed=1) + self.ledger.mark_artifact_check(self.run["id"], check_id, failed=True) self.ledger.event( self.run["id"], cycle_row["id"] if cycle_row else None, @@ -476,17 +446,15 @@ def check_artifacts(self, cycle_row) -> ArtifactOutcome: self.trailers(cycle_row, "artifact") if cycle_row else {}, ) if sha: - self.ledger.insert( - "commit_record", - run_id=self.run["id"], - cycle_id=cycle_row["id"] if cycle_row else None, + self.ledger.record_commit( + self.run["id"], + cycle_row["id"] if cycle_row else None, phase="artifact", sha=sha, message=f"chore({art.name}): regenerate", - files=json.dumps(staged), - at=now(), + files=staged, ) - self.ledger.update("artifact_check", check_id, regenerated=1) + self.ledger.mark_artifact_check(self.run["id"], check_id, regenerated=True) resolved = True regenerated.append(art.name) if not resolved: @@ -516,27 +484,22 @@ def open_cycle(self, ordinal: int): declared = self.declared_for(ordinal) if declared is None: return None - existing = self.ledger.one( - "SELECT * FROM cycle WHERE run_id = ? AND ordinal = ? AND closed_at IS NULL", - (self.run["id"], ordinal), - ) + existing = self.ledger.unclosed_cycle(self.run["id"], ordinal) if existing is not None: return existing phase = OPENING_PHASE.get(declared.kind, AWAITING_TEST) - cycle_id = self.ledger.insert( - "cycle", - run_id=self.run["id"], + cycle_id = self.ledger.add_cycle( + self.run["id"], ordinal=ordinal, kind=declared.kind, - projects=json.dumps(declared.projects), - declared_tests=json.dumps([self._qualify(declared, t) for t in declared.tests]), - target_tests=json.dumps([self._qualify(declared, t) for t in declared.tests]), + projects=declared.projects, + declared_tests=[self._qualify(declared, t) for t in declared.tests], + target_tests=[self._qualify(declared, t) for t in declared.tests], phase=phase, head_at_open=gitutil.head(self.worktree), title=declared.title, - opened_at=now(), ) - return self.ledger.one("SELECT * FROM cycle WHERE id = ?", (cycle_id,)) + return self.ledger.cycle(cycle_id) @staticmethod def _qualify(declared: DeclaredCycle, test_id: str) -> str: @@ -548,24 +511,17 @@ def _qualify(declared: DeclaredCycle, test_id: str) -> str: return f"{declared.projects[0]}::{test_id}" def transition(self, cycle_row, to_phase: str) -> None: - self.ledger.insert( - "transition", - cycle_id=cycle_row["id"], - from_phase=cycle_row["phase"], - to_phase=to_phase, - at=now(), - ) - self.ledger.update("cycle", cycle_row["id"], phase=to_phase) + self.ledger.move_cycle(cycle_row["id"], from_phase=cycle_row["phase"], to_phase=to_phase) def close_cycle(self, cycle_row): - row = self.ledger.one("SELECT closed_at FROM cycle WHERE id = ?", (cycle_row["id"],)) + row = self.ledger.cycle(cycle_row["id"]) if row and row["closed_at"] is not None: return self.ledger.open_cycle(self.run["id"]) self.transition(cycle_row, CLOSED) - self.ledger.update("cycle", cycle_row["id"], closed_at=now()) + self.ledger.mark_cycle_closed(cycle_row["id"]) nxt = next((c for c in self.declared if c.ordinal > cycle_row["ordinal"]), None) if nxt is None: - self.ledger.update("run", self.run["id"], ended_at=now(), outcome="complete") + self.ledger.end_run(self.run["id"], "complete") return None return self.open_cycle(nxt.ordinal) @@ -575,14 +531,9 @@ def close_undeclared_gate(self, cycle_row) -> list[str]: Cycle 1: returns all paths that appeared in any undeclared_file_touched event for this run (crude — cycle 2 will restrict to dirty-only). """ - rows = self.ledger.all( - "SELECT detail FROM integrity_event" - " WHERE run_id = ? AND kind = 'undeclared_file_touched'", - (self.run["id"],), - ) flagged: set[str] = set() - for row in rows: - flagged.update(json.loads(row["detail"])) + for detail in self.ledger.event_details(self.run["id"], "undeclared_file_touched"): + flagged.update(json.loads(detail)) dirty = gitutil.dirty_paths(self.worktree) tracked = gitutil.tracked_at_head(self.worktree, list(flagged)) dropped = sorted(p for p in flagged if p not in dirty and p not in tracked) @@ -591,15 +542,8 @@ def close_undeclared_gate(self, cycle_row) -> list[str]: return sorted(p for p in flagged if p in dirty) def record_commit(self, cycle_row, phase: str, sha: str, message: str, files: list[str]): - self.ledger.insert( - "commit_record", - run_id=self.run["id"], - cycle_id=cycle_row["id"], - phase=phase, - sha=sha, - message=message, - files=json.dumps(files), - at=now(), + self.ledger.record_commit( + self.run["id"], cycle_row["id"], phase=phase, sha=sha, message=message, files=files ) def opening_action(self, cycle_row) -> tuple[Verb, str]: @@ -623,10 +567,5 @@ def opening_action(self, cycle_row) -> tuple[Verb, str]: def missing_annotations(self, cycle_row) -> list[str]: if not self.annotation_keys: return [] - present = { - r["key"] - for r in self.ledger.all( - "SELECT key FROM annotation WHERE cycle_id = ?", (cycle_row["id"],) - ) - } + present = self.ledger.annotation_keys_of(cycle_row["id"]) return [k for k in self.annotation_keys if k not in present] From 980bf403212395b22fb9052c4b8abecfb8b3e2b0 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:36:11 +0100 Subject: [PATCH 03/88] refactor: the final close settles blocked-or-complete before ending the run TDD-Run: 1 TDD-Cycle: 3 TDD-Phase: refactor --- src/tddcli/advance.py | 15 +++++++++------ src/tddcli/ledger.py | 3 --- src/tddcli/machine.py | 5 +++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/tddcli/advance.py b/src/tddcli/advance.py index f5b4eb9..41558a9 100644 --- a/src/tddcli/advance.py +++ b/src/tddcli/advance.py @@ -575,14 +575,17 @@ def _handle_refactor(engine: Engine, cycle, retried: bool) -> Envelope: commit=sha, ) - nxt = engine.close_cycle(cycle) + # The last cycle settles the run's outcome before ending it: a run is never ended + # complete and then reopened as blocked. + last = not any(c.ordinal > cycle["ordinal"] for c in engine.declared) + blocking = engine.close_undeclared_gate(cycle) if last else [] + if blocking: + engine.ledger.add_blocker( + engine.run["id"], cycle["id"], "undeclared_file_uncommitted", json.dumps(blocking) + ) + nxt = engine.close_cycle(cycle, outcome="blocked" if blocking else "complete") if nxt is None: - blocking = engine.close_undeclared_gate(cycle) if blocking: - engine.ledger.add_blocker( - engine.run["id"], cycle["id"], "undeclared_file_uncommitted", json.dumps(blocking) - ) - engine.ledger.set_run_outcome(engine.run["id"], "blocked") return Envelope( run={ "id": engine.run["id"], diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 8580168..0b84a09 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -974,9 +974,6 @@ def move_cycle(self, cycle_id: int, *, from_phase: str, to_phase: str) -> None: def mark_cycle_closed(self, cycle_id: int) -> None: self.update("cycle", cycle_id, closed_at=now()) - def set_run_outcome(self, run_id: int, outcome: str) -> None: - self.update("run", run_id, outcome=outcome) - def run_target_tests(self, run_id: int) -> list[list[str]]: """Every cycle's targets in the run, one list per cycle.""" rows = self.all("SELECT target_tests FROM cycle WHERE run_id = ?", (run_id,)) diff --git a/src/tddcli/machine.py b/src/tddcli/machine.py index b42614d..5b3404c 100644 --- a/src/tddcli/machine.py +++ b/src/tddcli/machine.py @@ -513,7 +513,8 @@ def _qualify(declared: DeclaredCycle, test_id: str) -> str: def transition(self, cycle_row, to_phase: str) -> None: self.ledger.move_cycle(cycle_row["id"], from_phase=cycle_row["phase"], to_phase=to_phase) - def close_cycle(self, cycle_row): + def close_cycle(self, cycle_row, outcome: str = "complete"): + """Close the cycle and open the next; after the last, end the run with `outcome`.""" row = self.ledger.cycle(cycle_row["id"]) if row and row["closed_at"] is not None: return self.ledger.open_cycle(self.run["id"]) @@ -521,7 +522,7 @@ def close_cycle(self, cycle_row): self.ledger.mark_cycle_closed(cycle_row["id"]) nxt = next((c for c in self.declared if c.ordinal > cycle_row["ordinal"]), None) if nxt is None: - self.ledger.end_run(self.run["id"], "complete") + self.ledger.end_run(self.run["id"], outcome) return None return self.open_cycle(nxt.ordinal) From 4bd48b2e2e3e74ce1334feefc0e90a2bde8a47e5 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:38:29 +0100 Subject: [PATCH 04/88] refactor: fleet reads through a read-only Ledger TDD-Run: 1 TDD-Cycle: 4 TDD-Phase: refactor --- src/tddcli/cli.py | 4 ++- src/tddcli/fleet.py | 69 ++++++++++++++------------------------------ src/tddcli/ledger.py | 42 +++++++++++++++++++++++++++ tests/test_fleet.py | 9 +++--- 4 files changed, 72 insertions(+), 52 deletions(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 3ff7d2b..30cc9de 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1545,7 +1545,9 @@ def cmd_fleet(args) -> Envelope: `_context`: no tdd.toml, active run, or existing ledger is required, and the ledger is opened read-only so live agents cannot be perturbed.""" worktree = _worktree() - summary = fleet.summarise(ledger_path(gitutil.repo_identity(worktree))) + summary = fleet.summarise( + ledger_mod.open_readonly(ledger_path(gitutil.repo_identity(worktree))) + ) if args.json: return Envelope(result=summary, next_action=NextAction(Verb.COMPLETE, "Fleet reported.")) sys.stdout.write(fleet.render(summary)) diff --git a/src/tddcli/fleet.py b/src/tddcli/fleet.py index b5f0341..2246729 100644 --- a/src/tddcli/fleet.py +++ b/src/tddcli/fleet.py @@ -1,32 +1,20 @@ """Fleet view — every agent's progress on this repository, in one summary. The ledger is one SQLite database per repository, shared by all worktrees, so the -data already exists in one place; this module only reads it. Read-only is -structural, not conventional: the database is opened with SQLite's `mode=ro` URI, -so the command cannot create, migrate, or mutate the ledger that live agents are -writing mid-run. That is what makes it safe to run — from any worktree, on any -branch — while runs are in flight, even if this code's schema constant were ever -to drift from the one on disk. +data already exists in one place; this module only reads it, through a reader from +`ledger.open_readonly`. Read-only is structural, not conventional: that reader +cannot create, migrate, or mutate the ledger that live agents are writing mid-run. +That is what makes it safe to run — from any worktree, on any branch — while runs +are in flight. """ from __future__ import annotations import json -import sqlite3 from datetime import datetime, timezone -from pathlib import Path from . import leases -from .ledger import claim_is_stale - - -def open_readonly(path: Path) -> sqlite3.Connection | None: - """None when no ledger exists yet — `mode=ro` also refuses to create one.""" - if not path.is_file(): - return None - conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) - conn.row_factory = sqlite3.Row - return conn +from .ledger import Ledger, claim_is_stale def _age_s(iso: str | None) -> float | None: @@ -38,24 +26,11 @@ def _age_s(iso: str | None) -> float | None: return round((datetime.now(timezone.utc) - stamp).total_seconds(), 1) -def _runs(conn: sqlite3.Connection) -> list[dict]: - rows = conn.execute( - "SELECT r.id, r.worktree_path, r.executor_model, r.started_at," - " p.plan_path, p.declared_cycles" - " FROM run r JOIN plan_contract p ON p.id = r.plan_contract_id" - " WHERE r.ended_at IS NULL ORDER BY r.id" - ).fetchall() +def _runs(reader: Ledger) -> list[dict]: out = [] - for row in rows: - cycle = conn.execute( - "SELECT ordinal, phase, title FROM cycle" - " WHERE run_id = ? AND closed_at IS NULL ORDER BY ordinal LIMIT 1", - (row["id"],), - ).fetchone() - last = conn.execute( - "SELECT MAX(started_at) AS at FROM invocation WHERE run_id = ?", - (row["id"],), - ).fetchone() + for row in reader.active_runs(): + cycle = reader.open_cycle(row["id"]) + last_at = reader.last_invocation_at(row["id"]) out.append( { "run_id": row["id"], @@ -69,14 +44,14 @@ def _runs(conn: sqlite3.Connection) -> list[dict]: "title": cycle["title"] if cycle else None, # Staleness signal for a wedged agent: age of the newest suite # invocation, falling back to run start when none has landed yet. - "last_activity_age_s": _age_s(last["at"] or row["started_at"]), + "last_activity_age_s": _age_s(last_at or row["started_at"]), } ) return out -def _claims(conn: sqlite3.Connection) -> list[dict]: - rows = conn.execute("SELECT * FROM baseline_claim ORDER BY id").fetchall() +def _claims(reader: Ledger) -> list[dict]: + rows = reader.baseline_claims() return [ { "worktree": r["worktree_path"], @@ -92,8 +67,8 @@ def _claims(conn: sqlite3.Connection) -> list[dict]: ] -def _advance_claims(conn: sqlite3.Connection) -> list[dict]: - rows = conn.execute("SELECT * FROM advance_claim ORDER BY id").fetchall() +def _advance_claims(reader: Ledger) -> list[dict]: + rows = reader.advance_claims() return [ { "worktree": r["worktree_path"], @@ -106,19 +81,19 @@ def _advance_claims(conn: sqlite3.Connection) -> list[dict]: ] -def summarise(ledger_db: Path) -> dict: - conn = open_readonly(ledger_db) - if conn is None: +def summarise(reader: Ledger | None) -> dict: + """`reader` is from `ledger.open_readonly`: None when no ledger exists yet.""" + if reader is None: return {"runs": [], "collecting": [], "advancing": [], "suites": leases.snapshot()} try: return { - "runs": _runs(conn), - "collecting": _claims(conn), - "advancing": _advance_claims(conn), + "runs": _runs(reader), + "collecting": _claims(reader), + "advancing": _advance_claims(reader), "suites": leases.snapshot(), } finally: - conn.close() + reader.close() def render(summary: dict) -> str: diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 0b84a09..53be4c8 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -427,6 +427,9 @@ def __init__(self, repo_path: Path, *, path: Path | None = None): ) self.db.commit() + def close(self) -> None: + self.db.close() + def _stored_version(self) -> int | None: """The schema version already on disk, or None for a fresh database.""" try: @@ -1157,6 +1160,45 @@ def annotation_keys_of(self, cycle_id: int) -> set[str]: rows = self.all("SELECT key FROM annotation WHERE cycle_id = ?", (cycle_id,)) return {r["key"] for r in rows} + # -- the fleet's view ------------------------------------------------------ + + def active_runs(self) -> list[sqlite3.Row]: + """Every run not yet ended, in any worktree, with its plan.""" + return self.all( + "SELECT r.id, r.worktree_path, r.executor_model, r.started_at," + " p.plan_path, p.declared_cycles" + " FROM run r JOIN plan_contract p ON p.id = r.plan_contract_id" + " WHERE r.ended_at IS NULL ORDER BY r.id" + ) + + def last_invocation_at(self, run_id: int) -> str | None: + row = self.one("SELECT MAX(started_at) AS at FROM invocation WHERE run_id = ?", (run_id,)) + return row["at"] if row else None + + def baseline_claims(self) -> list[sqlite3.Row]: + return self.all("SELECT * FROM baseline_claim ORDER BY id") + + def advance_claims(self) -> list[sqlite3.Row]: + return self.all("SELECT * FROM advance_claim ORDER BY id") + + +def open_readonly(path: Path) -> Ledger | None: + """A reader on an existing ledger, or None when there is none yet. + + Read-only is structural: the file is opened with SQLite's `mode=ro` URI, which also + refuses to create it, and no schema script or migration runs. A reader therefore + cannot perturb a ledger that live agents are writing, even if this code's schema + constant were ever to drift from the one on disk. + """ + if not path.is_file(): + return None + reader = Ledger.__new__(Ledger) + reader.repo_path = None + reader.path = path + reader.db = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + reader.db.row_factory = sqlite3.Row + return reader + def open_ledger(repo_path: Path) -> Ledger: """The ledger of a repository. Every command opens its ledger through here.""" diff --git a/tests/test_fleet.py b/tests/test_fleet.py index 3e2c9e7..b93f028 100644 --- a/tests/test_fleet.py +++ b/tests/test_fleet.py @@ -17,6 +17,7 @@ from conftest import run_cli, write_plan from tddcli import fleet, leases +from tddcli import ledger as ledger_mod from tddcli.ledger import Ledger, ledger_path PLAN = """--- @@ -127,18 +128,18 @@ def test_fleet_human_output_renders_one_line_per_run(repo): def test_open_readonly_cannot_write(repo, ledger_home): Ledger(repo) # create the db - conn = fleet.open_readonly(ledger_path(repo)) + reader = ledger_mod.open_readonly(ledger_path(repo)) try: - conn.execute("INSERT INTO meta(key, value) VALUES ('x', 'y')") + reader.db.execute("INSERT INTO meta(key, value) VALUES ('x', 'y')") raise AssertionError("write succeeded on a read-only connection") except sqlite3.OperationalError: pass finally: - conn.close() + reader.close() def test_open_readonly_returns_none_for_missing_db(tmp_path): - assert fleet.open_readonly(tmp_path / "absent.sqlite3") is None + assert ledger_mod.open_readonly(tmp_path / "absent.sqlite3") is None # -- lease snapshot ------------------------------------------------------ From 5a3db99eda05e9fb37653f4646b934e0deabe5f1 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:40:17 +0100 Subject: [PATCH 05/88] test: only the ledger module speaks SQL TDD-Run: 1 TDD-Cycle: 5 TDD-Phase: red --- tests/test_ledger_surface.py | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/test_ledger_surface.py diff --git a/tests/test_ledger_surface.py b/tests/test_ledger_surface.py new file mode 100644 index 0000000..6c5cd1e --- /dev/null +++ b/tests/test_ledger_surface.py @@ -0,0 +1,63 @@ +"""The ledger's surface: one query layer, and every method of it classified for guests. + +A ledger service exposes `Ledger` methods by name over a socket, so the only SQL in the +codebase must be the SQL behind those names. A module that spoke SQL on its own would +be a query the service cannot see, scope or refuse. +""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +import tddcli + +SRC = Path(tddcli.__file__).resolve().parent + +SQL = re.compile( + r"^\s*(SELECT|INSERT\s+INTO|UPDATE\s+\w+\s+SET|DELETE\s+FROM|CREATE\s+(TABLE|INDEX)" + r"|ALTER\s+TABLE|PRAGMA)\b", + re.IGNORECASE, +) +GENERIC = {"one", "all", "insert", "update", "db", "_write"} + + +def _receiver_name(node: ast.expr) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return None + + +def _offences(path: Path) -> list[str]: + found: list[str] = [] + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if isinstance(node, ast.Import) and any(a.name == "sqlite3" for a in node.names): + found.append(f"{node.lineno}: import sqlite3") + elif isinstance(node, ast.ImportFrom) and node.module == "sqlite3": + found.append(f"{node.lineno}: from sqlite3") + elif ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and SQL.match(node.value) + ): + found.append(f"{node.lineno}: SQL {node.value.strip()[:40]!r}") + elif ( + isinstance(node, ast.Attribute) + and node.attr in GENERIC + and _receiver_name(node.value) == "ledger" + ): + found.append(f"{node.lineno}: ledger.{node.attr}") + return found + + +def test_no_module_but_the_ledger_speaks_sql(): + offenders = { + str(path.relative_to(SRC)): found + for path in sorted(SRC.rglob("*.py")) + if path.name != "ledger.py" or path.parent != SRC + if (found := _offences(path)) + } + assert offenders == {} From f416e6bfbb525da05cef9dbac22b90707f2823a3 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:44:10 +0100 Subject: [PATCH 06/88] refactor: render reaches the ledger through named domain methods TDD-Run: 1 TDD-Cycle: 5 TDD-Phase: green --- src/tddcli/ledger.py | 73 ++++++++++++++++++++++++++++ src/tddcli/render.py | 110 +++++++++++-------------------------------- 2 files changed, 101 insertions(+), 82 deletions(-) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 53be4c8..c608381 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -1181,6 +1181,79 @@ def baseline_claims(self) -> list[sqlite3.Row]: def advance_claims(self) -> list[sqlite3.Row]: return self.all("SELECT * FROM advance_claim ORDER BY id") + # -- projections: the friction log, progress and metrics ------------------- + + def runs_in(self, worktree: str) -> list[sqlite3.Row]: + return self.all("SELECT * FROM run WHERE worktree_path = ? ORDER BY id", (worktree,)) + + def run_events(self, run_id: int) -> list[sqlite3.Row]: + return self.all("SELECT * FROM integrity_event WHERE run_id = ? ORDER BY id", (run_id,)) + + def cycle_events(self, cycle_id: int) -> list[sqlite3.Row]: + return self.all("SELECT * FROM integrity_event WHERE cycle_id = ? ORDER BY id", (cycle_id,)) + + def event_counts(self, run_id: int) -> list[sqlite3.Row]: + """(kind, n) per integrity event kind in the run.""" + return self.all( + "SELECT kind, COUNT(*) n FROM integrity_event WHERE run_id = ? GROUP BY kind", + (run_id,), + ) + + def regenerated_artifacts(self, run_id: int) -> list[str]: + rows = self.all( + "SELECT DISTINCT artifact FROM artifact_check WHERE run_id = ? AND regenerated = 1", + (run_id,), + ) + return [r["artifact"] for r in rows] + + def interventions(self, run_id: int) -> list[sqlite3.Row]: + return self.all("SELECT * FROM human_intervention WHERE run_id = ? ORDER BY id", (run_id,)) + + def commits(self, cycle_id: int) -> list[sqlite3.Row]: + return self.all("SELECT * FROM commit_record WHERE cycle_id = ? ORDER BY id", (cycle_id,)) + + def annotations(self, cycle_id: int) -> list[sqlite3.Row]: + return self.all("SELECT * FROM annotation WHERE cycle_id = ? ORDER BY id", (cycle_id,)) + + def cycle_notes(self, cycle_id: int) -> list[sqlite3.Row]: + return self.all("SELECT * FROM note WHERE cycle_id = ? ORDER BY id", (cycle_id,)) + + def run_notes(self, run_id: int) -> list[sqlite3.Row]: + """Notes about the run as a whole, recorded while no cycle was open.""" + return self.all( + "SELECT * FROM note WHERE run_id = ? AND cycle_id IS NULL ORDER BY id", (run_id,) + ) + + def blockers(self, run_id: int) -> list[sqlite3.Row]: + return self.all("SELECT * FROM blocker WHERE run_id = ? ORDER BY id", (run_id,)) + + def blocker_counts(self, run_id: int) -> list[sqlite3.Row]: + """(kind, n) per blocker kind in the run.""" + return self.all( + "SELECT kind, COUNT(*) n FROM blocker WHERE run_id = ? GROUP BY kind", (run_id,) + ) + + def abandonment(self, run_id: int) -> sqlite3.Row | None: + return self.one("SELECT * FROM abandonment WHERE run_id = ?", (run_id,)) + + def suite_time_by_phase(self, run_id: int) -> list[sqlite3.Row]: + """(phase_at, n, ms) over every invocation of the run.""" + return self.all( + "SELECT phase_at, COUNT(*) n, SUM(duration_ms) ms FROM invocation" + " WHERE run_id = ? GROUP BY phase_at", + (run_id,), + ) + + def cycle_times(self, run_id: int) -> list[sqlite3.Row]: + """(ordinal, opened_at, closed_at, n, ms) per cycle: its suite runs and their time.""" + return self.all( + "SELECT c.ordinal, c.opened_at, c.closed_at," + " COUNT(i.id) n, COALESCE(SUM(i.duration_ms), 0) ms" + " FROM cycle c LEFT JOIN invocation i ON i.cycle_id = c.id" + " WHERE c.run_id = ? GROUP BY c.id ORDER BY c.ordinal", + (run_id,), + ) + def open_readonly(path: Path) -> Ledger | None: """A reader on an existing ledger, or None when there is none yet. diff --git a/src/tddcli/render.py b/src/tddcli/render.py index e4b5d1c..1fb8878 100644 --- a/src/tddcli/render.py +++ b/src/tddcli/render.py @@ -17,7 +17,7 @@ def _fmt_list(items) -> str: def friction_log(ledger: Ledger, run) -> str: - contract = ledger.one("SELECT * FROM plan_contract WHERE id = ?", (run["plan_contract_id"],)) + contract = ledger.contract(run["plan_contract_id"]) cycles = ledger.cycles(run["id"]) lines: list[str] = [] a = lines.append @@ -31,9 +31,7 @@ def friction_log(ledger: Ledger, run) -> str: f"- Started: {run['started_at']} Ended: {run['ended_at'] or '—'}" f" Outcome: {run['outcome'] or 'live'}" ) - all_baseline_rows = ledger.all( - "SELECT project, failing, source FROM baseline WHERE run_id = ?", (run["id"],) - ) + all_baseline_rows = ledger.baseline_rows(run["id"]) run_start_baselines = { r["project"]: json.loads(r["failing"]) for r in all_baseline_rows @@ -51,12 +49,9 @@ def friction_log(ledger: Ledger, run) -> str: f"{r['project']}={len(json.loads(r['failing']))}" for r in late_probe_rows ) a(f"- Late baselines: {parts} (at {sha7})") - regen_rows = ledger.all( - "SELECT DISTINCT artifact FROM artifact_check WHERE run_id = ? AND regenerated = 1", - (run["id"],), - ) - if regen_rows: - a("- Artifacts auto-regenerated: " + _fmt_list([r["artifact"] for r in regen_rows])) + regenerated = ledger.regenerated_artifacts(run["id"]) + if regenerated: + a("- Artifacts auto-regenerated: " + _fmt_list(regenerated)) a("") declared = json.loads(contract["declared_cycles"]) @@ -68,16 +63,12 @@ def friction_log(ledger: Ledger, run) -> str: a(f"- Delivered: {len(delivered)} Skipped: {len(skipped)}") missing = sorted({d["n"] for d in declared} - delivered - skipped) a(f"- Never reached: {missing or 'none'}") - interventions = ledger.all("SELECT * FROM human_intervention WHERE run_id = ?", (run["id"],)) + interventions = ledger.interventions(run["id"]) a(f"- Human interventions: {len(interventions)}") for i in interventions: a(f" - {i['at']}: {i['note']}") - amended_events = ledger.all( - "SELECT detail FROM integrity_event WHERE run_id = ? AND kind = 'baseline_amended'", - (run["id"],), - ) - for ev in amended_events: - detail = json.loads(ev["detail"]) + for amended in ledger.event_details(run["id"], "baseline_amended"): + detail = json.loads(amended) for project, entry in sorted(detail.items()): for test_id, verdict in sorted(entry.get("accepted", {}).items()): a(f" - accepted: `{test_id}` ({verdict})") @@ -86,7 +77,7 @@ def friction_log(ledger: Ledger, run) -> str: a("") events = defaultdict(list) - for e in ledger.all("SELECT * FROM integrity_event WHERE run_id = ?", (run["id"],)): + for e in ledger.run_events(run["id"]): events[e["cycle_id"]].append(e) for cycle in reversed(cycles): @@ -131,9 +122,7 @@ def friction_log(ledger: Ledger, run) -> str: snippet = sens["observed_failure"].strip().splitlines() a(f" - observed: `{snippet[0][:160] if snippet else ''}`") - commits = ledger.all( - "SELECT * FROM commit_record WHERE cycle_id = ? ORDER BY id", (cycle["id"],) - ) + commits = ledger.commits(cycle["id"]) if commits: a("- **Commits:**") for c in commits: @@ -145,22 +134,15 @@ def friction_log(ledger: Ledger, run) -> str: for e in events.get(cycle["id"], []): a(f"- **Event — {e['kind']}:** {e['detail'][:300]}") - annotations = ledger.all( - "SELECT * FROM annotation WHERE cycle_id = ? ORDER BY id", (cycle["id"],) - ) - for ann in annotations: + for ann in ledger.annotations(cycle["id"]): a(f"- **{ann['key']}:** {ann['value']}") - notes = ledger.all("SELECT * FROM note WHERE cycle_id = ? ORDER BY id", (cycle["id"],)) - for n in notes: + for n in ledger.cycle_notes(cycle["id"]): a(f"> **note** _(during {n['phase']})_: {n['text']}") a("") lines.extend(_time_section(_time_summary(ledger, run))) - run_notes = ledger.all( - "SELECT * FROM note WHERE run_id = ? AND cycle_id IS NULL ORDER BY id", - (run["id"],), - ) + run_notes = ledger.run_notes(run["id"]) if run_notes: a("## Executor narrative") a("") @@ -170,7 +152,7 @@ def friction_log(ledger: Ledger, run) -> str: a(f"> {n['text']}") a("") - blockers = ledger.all("SELECT * FROM blocker WHERE run_id = ?", (run["id"],)) + blockers = ledger.blockers(run["id"]) if blockers: a("## Blockers") a("") @@ -207,7 +189,7 @@ def _elapsed(start: str, end: str | None) -> str: def progress(ledger: Ledger, run) -> str: """A human's view of where the run is. Never consulted by an agent.""" - contract = ledger.one("SELECT * FROM plan_contract WHERE id = ?", (run["plan_contract_id"],)) + contract = ledger.contract(run["plan_contract_id"]) declared = json.loads(contract["declared_cycles"]) rows = {c["ordinal"]: c for c in ledger.cycles(run["id"])} @@ -236,11 +218,8 @@ def progress(ledger: Ledger, run) -> str: continue runs = len(ledger.invocations(row["id"])) - commits = ledger.all( - "SELECT phase, sha FROM commit_record WHERE cycle_id = ? ORDER BY id", - (row["id"],), - ) - events = ledger.all("SELECT kind FROM integrity_event WHERE cycle_id = ?", (row["id"],)) + commits = ledger.commits(row["id"]) + events = ledger.cycle_events(row["id"]) detail = f"{runs} suite run{'' if runs == 1 else 's'}" if commits: detail += " " + " ".join(f"{c['phase']}:{c['sha'][:7]}" for c in commits) @@ -257,10 +236,7 @@ def progress(ledger: Ledger, run) -> str: a(f" ! {e['kind']}") a("") - total_events = ledger.all( - "SELECT kind, COUNT(*) n FROM integrity_event WHERE run_id = ? GROUP BY kind", - (run["id"],), - ) + total_events = ledger.event_counts(run["id"]) summary = f"{closed}/{len(declared)} closed" if skipped: summary += f" · {skipped} skipped" @@ -269,7 +245,7 @@ def progress(ledger: Ledger, run) -> str: if not total_events else " · " + ", ".join(f"{e['kind']}×{e['n']}" for e in total_events) ) - blockers = ledger.all("SELECT kind, detail FROM blocker WHERE run_id = ?", (run["id"],)) + blockers = ledger.blockers(run["id"]) a(summary) for b in blockers: a(f"BLOCKED ({b['kind']}): {b['detail']}") @@ -284,11 +260,7 @@ def _time_summary(ledger: Ledger, run) -> dict: """Where a run's time went (#147): its wall clock, and the suite's share of it by phase. One source for `tdd metrics` and the friction log, so they cannot disagree. Every invocation of the run counts, so the phases sum to the suite total.""" - rows = ledger.all( - "SELECT phase_at, COUNT(*) n, SUM(duration_ms) ms FROM invocation" - " WHERE run_id = ? GROUP BY phase_at", - (run["id"],), - ) + rows = ledger.suite_time_by_phase(run["id"]) by_phase = {r["phase_at"]: {"runs": r["n"], "suite_s": r["ms"] / 1000} for r in rows} def order(phase: str) -> tuple: @@ -302,13 +274,7 @@ def order(phase: str) -> tuple: "suite_s": c["ms"] / 1000, "runs": c["n"], } - for c in ledger.all( - "SELECT c.ordinal, c.opened_at, c.closed_at," - " COUNT(i.id) n, COALESCE(SUM(i.duration_ms), 0) ms" - " FROM cycle c LEFT JOIN invocation i ON i.cycle_id = c.id" - " WHERE c.run_id = ? GROUP BY c.id ORDER BY c.ordinal", - (run["id"],), - ) + for c in ledger.cycle_times(run["id"]) ] wall = _seconds(run["started_at"], run["ended_at"]) suite = sum(p["suite_s"] for p in by_phase.values()) @@ -370,7 +336,7 @@ def _impl_attempts(rows) -> int: def metrics(ledger: Ledger, worktree: str) -> dict: - runs = ledger.all("SELECT * FROM run WHERE worktree_path = ? ORDER BY id", (worktree,)) + runs = ledger.runs_in(worktree) out = { "runs": [], "note": ( @@ -387,17 +353,11 @@ def metrics(ledger: Ledger, worktree: str) -> dict: cycles = ledger.cycles(run["id"]) # Pin cycles pass on arrival by design; refactor cycles have no test at all. standard = [c for c in cycles if c["kind"] not in ("pin", "refactor")] - red_violations = ledger.all( - "SELECT * FROM integrity_event WHERE run_id = ? AND kind = 'red_first_violation'", - (run["id"],), - ) + red_violations = ledger.event_details(run["id"], "red_first_violation") impl_attempts = [ _impl_attempts(ledger.invocations(c["id"], "AWAITING_IMPL")) for c in cycles ] - blockers = ledger.all( - "SELECT kind, COUNT(*) n FROM blocker WHERE run_id = ? GROUP BY kind", - (run["id"],), - ) + blockers = ledger.blocker_counts(run["id"]) by_project = defaultdict(int) for c in cycles: for p in json.loads(c["projects"]): @@ -410,12 +370,7 @@ def metrics(ledger: Ledger, worktree: str) -> dict: "executor_source": run["executor_source"], "outcome": run["outcome"], "cycles_declared": len( - json.loads( - ledger.one( - "SELECT declared_cycles FROM plan_contract WHERE id = ?", - (run["plan_contract_id"],), - )["declared_cycles"] - ) + json.loads(ledger.contract(run["plan_contract_id"])["declared_cycles"]) ), "cycles_closed": sum(1 for c in cycles if c["phase"] == "CLOSED"), "cycles_skipped": sum(1 for c in cycles if c["phase"] == "SKIPPED"), @@ -427,21 +382,12 @@ def metrics(ledger: Ledger, worktree: str) -> dict: "impl_attempts_max": max(impl_attempts, default=0), "cycles_by_project": dict(by_project), "blockers": {b["kind"]: b["n"] for b in blockers}, - "human_interventions": len( - ledger.all("SELECT id FROM human_intervention WHERE run_id = ?", (run["id"],)) - ), + "human_interventions": len(ledger.interventions(run["id"])), "time": _time_json(_time_summary(ledger, run)), - "integrity_events": { - r["kind"]: r["n"] - for r in ledger.all( - "SELECT kind, COUNT(*) n FROM integrity_event WHERE run_id = ?" - " GROUP BY kind", - (run["id"],), - ) - }, + "integrity_events": {r["kind"]: r["n"] for r in ledger.event_counts(run["id"])}, } ) - abandonment = ledger.one("SELECT * FROM abandonment WHERE run_id = ?", (run["id"],)) + abandonment = ledger.abandonment(run["id"]) if abandonment is not None: out["runs"][-1]["abandoned"] = { "reason": abandonment["reason"], From 1b3a92bc827d1c9d9f3eb6638c0c49dcc0ade2db Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:44:38 +0100 Subject: [PATCH 07/88] test: a v12 ledger migrates with its rows tagged local TDD-Run: 1 TDD-Cycle: 6 TDD-Phase: red --- tests/test_ledger_sources.py | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tests/test_ledger_sources.py diff --git a/tests/test_ledger_sources.py b/tests/test_ledger_sources.py new file mode 100644 index 0000000..dff9bad --- /dev/null +++ b/tests/test_ledger_sources.py @@ -0,0 +1,94 @@ +"""Sources in the ledger: every run, contract, claim and cached baseline belongs to one. + +A source is where a run came in from. Single-user mode and single-machine split mode +are both the source `local`; a ledger service names one source per guest socket. +""" + +from __future__ import annotations + +from tddcli.ledger import Ledger, now + +#: The v12 shape of the tables v13 rebuilds, copied from `SCHEMA` before v13. +V12_REBUILT = """ +CREATE TABLE baseline_claim ( + id INTEGER PRIMARY KEY, + worktree_path TEXT NOT NULL UNIQUE, + hostname TEXT NOT NULL, + pid INTEGER NOT NULL, + projects_total INTEGER NOT NULL DEFAULT 0, + projects_done INTEGER NOT NULL DEFAULT 0, + current_project TEXT, + started_at TEXT NOT NULL +); + +CREATE TABLE advance_claim ( + id INTEGER PRIMARY KEY, + worktree_path TEXT NOT NULL UNIQUE, + hostname TEXT NOT NULL, + pid INTEGER NOT NULL, + started_at TEXT NOT NULL +); + +CREATE TABLE baseline_cache ( + id INTEGER PRIMARY KEY, + project TEXT NOT NULL, + tree_hash TEXT NOT NULL, + config_sha TEXT NOT NULL, + failing TEXT NOT NULL, + tests TEXT NOT NULL, + failed_files TEXT NOT NULL, + created_at TEXT NOT NULL, + UNIQUE(project, tree_hash, config_sha) +); +""" + + +def _columns(ledger: Ledger, table: str) -> set[str]: + return {r[1] for r in ledger.db.execute(f"PRAGMA table_info({table})").fetchall()} + + +def _make_v12(ledger: Ledger) -> None: + """Put a freshly opened ledger back in its v12 shape, keeping its rows.""" + for table in ("run", "plan_contract"): + if "source" in _columns(ledger, table): + ledger.db.execute(f"ALTER TABLE {table} DROP COLUMN source") + rows = [dict(r) for r in ledger.db.execute("SELECT * FROM baseline_claim").fetchall()] + ledger.db.executescript( + "DROP TABLE baseline_claim; DROP TABLE advance_claim; DROP TABLE baseline_cache;" + + V12_REBUILT + ) + for row in rows: + row.pop("source", None) + ledger.insert("baseline_claim", **row) + ledger._write("UPDATE meta SET value = '12' WHERE key = 'schema_version'", ()) + + +def test_a_v12_ledger_migrates_with_its_rows_tagged_local(tmp_path, ledger_home): + ledger = Ledger(tmp_path / "somerepo") + contract = ledger.insert( + "plan_contract", + plan_path="tasks/p.md", + status="declared", + declared_cycles="[]", + annotation_keys="[]", + registered_at=now(), + ) + ledger.insert( + "run", + plan_contract_id=contract, + executor_model="m", + executor_source="human", + worktree_path="/w", + started_at=now(), + preexisting_dirty="[]", + ) + ledger.insert("baseline_claim", worktree_path="/w", hostname="h", pid=1, started_at=now()) + _make_v12(ledger) + ledger.close() + + reopened = Ledger(tmp_path / "somerepo") + sources = { + table: dict(reopened.db.execute(f"SELECT * FROM {table}").fetchone()).get("source") + for table in ("run", "plan_contract", "baseline_claim") + } + assert sources == {"run": "local", "plan_contract": "local", "baseline_claim": "local"} From af36dfb4cd77d765a508682bde5afe171cc468fe Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:48:22 +0100 Subject: [PATCH 08/88] feat: ledger schema v13 records each run's source TDD-Run: 1 TDD-Cycle: 6 TDD-Phase: green --- src/tddcli/ledger.py | 87 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 9 deletions(-) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index c608381..37156c5 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -15,7 +15,7 @@ from . import runner -SCHEMA_VERSION = 12 +SCHEMA_VERSION = 13 class LedgerVersionError(RuntimeError): @@ -54,28 +54,94 @@ class LedgerVersionError(RuntimeError): ), # v11 -> v12 added others_observed column to invocation; ALTER TABLE covers old ledgers. 11: "ALTER TABLE invocation ADD COLUMN others_observed INTEGER NOT NULL DEFAULT 1;", + # v12 -> v13 records each row's source. `run` and `plan_contract` gain the column; + # the claims and the baseline cache are rebuilt, because their UNIQUE constraints + # gain it too and SQLite cannot alter a constraint in place. + 12: """ +ALTER TABLE run ADD COLUMN source TEXT NOT NULL DEFAULT 'local'; +ALTER TABLE plan_contract ADD COLUMN source TEXT NOT NULL DEFAULT 'local'; + +CREATE TABLE baseline_claim_v13 ( + id INTEGER PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'local', + worktree_path TEXT NOT NULL, + hostname TEXT NOT NULL, + pid INTEGER NOT NULL, + projects_total INTEGER NOT NULL DEFAULT 0, + projects_done INTEGER NOT NULL DEFAULT 0, + current_project TEXT, + started_at TEXT NOT NULL, + UNIQUE(source, worktree_path) +); +INSERT INTO baseline_claim_v13 (id, worktree_path, hostname, pid, projects_total, + projects_done, current_project, started_at) + SELECT id, worktree_path, hostname, pid, projects_total, projects_done, + current_project, started_at FROM baseline_claim; +DROP TABLE baseline_claim; +ALTER TABLE baseline_claim_v13 RENAME TO baseline_claim; + +CREATE TABLE advance_claim_v13 ( + id INTEGER PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'local', + worktree_path TEXT NOT NULL, + hostname TEXT NOT NULL, + pid INTEGER NOT NULL, + started_at TEXT NOT NULL, + UNIQUE(source, worktree_path) +); +INSERT INTO advance_claim_v13 (id, worktree_path, hostname, pid, started_at) + SELECT id, worktree_path, hostname, pid, started_at FROM advance_claim; +DROP TABLE advance_claim; +ALTER TABLE advance_claim_v13 RENAME TO advance_claim; + +CREATE TABLE baseline_cache_v13 ( + id INTEGER PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'local', + project TEXT NOT NULL, + tree_hash TEXT NOT NULL, + config_sha TEXT NOT NULL, + failing TEXT NOT NULL, + tests TEXT NOT NULL, + failed_files TEXT NOT NULL, + created_at TEXT NOT NULL, + UNIQUE(source, project, tree_hash, config_sha) +); +INSERT INTO baseline_cache_v13 (id, project, tree_hash, config_sha, failing, tests, + failed_files, created_at) + SELECT id, project, tree_hash, config_sha, failing, tests, failed_files, created_at + FROM baseline_cache; +DROP TABLE baseline_cache; +ALTER TABLE baseline_cache_v13 RENAME TO baseline_cache; +""", } SCHEMA = """ CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); +-- No index may name a `source` column: SCHEMA runs before the migrations, so on a +-- pre-v13 ledger that column does not exist yet when this script does. + CREATE TABLE IF NOT EXISTS baseline_claim ( id INTEGER PRIMARY KEY, - worktree_path TEXT NOT NULL UNIQUE, + source TEXT NOT NULL DEFAULT 'local', + worktree_path TEXT NOT NULL, hostname TEXT NOT NULL, pid INTEGER NOT NULL, projects_total INTEGER NOT NULL DEFAULT 0, projects_done INTEGER NOT NULL DEFAULT 0, current_project TEXT, - started_at TEXT NOT NULL + started_at TEXT NOT NULL, + UNIQUE(source, worktree_path) ); CREATE TABLE IF NOT EXISTS advance_claim ( id INTEGER PRIMARY KEY, - worktree_path TEXT NOT NULL UNIQUE, + source TEXT NOT NULL DEFAULT 'local', + worktree_path TEXT NOT NULL, hostname TEXT NOT NULL, pid INTEGER NOT NULL, - started_at TEXT NOT NULL + started_at TEXT NOT NULL, + UNIQUE(source, worktree_path) ); CREATE TABLE IF NOT EXISTS plan_contract ( @@ -87,7 +153,8 @@ class LedgerVersionError(RuntimeError): declared_cycles TEXT NOT NULL, -- json annotation_keys TEXT NOT NULL, -- json ancillary_files TEXT NOT NULL DEFAULT '[]', -- json - registered_at TEXT NOT NULL + registered_at TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'local' ); CREATE TABLE IF NOT EXISTS run ( @@ -103,7 +170,8 @@ class LedgerVersionError(RuntimeError): allow_dirty INTEGER NOT NULL DEFAULT 0, preexisting_dirty TEXT NOT NULL, -- json: excluded from authorship forever (R9.21) config_sha TEXT, -- tdd.toml as of run start; drift is an event - start_sha TEXT -- HEAD at run start; late probes and accept-failures run suites here + start_sha TEXT, -- HEAD at run start; late probes and accept-failures run suites here + source TEXT NOT NULL DEFAULT 'local' -- where the run came in from: `local`, or a ledger service's source ); CREATE TABLE IF NOT EXISTS baseline ( @@ -272,6 +340,7 @@ class LedgerVersionError(RuntimeError): CREATE TABLE IF NOT EXISTS baseline_cache ( id INTEGER PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'local', project TEXT NOT NULL, tree_hash TEXT NOT NULL, config_sha TEXT NOT NULL, @@ -279,7 +348,7 @@ class LedgerVersionError(RuntimeError): tests TEXT NOT NULL, -- json list failed_files TEXT NOT NULL, -- json map path -> error created_at TEXT NOT NULL, - UNIQUE(project, tree_hash, config_sha) + UNIQUE(source, project, tree_hash, config_sha) ); CREATE INDEX IF NOT EXISTS idx_cycle_run ON cycle(run_id); @@ -590,7 +659,7 @@ def cache_baseline( """ INSERT INTO baseline_cache (project, tree_hash, config_sha, failing, tests, failed_files, created_at) VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(project, tree_hash, config_sha) DO UPDATE SET + ON CONFLICT(source, project, tree_hash, config_sha) DO UPDATE SET failing = excluded.failing, tests = excluded.tests, failed_files = excluded.failed_files, From 87008b8653049201035ff28901e53d61208a9432 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:48:47 +0100 Subject: [PATCH 09/88] test: a source reads only its own runs and contracts TDD-Run: 1 TDD-Cycle: 7 TDD-Phase: red --- tests/test_ledger_sources.py | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/test_ledger_sources.py b/tests/test_ledger_sources.py index dff9bad..0893134 100644 --- a/tests/test_ledger_sources.py +++ b/tests/test_ledger_sources.py @@ -92,3 +92,62 @@ def test_a_v12_ledger_migrates_with_its_rows_tagged_local(tmp_path, ledger_home) for table in ("run", "plan_contract", "baseline_claim") } assert sources == {"run": "local", "plan_contract": "local", "baseline_claim": "local"} + + +def _contract(ledger: Ledger) -> int: + return ledger.register_contract( + plan_path="tasks/p.md", + blob_sha="abc", + commit_sha="def", + status="declared", + declared_cycles="[]", + annotation_keys="[]", + ancillary_files="[]", + ) + + +def _start(ledger: Ledger, contract: int, worktree: str) -> int: + return ledger.start_run( + contract, + executor_model="m", + executor_session=None, + executor_source="human", + worktree=worktree, + allow_dirty=False, + preexisting_dirty=[], + config_sha=None, + start_sha=None, + ) + + +def test_a_source_reads_only_its_own_runs_and_contracts(tmp_path, ledger_home): + repo = tmp_path / "somerepo" + a = Ledger(repo, source="a") + b = Ledger(repo, source="b") + worktree = str(tmp_path / "wt") + run_id = _start(a, _contract(a), worktree) + a.record_baseline(run_id, "app", ["t::x"], source="probed") + a.record_invocation( + run_id, + None, + phase_at="CLOSE_SWEEP", + project="app", + adapter="pytest", + target_test=None, + target_outcome=None, + target_failure="", + total_passed=1, + total_failed=0, + other_failures=[], + duration_ms=500, + tree_hash=None, + ) + + seen_by_b = { + "active_run": b.active_run(worktree), + "latest_contract": b.latest_contract("tasks/p.md"), + "previous_baseline": b.previous_baseline(worktree, "app", 10**9), + "max_suite_duration_ms": b.max_suite_duration_ms("app"), + "latest_run": b.latest_run(worktree), + } + assert seen_by_b == dict.fromkeys(seen_by_b) From 3c5b374bfcc3ebf3faee5996628469297670898a Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:52:44 +0100 Subject: [PATCH 10/88] feat: source-rooted ledger reads filter by source TDD-Run: 1 TDD-Cycle: 7 TDD-Phase: green --- src/tddcli/ledger.py | 78 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 18 deletions(-) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 37156c5..b295530 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -403,6 +403,10 @@ def ledger_path(repo_path: Path) -> Path: #: while the agent's uid could still open the ledger. PRE_SPLIT_IMPORT = "pre_split_import" +#: The source of every run recorded on this machine's own ledger: single-user mode and +#: single-machine split mode alike. A ledger service names one source per guest socket. +LOCAL_SOURCE = "local" + def import_legacy(source: Path) -> tuple[Path, dict]: """Bring a single-user ledger under the runner, marked as pre-split history. @@ -453,8 +457,13 @@ def claim_is_stale(hostname: str, pid: int, started_at: str) -> bool: class Ledger: _claim_is_stale = staticmethod(claim_is_stale) - def __init__(self, repo_path: Path, *, path: Path | None = None): + def __init__( + self, repo_path: Path, *, path: Path | None = None, source: str | None = LOCAL_SOURCE + ): self.repo_path = repo_path + # The source this ledger reads and writes as. None is the host's unscoped view + # of every source, built only by the ledger service and by tests. + self.source = source # `path` is for a ledger that is not found by its repository: an imported one. self.path = path or ledger_path(repo_path) # A generous busy timeout: two `run start` calls against one worktree open @@ -558,13 +567,26 @@ def update(self, table: str, row_id: int, **cols) -> None: sets = ", ".join(f"{k} = ?" for k in cols) self._write(f"UPDATE {table} SET {sets} WHERE id = ?", (*cols.values(), row_id)) + # -- sources ----------------------------------------------------------- + + def _scoped(self, column: str = "source") -> tuple[str, tuple]: + """An `AND = ?` clause and its parameter, empty for the unscoped view.""" + if self.source is None: + return "", () + return f" AND {column} = ?", (self.source,) + + def _stamp(self) -> dict: + """The source column for a new row; the unscoped view leaves the default.""" + return {} if self.source is None else {"source": self.source} + # -- domain queries -------------------------------------------------- def active_run(self, worktree: str) -> sqlite3.Row | None: + scope, extra = self._scoped() return self.one( - "SELECT * FROM run WHERE worktree_path = ? AND ended_at IS NULL" + f"SELECT * FROM run WHERE worktree_path = ? AND ended_at IS NULL{scope}" " ORDER BY id DESC LIMIT 1", - (worktree,), + (worktree, *extra), ) def open_cycle(self, run_id: int) -> sqlite3.Row | None: @@ -581,11 +603,12 @@ def baselines(self, run_id: int) -> dict[str, set[str]]: return {r["project"]: set(json.loads(r["failing"])) for r in rows} def previous_baseline(self, worktree: str, project: str, before_run_id: int) -> set[str] | None: + scope, extra = self._scoped("r.source") row = self.one( "SELECT b.failing FROM baseline b JOIN run r ON b.run_id = r.id" - " WHERE r.worktree_path = ? AND b.project = ? AND r.id < ?" + f" WHERE r.worktree_path = ? AND b.project = ? AND r.id < ?{scope}" " ORDER BY r.id DESC LIMIT 1", - (worktree, project, before_run_id), + (worktree, project, before_run_id, *extra), ) if row is None: return None @@ -612,10 +635,16 @@ def max_suite_duration_ms(self, project: str) -> int | None: skip the timeout doctor check rather than emitting a false alarm. Full-suite runs have `target_test IS NULL`. """ + # Excludes other sources' runs rather than joining on this one's, so that an + # invocation is never dropped for want of a run row it can be joined to. + scope, extra = ("", ()) + if self.source is not None: + scope = " AND run_id NOT IN (SELECT id FROM run WHERE source != ?)" + extra = (self.source,) row = self.one( "SELECT MAX(duration_ms) AS m FROM invocation" - " WHERE project = ? AND target_test IS NULL", - (project,), + f" WHERE project = ? AND target_test IS NULL{scope}", + (project, *extra), ) return None if row is None or row["m"] is None else int(row["m"]) @@ -784,15 +813,17 @@ def advance_claim_row(self, worktree: str) -> sqlite3.Row | None: # -- plan contracts ---------------------------------------------------- def contract_by_blob(self, plan_path: str, blob_sha: str | None) -> sqlite3.Row | None: + scope, extra = self._scoped() return self.one( - "SELECT * FROM plan_contract WHERE plan_path = ? AND git_blob_sha IS ?", - (plan_path, blob_sha), + f"SELECT * FROM plan_contract WHERE plan_path = ? AND git_blob_sha IS ?{scope}", + (plan_path, blob_sha, *extra), ) def latest_contract(self, plan_path: str) -> sqlite3.Row | None: + scope, extra = self._scoped() return self.one( - "SELECT * FROM plan_contract WHERE plan_path = ? ORDER BY id DESC LIMIT 1", - (plan_path,), + f"SELECT * FROM plan_contract WHERE plan_path = ?{scope} ORDER BY id DESC LIMIT 1", + (plan_path, *extra), ) def register_contract( @@ -816,6 +847,7 @@ def register_contract( annotation_keys=annotation_keys, ancillary_files=ancillary_files, registered_at=now(), + **self._stamp(), ) # -- runs ---------------------------------------------------------------- @@ -824,15 +856,18 @@ def run(self, run_id: int) -> sqlite3.Row | None: return self.one("SELECT * FROM run WHERE id = ?", (run_id,)) def latest_run(self, worktree: str) -> sqlite3.Row | None: + scope, extra = self._scoped() return self.one( - "SELECT * FROM run WHERE worktree_path = ? ORDER BY id DESC LIMIT 1", (worktree,) + f"SELECT * FROM run WHERE worktree_path = ?{scope} ORDER BY id DESC LIMIT 1", + (worktree, *extra), ) def latest_blocked_run(self, worktree: str) -> sqlite3.Row | None: + scope, extra = self._scoped() return self.one( - "SELECT * FROM run WHERE worktree_path = ? AND outcome = 'blocked'" + f"SELECT * FROM run WHERE worktree_path = ? AND outcome = 'blocked'{scope}" " ORDER BY id DESC LIMIT 1", - (worktree,), + (worktree, *extra), ) def start_run( @@ -860,6 +895,7 @@ def start_run( preexisting_dirty=json.dumps(preexisting_dirty), config_sha=config_sha, start_sha=start_sha, + **self._stamp(), ) def end_run(self, run_id: int, outcome: str) -> None: @@ -1233,11 +1269,13 @@ def annotation_keys_of(self, cycle_id: int) -> set[str]: def active_runs(self) -> list[sqlite3.Row]: """Every run not yet ended, in any worktree, with its plan.""" + scope, extra = self._scoped("r.source") return self.all( - "SELECT r.id, r.worktree_path, r.executor_model, r.started_at," + "SELECT r.id, r.worktree_path, r.executor_model, r.started_at, r.source," " p.plan_path, p.declared_cycles" " FROM run r JOIN plan_contract p ON p.id = r.plan_contract_id" - " WHERE r.ended_at IS NULL ORDER BY r.id" + f" WHERE r.ended_at IS NULL{scope} ORDER BY r.id", + extra, ) def last_invocation_at(self, run_id: int) -> str | None: @@ -1253,7 +1291,10 @@ def advance_claims(self) -> list[sqlite3.Row]: # -- projections: the friction log, progress and metrics ------------------- def runs_in(self, worktree: str) -> list[sqlite3.Row]: - return self.all("SELECT * FROM run WHERE worktree_path = ? ORDER BY id", (worktree,)) + scope, extra = self._scoped() + return self.all( + f"SELECT * FROM run WHERE worktree_path = ?{scope} ORDER BY id", (worktree, *extra) + ) def run_events(self, run_id: int) -> list[sqlite3.Row]: return self.all("SELECT * FROM integrity_event WHERE run_id = ? ORDER BY id", (run_id,)) @@ -1324,7 +1365,7 @@ def cycle_times(self, run_id: int) -> list[sqlite3.Row]: ) -def open_readonly(path: Path) -> Ledger | None: +def open_readonly(path: Path, *, source: str | None = LOCAL_SOURCE) -> Ledger | None: """A reader on an existing ledger, or None when there is none yet. Read-only is structural: the file is opened with SQLite's `mode=ro` URI, which also @@ -1336,6 +1377,7 @@ def open_readonly(path: Path) -> Ledger | None: return None reader = Ledger.__new__(Ledger) reader.repo_path = None + reader.source = source reader.path = path reader.db = sqlite3.connect(f"file:{path}?mode=ro", uri=True) reader.db.row_factory = sqlite3.Row From add5f3410700f6efea9492ede079562ed43f63b3 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:53:01 +0100 Subject: [PATCH 11/88] test: claims and baseline cache are per source TDD-Run: 1 TDD-Cycle: 8 TDD-Phase: red --- tests/test_ledger_sources.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_ledger_sources.py b/tests/test_ledger_sources.py index 0893134..2978361 100644 --- a/tests/test_ledger_sources.py +++ b/tests/test_ledger_sources.py @@ -151,3 +151,17 @@ def test_a_source_reads_only_its_own_runs_and_contracts(tmp_path, ledger_home): "latest_run": b.latest_run(worktree), } assert seen_by_b == dict.fromkeys(seen_by_b) + + +def test_two_sources_claim_and_cache_the_same_worktree_independently(tmp_path, ledger_home): + repo = tmp_path / "somerepo" + a = Ledger(repo, source="a") + b = Ledger(repo, source="b") + worktree = "/work/repo" + a.claim(worktree, "h", 1, 1) + a.cache_baseline("app", "tree", "cfg", failing=["t::x"], tests=["t::x"], failed_files={}) + + assert (b.claim(worktree, "h", 2, 1) is not None, b.cached_baseline("app", "tree", "cfg")) == ( + True, + None, + ) From 85c81ae2a2fa721d7a2356c03f45875903f718d0 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:56:36 +0100 Subject: [PATCH 12/88] feat: claims and the baseline cache are keyed by source TDD-Run: 1 TDD-Cycle: 8 TDD-Phase: green --- src/tddcli/ledger.py | 58 +++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index b295530..10dbc92 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -684,10 +684,12 @@ def cache_baseline( tests: list[str], failed_files: dict, ) -> None: + # Per source: a guest that could write a cache row another guest reads could + # hand that guest a forged baseline and hide a regression. self.db.execute( """ - INSERT INTO baseline_cache (project, tree_hash, config_sha, failing, tests, failed_files, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO baseline_cache (source, project, tree_hash, config_sha, failing, tests, failed_files, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(source, project, tree_hash, config_sha) DO UPDATE SET failing = excluded.failing, tests = excluded.tests, @@ -695,6 +697,7 @@ def cache_baseline( created_at = excluded.created_at """, ( + LOCAL_SOURCE if self.source is None else self.source, project, tree_hash, config_sha, @@ -713,15 +716,17 @@ def cached_baseline( config_sha: str, max_age_s: float | None = None, ) -> sqlite3.Row | None: + scope, extra = self._scoped() if max_age_s is not None: cutoff = (datetime.now(timezone.utc) - timedelta(seconds=max_age_s)).isoformat() return self.one( - "SELECT * FROM baseline_cache WHERE project=? AND tree_hash=? AND config_sha=? AND created_at >= ?", - (project, tree_hash, config_sha, cutoff), + "SELECT * FROM baseline_cache WHERE project=? AND tree_hash=? AND config_sha=?" + f" AND created_at >= ?{scope}", + (project, tree_hash, config_sha, cutoff, *extra), ) return self.one( - "SELECT * FROM baseline_cache WHERE project=? AND tree_hash=? AND config_sha=?", - (project, tree_hash, config_sha), + f"SELECT * FROM baseline_cache WHERE project=? AND tree_hash=? AND config_sha=?{scope}", + (project, tree_hash, config_sha, *extra), ) # -- baseline claim ---------------------------------------------------- @@ -729,8 +734,9 @@ def cached_baseline( def claim(self, worktree: str, hostname: str, pid: int, projects_total: int) -> int | None: """Insert the claim row, or return None when the worktree is already claimed. - The insert is the lock: `worktree_path` carries `UNIQUE`, so a second claim - fails at the insert rather than racing a read-then-write check. The conflict + The insert is the lock: `(source, worktree_path)` carries `UNIQUE`, so a second + claim fails at the insert rather than racing a read-then-write check. Two + guests commonly check out at the same path, so the lock is per source. The conflict is answered as None, not as `sqlite3.IntegrityError`, because no sqlite exception can cross a socket to a remote caller.""" try: @@ -743,23 +749,26 @@ def claim(self, worktree: str, hostname: str, pid: int, projects_total: int) -> projects_done=0, current_project=None, started_at=now(), + **self._stamp(), ) except sqlite3.IntegrityError: return None def release_claim(self, worktree: str) -> None: - self.db.execute("DELETE FROM baseline_claim WHERE worktree_path = ?", (worktree,)) - self.db.commit() + scope, extra = self._scoped() + self._write( + f"DELETE FROM baseline_claim WHERE worktree_path = ?{scope}", (worktree, *extra) + ) def update_claim(self, worktree: str, projects_done: int, current_project: str) -> None: """Counters and progress only — no per-project timing history, that lives in the stderr heartbeat lines.""" - self.db.execute( + scope, extra = self._scoped() + self._write( "UPDATE baseline_claim SET projects_done = ?, current_project = ?" - " WHERE worktree_path = ?", - (projects_done, current_project, worktree), + f" WHERE worktree_path = ?{scope}", + (projects_done, current_project, worktree, *extra), ) - self.db.commit() def claim_advance(self, worktree: str, hostname: str, pid: int) -> int | None: """Insert the advance claim row, or return None when the worktree is already @@ -771,13 +780,14 @@ def claim_advance(self, worktree: str, hostname: str, pid: int) -> int | None: hostname=hostname, pid=pid, started_at=now(), + **self._stamp(), ) except sqlite3.IntegrityError: return None def release_advance_claim(self, worktree: str) -> None: - self.db.execute("DELETE FROM advance_claim WHERE worktree_path = ?", (worktree,)) - self.db.commit() + scope, extra = self._scoped() + self._write(f"DELETE FROM advance_claim WHERE worktree_path = ?{scope}", (worktree, *extra)) def active_advance_claim(self, worktree: str) -> dict | None: """Read-only observer — staleness computed but no row deleted.""" @@ -804,11 +814,17 @@ def active_claim(self, worktree: str) -> dict | None: def claim_row(self, worktree: str) -> sqlite3.Row | None: """The raw baseline claim, with no judgement of whether its owner lives.""" - return self.one("SELECT * FROM baseline_claim WHERE worktree_path = ?", (worktree,)) + scope, extra = self._scoped() + return self.one( + f"SELECT * FROM baseline_claim WHERE worktree_path = ?{scope}", (worktree, *extra) + ) def advance_claim_row(self, worktree: str) -> sqlite3.Row | None: """The raw advance claim, with no judgement of whether its owner lives.""" - return self.one("SELECT * FROM advance_claim WHERE worktree_path = ?", (worktree,)) + scope, extra = self._scoped() + return self.one( + f"SELECT * FROM advance_claim WHERE worktree_path = ?{scope}", (worktree, *extra) + ) # -- plan contracts ---------------------------------------------------- @@ -1283,10 +1299,12 @@ def last_invocation_at(self, run_id: int) -> str | None: return row["at"] if row else None def baseline_claims(self) -> list[sqlite3.Row]: - return self.all("SELECT * FROM baseline_claim ORDER BY id") + scope, extra = self._scoped() + return self.all(f"SELECT * FROM baseline_claim WHERE 1 = 1{scope} ORDER BY id", extra) def advance_claims(self) -> list[sqlite3.Row]: - return self.all("SELECT * FROM advance_claim ORDER BY id") + scope, extra = self._scoped() + return self.all(f"SELECT * FROM advance_claim WHERE 1 = 1{scope} ORDER BY id", extra) # -- projections: the friction log, progress and metrics ------------------- From c4ee859938efd8138809a63360740dcdbc85a501 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:56:57 +0100 Subject: [PATCH 13/88] test: metrics names each run's source TDD-Run: 1 TDD-Cycle: 9 TDD-Phase: red --- tests/test_ledger_sources.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_ledger_sources.py b/tests/test_ledger_sources.py index 2978361..ccf556a 100644 --- a/tests/test_ledger_sources.py +++ b/tests/test_ledger_sources.py @@ -6,6 +6,7 @@ from __future__ import annotations +from conftest import run_cli, write_plan from tddcli.ledger import Ledger, now #: The v12 shape of the tables v13 rebuilds, copied from `SCHEMA` before v13. @@ -153,6 +154,24 @@ def test_a_source_reads_only_its_own_runs_and_contracts(tmp_path, ledger_home): assert seen_by_b == dict.fromkeys(seen_by_b) +PLAN = """--- +cycles: + - n: 1 + project: backend + title: "adding two numbers" + test: "tests/test_add.py::test_add" +--- +""" + + +def test_metrics_names_each_runs_source(repo): + plan = write_plan(repo, PLAN) + run_cli(repo, "plan", "register", plan) + run_cli(repo, "run", "start", "--plan", plan) + + assert [r.get("source") for r in run_cli(repo, "metrics")["result"]["runs"]] == ["local"] + + def test_two_sources_claim_and_cache_the_same_worktree_independently(tmp_path, ledger_home): repo = tmp_path / "somerepo" a = Ledger(repo, source="a") From fe96ef80b129b5761d93f2c6f9f844fb5ab15e34 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:00:15 +0100 Subject: [PATCH 14/88] feat: metrics names each run's source TDD-Run: 1 TDD-Cycle: 9 TDD-Phase: green --- src/tddcli/render.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tddcli/render.py b/src/tddcli/render.py index 1fb8878..e384d0d 100644 --- a/src/tddcli/render.py +++ b/src/tddcli/render.py @@ -366,6 +366,7 @@ def metrics(ledger: Ledger, worktree: str) -> dict: { "run": run["id"], "plan_contract": run["plan_contract_id"], + "source": run["source"], "executor": run["executor_model"], "executor_source": run["executor_source"], "outcome": run["outcome"], From 86521cb8252acd452bcb1a0387bf42ffeabe5959 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:00:55 +0100 Subject: [PATCH 15/88] test: a source socket answers ping TDD-Run: 1 TDD-Cycle: 10 TDD-Phase: red --- src/tddcli/service.py | 23 +++++++++++++++++++++++ src/tddcli/wire.py | 9 +++++++++ tests/test_ledger_service.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 src/tddcli/service.py create mode 100644 src/tddcli/wire.py create mode 100644 tests/test_ledger_service.py diff --git a/src/tddcli/service.py b/src/tddcli/service.py new file mode 100644 index 0000000..064287a --- /dev/null +++ b/src/tddcli/service.py @@ -0,0 +1,23 @@ +"""The ledger service: the host's only writer of its ledgers, reached over unix sockets.""" + +from __future__ import annotations + +from pathlib import Path + + +def load_config(path: Path | None = None): + raise NotImplementedError + + +class Service: + def __init__(self, config) -> None: + raise NotImplementedError + + def start(self) -> None: + raise NotImplementedError + + def stop(self) -> None: + raise NotImplementedError + + def add_source(self, name: str, socket_path: Path, executor: str | None = None) -> None: + raise NotImplementedError diff --git a/src/tddcli/wire.py b/src/tddcli/wire.py new file mode 100644 index 0000000..ba774af --- /dev/null +++ b/src/tddcli/wire.py @@ -0,0 +1,9 @@ +"""The ledger service's wire: newline-delimited JSON over a unix socket.""" + +from __future__ import annotations + +from pathlib import Path + + +def call(socket_path: Path, method: str, *args, **kwargs): + raise NotImplementedError diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py new file mode 100644 index 0000000..a1c884a --- /dev/null +++ b/tests/test_ledger_service.py @@ -0,0 +1,33 @@ +"""The ledger service: the host owns the ledger, and each guest reaches it over a socket. + +A guest's runner keeps running suites, gates and git as the agent, but keeps no ledger +of its own. It reads and writes the host's through a unix socket, and the source its +runs are recorded under is the socket they arrived on, never anything it says. +""" + +from __future__ import annotations + +import shutil +import tempfile +from pathlib import Path + +from tddcli import service, wire + + +def test_a_source_socket_answers_ping_with_its_name(tmp_path): + # Under /tmp, not tmp_path: a socket path under tmp_path is longer than AF_UNIX allows. + sockets = Path(tempfile.mkdtemp(dir="/tmp", prefix="tdd-")) + config = tmp_path / "ledger.toml" + config.write_text( + f'[service]\nadmin_socket = "{sockets / "admin.sock"}"\n' + f'home = "{tmp_path / "host-ledgers"}"\n' + ) + svc = service.Service(service.load_config(config)) + try: + svc.start() + svc.add_source("vm-1", sockets / "vm-1.sock", executor=None) + reply = wire.call(sockets / "vm-1.sock", "ping") + finally: + svc.stop() + shutil.rmtree(sockets, ignore_errors=True) + assert reply == {"source": "vm-1", "executor": None} From 285c2a64bc07231545173036280ae7df6cd9218f Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:04:35 +0100 Subject: [PATCH 16/88] feat: ledger service listens on a socket per source TDD-Run: 1 TDD-Cycle: 10 TDD-Phase: green --- src/tddcli/service.py | 114 +++++++++++++++++++++++++++++++++++--- src/tddcli/wire.py | 45 ++++++++++++++- tests/test_actor_seams.py | 5 +- 3 files changed, 152 insertions(+), 12 deletions(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 064287a..5dfd0e5 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -1,23 +1,121 @@ -"""The ledger service: the host's only writer of its ledgers, reached over unix sockets.""" +"""The ledger service: the host's only writer of its ledgers, reached over unix sockets. + +When agents run in disposable guests, the runner has to be in the guest, because that +is where the suites run. Its ledger must not be: it would vanish with the guest, and +guest root could rewrite it. So the runner keeps none, and reads and writes the host's +through a socket the host forwards into the guest. + +The service listens on one socket per source, plus an admin socket for the operator. +A connection's source is the listener it arrived on, never anything the client sends: +the protocol has no source field at all. +""" from __future__ import annotations +import contextlib +import os +import socketserver +import threading +import tomllib +from dataclasses import dataclass from pathlib import Path +from . import runner, wire + +CONFIG_PATH = Path("/etc/tdd-cli/ledger.toml") +CONFIG_ENV = "TDD_LEDGER_SERVICE_CONFIG" + + +class ServiceConfigError(RuntimeError): + pass + + +@dataclass +class ServiceConfig: + admin_socket: Path + home: Path + + +def config_path() -> Path: + override = os.environ.get(CONFIG_ENV) + return Path(override) if override else CONFIG_PATH -def load_config(path: Path | None = None): - raise NotImplementedError + +def load_config(path: Path | None = None) -> ServiceConfig: + """The service's config, held to the same trust rule as the runner's.""" + path = path or config_path() + try: + runner._require_trusted(path) + raw = tomllib.loads(path.read_text()) + except runner.RunnerConfigError as exc: + raise ServiceConfigError(str(exc)) from exc + except tomllib.TOMLDecodeError as exc: + raise ServiceConfigError(f"{path}: {exc}") from exc + section = raw.get("service") or {} + if not section.get("admin_socket"): + raise ServiceConfigError(f"{path}: [service] needs `admin_socket`") + home = section.get("home") + return ServiceConfig( + admin_socket=Path(section["admin_socket"]), + home=Path(home) if home else Path.home() / ".local" / "share" / "tdd-cli", + ) + + +class _Handler(socketserver.StreamRequestHandler): + """One connection: requests answered in order until the client closes it.""" + + def handle(self) -> None: + listener: _Listener = self.server # type: ignore[assignment] + while (request := wire.receive(self.rfile)) is not None: + wire.send(self.wfile, listener.service.answer(listener.source, request)) + + +class _Listener(socketserver.ThreadingUnixStreamServer): + # Both, or `shutdown()` hangs while a client connection is still open. + daemon_threads = True + block_on_close = False + + def __init__(self, path: Path, service: Service, source: str | None): + self.path = path + self.service = service + self.source = source + # A socket file left by a service that died is not a listener: replace it. + with contextlib.suppress(FileNotFoundError): + path.unlink() + super().__init__(str(path), _Handler) + self.thread = threading.Thread(target=self.serve_forever, daemon=True) + self.thread.start() + + def close(self) -> None: + self.shutdown() + self.server_close() + with contextlib.suppress(FileNotFoundError): + self.path.unlink() class Service: - def __init__(self, config) -> None: - raise NotImplementedError + def __init__(self, config: ServiceConfig) -> None: + self.config = config + self.listeners: dict[str | None, _Listener] = {} + self.executors: dict[str, str | None] = {} def start(self) -> None: - raise NotImplementedError + self.config.home.mkdir(parents=True, exist_ok=True) + self.listeners[None] = _Listener(self.config.admin_socket, self, None) def stop(self) -> None: - raise NotImplementedError + for listener in list(self.listeners.values()): + listener.close() + self.listeners.clear() def add_source(self, name: str, socket_path: Path, executor: str | None = None) -> None: - raise NotImplementedError + self.executors[name] = executor + self.listeners[name] = _Listener(Path(socket_path), self, name) + + def answer(self, source: str | None, request: dict) -> dict: + if request.get("method") == "ping": + return { + "ok": True, + "result": {"source": source, "executor": self.executors.get(source)}, + } + return {"ok": False, "refusal": "method_not_allowed", "error": "unknown method"} diff --git a/src/tddcli/wire.py b/src/tddcli/wire.py index ba774af..b355e06 100644 --- a/src/tddcli/wire.py +++ b/src/tddcli/wire.py @@ -1,9 +1,50 @@ -"""The ledger service's wire: newline-delimited JSON over a unix socket.""" +"""The ledger service's wire: newline-delimited JSON over a unix socket. + +One request and one response per line. A response is `{"ok": true, "result": ...}` +or `{"ok": false, "refusal": , "error": }`. +""" from __future__ import annotations +import json +import socket from pathlib import Path +def send(stream, message: dict) -> None: + stream.write(json.dumps(message).encode() + b"\n") + stream.flush() + + +def receive(stream) -> dict | None: + """The next message on the stream, or None once the other end has closed it.""" + line = stream.readline() + return json.loads(line) if line else None + + +class Connection: + """One persistent connection to a ledger service socket.""" + + def __init__(self, socket_path: Path): + self.socket_path = Path(socket_path) + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.connect(str(self.socket_path)) + self.stream = self.sock.makefile("rwb") + + def request(self, method: str, *args, **kwargs): + send(self.stream, {"method": method, "args": list(args), "kwargs": kwargs}) + reply = receive(self.stream) + return reply["result"] + + def close(self) -> None: + self.stream.close() + self.sock.close() + + def call(socket_path: Path, method: str, *args, **kwargs): - raise NotImplementedError + """One request on a connection of its own.""" + conn = Connection(socket_path) + try: + return conn.request(method, *args, **kwargs) + finally: + conn.close() diff --git a/tests/test_actor_seams.py b/tests/test_actor_seams.py index 3c738b3..d07dbfe 100644 --- a/tests/test_actor_seams.py +++ b/tests/test_actor_seams.py @@ -36,8 +36,9 @@ def test_only_the_actor_spawns_processes(): r"|symlink_to\(|\.mkdir\(" ) -#: Leases and the ledger are the tool's own state: they stay the runner's. -OWN_STATE = {"actor.py", "agentfs.py", "leases.py", "ledger.py"} +#: Leases and the ledger are the tool's own state: they stay the runner's. The ledger +#: service's sockets and its sources file are the service's own. +OWN_STATE = {"actor.py", "agentfs.py", "leases.py", "ledger.py", "service.py"} def test_only_the_actor_writes_outside_the_tools_own_state(): From 5b26ad56b1f3a9fc849964392902db1450359bfd Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:04:52 +0100 Subject: [PATCH 17/88] test: every ledger method is classified for guests TDD-Run: 1 TDD-Cycle: 11 TDD-Phase: red --- tests/test_ledger_surface.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/tests/test_ledger_surface.py b/tests/test_ledger_surface.py index 6c5cd1e..cba3c40 100644 --- a/tests/test_ledger_surface.py +++ b/tests/test_ledger_surface.py @@ -8,10 +8,12 @@ from __future__ import annotations import ast +import inspect import re from pathlib import Path import tddcli +from tddcli import ledger as ledger_mod SRC = Path(tddcli.__file__).resolve().parent @@ -39,9 +41,7 @@ def _offences(path: Path) -> list[str]: elif isinstance(node, ast.ImportFrom) and node.module == "sqlite3": found.append(f"{node.lineno}: from sqlite3") elif ( - isinstance(node, ast.Constant) - and isinstance(node.value, str) - and SQL.match(node.value) + isinstance(node, ast.Constant) and isinstance(node.value, str) and SQL.match(node.value) ): found.append(f"{node.lineno}: SQL {node.value.strip()[:40]!r}") elif ( @@ -61,3 +61,29 @@ def test_no_module_but_the_ledger_speaks_sql(): if (found := _offences(path)) } assert offenders == {} + + +#: A method that touches one run's rows names that run by one of these parameters. +OWNERSHIP = {"run_id", "cycle_id", "contract_id"} + + +def test_every_ledger_method_is_classified_and_scoped(): + reads = getattr(ledger_mod, "GUEST_READS", frozenset()) + writes = getattr(ledger_mod, "GUEST_WRITES", frozenset()) + host_only = getattr(ledger_mod, "HOST_ONLY", frozenset()) + rooted = getattr(ledger_mod, "SOURCE_ROOTED", frozenset()) + public = sorted( + name + for name, value in vars(ledger_mod.Ledger).items() + if not name.startswith("_") and callable(value) + ) + problems = [] + for name in public: + classes = [s for s in (reads, writes, host_only) if name in s] + if len(classes) != 1: + problems.append(f"{name}: in {len(classes)} of GUEST_READS/GUEST_WRITES/HOST_ONLY") + elif (name in reads or name in writes) and name not in rooted: + params = set(inspect.signature(getattr(ledger_mod.Ledger, name)).parameters) + if not params & OWNERSHIP: + problems.append(f"{name}: a guest method with no run, cycle or contract id") + assert problems == [] From f79a214ad549c3d67915c772c89e13948184673c Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:08:29 +0100 Subject: [PATCH 18/88] feat: classify the ledger's methods for guests TDD-Run: 1 TDD-Cycle: 11 TDD-Phase: green --- src/tddcli/ledger.py | 147 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 10dbc92..67db429 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -407,6 +407,153 @@ def ledger_path(repo_path: Path) -> Path: #: single-machine split mode alike. A ledger service names one source per guest socket. LOCAL_SOURCE = "local" +# -- the guest registry ---------------------------------------------------------- +# +# A ledger service exposes `Ledger` methods to guests by name, so every public method +# is classified here, exactly once. A guest method that touches one run's rows names +# that run by a parameter called exactly `run_id`, `cycle_id` or `contract_id`, which +# the service checks belongs to the guest's source; one that finds rows by worktree, +# project or cache key is SOURCE_ROOTED, and filters by the ledger's own source. + +#: Reads a guest may make. +GUEST_READS = frozenset( + { + "abandonment", + "active_run", + "active_runs", + "advance_claim_row", + "advance_claims", + "annotation_keys_of", + "annotations", + "baseline_claims", + "baseline_rows", + "baselines", + "blocker_counts", + "blockers", + "cached_baseline", + "claim_row", + "collection", + "commits", + "completed_sensitivity", + "contract", + "contract_by_blob", + "cycle", + "cycle_event_kinds", + "cycle_events", + "cycle_has_event", + "cycle_has_note", + "cycle_notes", + "cycle_times", + "cycles", + "event_counts", + "event_details", + "get_meta", + "interventions", + "invocations", + "last_cycle_event_detail", + "last_gate", + "last_invocation_at", + "latest_blocked_run", + "latest_close_sweeps", + "latest_contract", + "latest_run", + "max_suite_duration_ms", + "open_cycle", + "open_sensitivity", + "previous_baseline", + "regenerated_artifacts", + "run", + "run_events", + "run_has_event", + "run_notes", + "run_target_tests", + "runs_in", + "suite_time_by_phase", + "unclosed_cycle", + } +) + +#: Writes a guest may make. +GUEST_WRITES = frozenset( + { + "abandon_run", + "add_annotation", + "add_blocker", + "add_cycle", + "add_note", + "amend_baseline", + "cache_baseline", + "claim", + "claim_advance", + "close_sensitivity_check", + "end_run", + "event", + "mark_artifact_check", + "mark_cycle_closed", + "move_cycle", + "open_sensitivity_check", + "record_artifact_check", + "record_baseline", + "record_collection", + "record_commit", + "record_gate", + "record_invocation", + "record_sensitivity_mutation", + "register_contract", + "release_advance_claim", + "release_claim", + "reopen_run", + "set_targets", + "skip_cycle", + "start_run", + "update_claim", + } +) + +#: Never answered for a guest. The generic helpers would let it name any row of any +#: source; `active_claim` and `active_advance_claim` judge a pid's liveness, which only +#: the guest that owns the pid can do, so a guest composes them from the raw rows. +HOST_ONLY = frozenset( + { + "active_advance_claim", + "active_claim", + "all", + "close", + "insert", + "one", + "set_meta", + "update", + } +) + +#: Guest methods with no run, cycle or contract id: each filters by the ledger's source. +SOURCE_ROOTED = frozenset( + { + "active_run", + "active_runs", + "advance_claim_row", + "advance_claims", + "baseline_claims", + "cache_baseline", + "cached_baseline", + "claim", + "claim_advance", + "claim_row", + "contract_by_blob", + "get_meta", + "latest_blocked_run", + "latest_contract", + "latest_run", + "max_suite_duration_ms", + "previous_baseline", + "register_contract", + "release_advance_claim", + "release_claim", + "runs_in", + "update_claim", + } +) + def import_legacy(source: Path) -> tuple[Path, dict]: """Bring a single-user ledger under the runner, marked as pre-split history. From 7bd0a9bd693f334c52769c95a19029410ae022e0 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:09:52 +0100 Subject: [PATCH 19/88] test: a guest run lands in the host ledger TDD-Run: 1 TDD-Cycle: 12 TDD-Phase: red --- tests/conftest.py | 55 ++++++++++++++++++++++++++++++++++++ tests/test_ledger_service.py | 35 ++++++++++++++++++++++- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 45d0099..4fcf285 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,8 +4,10 @@ import os import pwd import shlex +import shutil import subprocess import sys +import tempfile from pathlib import Path from types import SimpleNamespace @@ -123,6 +125,59 @@ def split_client(tmp_path, monkeypatch, sudo_shim): ) +@pytest.fixture +def ledger_service(tmp_path, monkeypatch): + """A ledger service on the host, running in a thread of this process. + + Its sockets live under /tmp, never under `tmp_path`: a socket path there is longer + than AF_UNIX allows (108 bytes on Linux, 104 on macOS). + """ + from tddcli import service + + sockets = Path(tempfile.mkdtemp(dir="/tmp", prefix="tdd-")) + home = tmp_path / "host-ledgers" + cfg = tmp_path / "ledger.toml" + cfg.write_text(f'[service]\nadmin_socket = "{sockets / "admin.sock"}"\nhome = "{home}"\n') + monkeypatch.setenv("TDD_LEDGER_SERVICE_CONFIG", str(cfg)) + svc = service.Service(service.load_config(cfg)) + svc.start() + try: + yield SimpleNamespace(service=svc, home=home, dir=sockets, config=cfg) + finally: + svc.stop() + shutil.rmtree(sockets, ignore_errors=True) + + +def guest_config(split_runner, socket_path: Path, extra: str = "") -> None: + """Point the guest's runner at a ledger service socket, with no ledger home.""" + split_runner.config.write_text( + f'[runner]\nuser = "{current_user()}"\ncommand = "tdd"\n' + f'sudo = "{split_runner.shim.path}"\nledger_socket = "{socket_path}"\n' + extra + ) + + +@pytest.fixture +def split_guest(tmp_path, monkeypatch, split_runner, ledger_service): + """This process is a runner inside a guest, keeping its ledger on the host's service. + + The guest is source `vm-1`. To switch it to another source, rewrite its config's + `ledger_socket` (`guest_config`): `runner.load()` reads the file on every call. + """ + socket_path = ledger_service.dir / "vm-1.sock" + ledger_service.service.add_source("vm-1", socket_path, executor=None) + guest_config(split_runner, socket_path) + # Whatever the runner falls back to must never be the developer's real home. + guest_home = tmp_path / "guest-home" + guest_home.mkdir() + monkeypatch.setenv("HOME", str(guest_home)) + return SimpleNamespace( + socket=socket_path, + config=split_runner.config, + guest_home=guest_home, + runner=split_runner, + ) + + @pytest.fixture def ledger_home(tmp_path, monkeypatch): home = tmp_path / "ledger-home" diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index a1c884a..f28ac9e 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -11,7 +11,33 @@ import tempfile from pathlib import Path -from tddcli import service, wire +from conftest import run_cli, write_plan +from tddcli import gitutil, service, wire +from tddcli import ledger as ledger_mod + +MINIMAL_PLAN = """\ +--- +cycles: + - n: 1 + project: backend + title: "placeholder" + test: "tests/test_smoke.py::test_smoke" + commit_red: "test: placeholder" + commit_green: "feat: placeholder" +--- +# Minimal plan for ledger service tests +""" + + +def start_run(repo) -> dict: + plan = write_plan(repo, MINIMAL_PLAN) + run_cli(repo, "plan", "register", plan) + return run_cli(repo, "run", "start", "--plan", plan) + + +def host_file(ledger_service, repo) -> Path: + """Where the host keeps this repository's ledger.""" + return ledger_service.home / f"{ledger_mod.slug(gitutil.repo_identity(repo))}.sqlite3" def test_a_source_socket_answers_ping_with_its_name(tmp_path): @@ -31,3 +57,10 @@ def test_a_source_socket_answers_ping_with_its_name(tmp_path): svc.stop() shutil.rmtree(sockets, ignore_errors=True) assert reply == {"source": "vm-1", "executor": None} + + +def test_a_guest_run_lands_in_the_host_ledger_under_its_source(repo, split_guest, ledger_service): + start_run(repo) + + reader = ledger_mod.open_readonly(host_file(ledger_service, repo), source=None) + assert reader is not None and [r["source"] for r in reader.active_runs()] == ["vm-1"] From 2dec7d5cbbcf540d9f036047e215aabe8a48a6df Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:13:49 +0100 Subject: [PATCH 20/88] feat: a runner with ledger_socket keeps its ledger on the host TDD-Run: 1 TDD-Cycle: 12 TDD-Phase: green --- src/tddcli/cli.py | 11 +++++-- src/tddcli/ledger.py | 21 ++++++++++--- src/tddcli/remote.py | 35 ++++++++++++++++++++++ src/tddcli/runner.py | 4 +++ src/tddcli/service.py | 59 +++++++++++++++++++++++++++++------- src/tddcli/wire.py | 69 +++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 179 insertions(+), 20 deletions(-) create mode 100644 src/tddcli/remote.py diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 30cc9de..dcaa476 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1545,9 +1545,14 @@ def cmd_fleet(args) -> Envelope: `_context`: no tdd.toml, active run, or existing ledger is required, and the ledger is opened read-only so live agents cannot be perturbed.""" worktree = _worktree() - summary = fleet.summarise( - ledger_mod.open_readonly(ledger_path(gitutil.repo_identity(worktree))) - ) + repo = gitutil.repo_identity(worktree) + split = runner_mod.load() + if split is not None and split.role == "runner" and split.ledger_socket is not None: + # The ledger is on the host: read this source's view of it through the socket. + reader = open_ledger(repo) + else: + reader = ledger_mod.open_readonly(ledger_path(repo)) + summary = fleet.summarise(reader) if args.json: return Envelope(result=summary, next_action=NextAction(Verb.COMPLETE, "Fleet reported.")) sys.stdout.write(fleet.render(summary)) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 67db429..7066d12 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -394,9 +394,13 @@ def _ensured_home() -> Path: return root +def slug(repo_path: Path | str) -> str: + """A repository's ledger file name, without its suffix.""" + return str(repo_path).replace(os.sep, "-").strip("-") + + def ledger_path(repo_path: Path) -> Path: - slug = str(repo_path).replace(os.sep, "-").strip("-") - return _ensured_home() / f"{slug}.sqlite3" + return _ensured_home() / f"{slug(repo_path)}.sqlite3" #: `meta` key written by `import_legacy`: everything up to `last_run_id` was recorded @@ -1549,6 +1553,15 @@ def open_readonly(path: Path, *, source: str | None = LOCAL_SOURCE) -> Ledger | return reader -def open_ledger(repo_path: Path) -> Ledger: - """The ledger of a repository. Every command opens its ledger through here.""" +def open_ledger(repo_path: Path): + """The ledger of a repository. Every command opens its ledger through here. + + A runner configured with a ledger service's socket keeps no ledger of its own: it + gets a `RemoteLedger` that reads and writes the host's. + """ + split = _as_runner() + if split is not None and split.ledger_socket is not None: + from .remote import RemoteLedger + + return RemoteLedger(split.ledger_socket, repo_path) return Ledger(repo_path) diff --git a/src/tddcli/remote.py b/src/tddcli/remote.py new file mode 100644 index 0000000..824148f --- /dev/null +++ b/src/tddcli/remote.py @@ -0,0 +1,35 @@ +"""The guest's side of the ledger service: a ledger it can use but never holds. + +A runner configured with `ledger_socket` keeps no ledger. `RemoteLedger` stands in for +`Ledger` and sends each call through the socket, by method name, to the service that +owns the host's SQLite files. The guest never sees a file, a connection or a line of +SQL: only the named methods the service is willing to answer. +""" + +from __future__ import annotations + +from pathlib import Path + +from . import wire + + +class RemoteLedger: + def __init__(self, socket_path: Path, repo_path: Path): + self.repo_path = repo_path + self.path = Path(socket_path) + # One connection for the whole command: an `advance` makes hundreds of calls, + # and over a forwarded socket each new connection is a new channel. + self._conn = wire.Connection(self.path) + self._conn.request("open", str(repo_path)) + + def __getattr__(self, name: str): + if name.startswith("_"): + raise AttributeError(name) + + def call(*args, **kwargs): + return self._conn.request(name, *args, **kwargs) + + return call + + def close(self) -> None: + self._conn.close() diff --git a/src/tddcli/runner.py b/src/tddcli/runner.py index d2fb557..a3ffeda 100644 --- a/src/tddcli/runner.py +++ b/src/tddcli/runner.py @@ -34,6 +34,8 @@ class RunnerConfig: role: str # runner | client sudo: str = DEFAULT_SUDO ledger_home: Path | None = None + #: A ledger service's socket: the ledger is kept on the host, never here. + ledger_socket: Path | None = None executors: dict[str, str] = field(default_factory=dict) @@ -100,11 +102,13 @@ def load() -> RunnerConfig | None: f"{path}: [runner] user {section['user']!r} does not exist" ) from exc ledger_home = section.get("ledger_home") + ledger_socket = section.get("ledger_socket") return RunnerConfig( user=section["user"], command=section["command"], role="runner" if runner_uid == os.geteuid() else "client", sudo=section.get("sudo", DEFAULT_SUDO), ledger_home=Path(ledger_home) if ledger_home else None, + ledger_socket=Path(ledger_socket) if ledger_socket else None, executors=dict(raw.get("executor") or {}), ) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 5dfd0e5..35df96d 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -20,6 +20,7 @@ from dataclasses import dataclass from pathlib import Path +from . import ledger as ledger_mod from . import runner, wire CONFIG_PATH = Path("/etc/tdd-cli/ledger.toml") @@ -61,13 +62,59 @@ def load_config(path: Path | None = None) -> ServiceConfig: ) +class _Session: + """One client connection, and the ledger it opened. + + Each connection opens its own `Ledger`, in its own handler thread: an SQLite + connection must not cross threads. + """ + + def __init__(self, service: Service, source: str | None): + self.service = service + self.source = source + self.ledger: ledger_mod.Ledger | None = None + + def answer(self, request: dict) -> dict: + method = request.get("method") + args = request.get("args") or [] + kwargs = request.get("kwargs") or {} + if method == "ping": + return _ok({"source": self.source, "executor": self.service.executors.get(self.source)}) + if method == "open": + repo = str(args[0]) + self.ledger = ledger_mod.Ledger( + Path(repo), + source=self.source, + path=self.service.config.home / f"{ledger_mod.slug(repo)}.sqlite3", + ) + return _ok(None) + if self.ledger is None: + return {"ok": False, "error": "no ledger is open on this connection: send `open`"} + try: + return _ok(getattr(self.ledger, method)(*args, **kwargs)) + except Exception as exc: # the client gets the reason, the service keeps serving + return {"ok": False, "error": f"{method}: {exc}"} + + def close(self) -> None: + if self.ledger is not None: + self.ledger.close() + + +def _ok(result) -> dict: + return {"ok": True, "result": wire.encode(result)} + + class _Handler(socketserver.StreamRequestHandler): """One connection: requests answered in order until the client closes it.""" def handle(self) -> None: listener: _Listener = self.server # type: ignore[assignment] - while (request := wire.receive(self.rfile)) is not None: - wire.send(self.wfile, listener.service.answer(listener.source, request)) + session = _Session(listener.service, listener.source) + try: + while (request := wire.receive(self.rfile)) is not None: + wire.send(self.wfile, session.answer(request)) + finally: + session.close() class _Listener(socketserver.ThreadingUnixStreamServer): @@ -111,11 +158,3 @@ def stop(self) -> None: def add_source(self, name: str, socket_path: Path, executor: str | None = None) -> None: self.executors[name] = executor self.listeners[name] = _Listener(Path(socket_path), self, name) - - def answer(self, source: str | None, request: dict) -> dict: - if request.get("method") == "ping": - return { - "ok": True, - "result": {"source": source, "executor": self.executors.get(source)}, - } - return {"ok": False, "refusal": "method_not_allowed", "error": "unknown method"} diff --git a/src/tddcli/wire.py b/src/tddcli/wire.py index b355e06..f1cf81a 100644 --- a/src/tddcli/wire.py +++ b/src/tddcli/wire.py @@ -1,7 +1,11 @@ """The ledger service's wire: newline-delimited JSON over a unix socket. -One request and one response per line. A response is `{"ok": true, "result": ...}` -or `{"ok": false, "refusal": , "error": }`. +One request and one response per line. A request is `{"method", "args", "kwargs"}`; +a response is `{"ok": true, "result": ...}` or `{"ok": false, "error": }`. + +Results keep their shape across the socket: a ledger row is sent as +`{"$row": {"cols": [...], "vals": [...]}}` and comes back as a `Row`, which reads the +way an `sqlite3.Row` does; a set is sent as `{"$set": [...]}` and comes back a set. """ from __future__ import annotations @@ -11,6 +15,63 @@ from pathlib import Path +class Row: + """A ledger row received over the wire: `row["col"]`, `row[i]`, `keys()`, `dict(row)`.""" + + def __init__(self, cols: list[str], vals: list): + self._cols = list(cols) + self._vals = list(vals) + + def keys(self) -> list[str]: + return list(self._cols) + + def __getitem__(self, key): + if isinstance(key, (int, slice)): + return self._vals[key] + try: + return self._vals[self._cols.index(key)] + except ValueError: + raise IndexError(f"No item with that key: {key!r}") from None + + def __iter__(self): + return iter(self._vals) + + def __len__(self) -> int: + return len(self._vals) + + def __eq__(self, other) -> bool: + return isinstance(other, Row) and (self._cols, self._vals) == (other._cols, other._vals) + + def __repr__(self) -> str: + return f"Row({dict(zip(self._cols, self._vals, strict=True))!r})" + + +def encode(value): + """A ledger method's result, as JSON-able data.""" + if isinstance(value, dict): + return {k: encode(v) for k, v in value.items()} + if isinstance(value, (set, frozenset)): + return {"$set": [encode(v) for v in sorted(value, key=repr)]} + if isinstance(value, (list, tuple)): + return [encode(v) for v in value] + if hasattr(value, "keys") and hasattr(value, "__getitem__"): # a row + cols = list(value.keys()) + return {"$row": {"cols": cols, "vals": [value[c] for c in cols]}} + return value + + +def decode(value): + if isinstance(value, list): + return [decode(v) for v in value] + if isinstance(value, dict): + if set(value) == {"$row"}: + return Row(value["$row"]["cols"], value["$row"]["vals"]) + if set(value) == {"$set"}: + return {decode(v) for v in value["$set"]} + return {k: decode(v) for k, v in value.items()} + return value + + def send(stream, message: dict) -> None: stream.write(json.dumps(message).encode() + b"\n") stream.flush() @@ -34,7 +95,9 @@ def __init__(self, socket_path: Path): def request(self, method: str, *args, **kwargs): send(self.stream, {"method": method, "args": list(args), "kwargs": kwargs}) reply = receive(self.stream) - return reply["result"] + if not reply.get("ok"): + raise RuntimeError(reply.get("error") or f"the ledger service refused {method}") + return decode(reply.get("result")) def close(self) -> None: self.stream.close() From c8c6d5ec10c1bddd099d8efb47372be386ea9af5 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:14:04 +0100 Subject: [PATCH 21/88] test: pin a guest renders its own friction log through the socket TDD-Run: 1 TDD-Cycle: 13 TDD-Phase: pin --- tests/test_ledger_service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index f28ac9e..3f6da30 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -11,7 +11,7 @@ import tempfile from pathlib import Path -from conftest import run_cli, write_plan +from conftest import run_cli, run_cli_text, write_plan from tddcli import gitutil, service, wire from tddcli import ledger as ledger_mod @@ -64,3 +64,9 @@ def test_a_guest_run_lands_in_the_host_ledger_under_its_source(repo, split_guest reader = ledger_mod.open_readonly(host_file(ledger_service, repo), source=None) assert reader is not None and [r["source"] for r in reader.active_runs()] == ["vm-1"] + + +def test_a_guest_renders_its_own_friction_log(repo, split_guest): + start_run(repo) + + assert "tasks/plan.md" in run_cli_text(repo, "log", "render") From 62c9084e9fbf26834d81ba52c787b53d11f95e00 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:16:11 +0100 Subject: [PATCH 22/88] test: pin a blocked guest run can still be unblocked TDD-Run: 1 TDD-Cycle: 14 TDD-Phase: pin --- tests/test_ledger_service.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index 3f6da30..1ba4dc3 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -70,3 +70,11 @@ def test_a_guest_renders_its_own_friction_log(repo, split_guest): start_run(repo) assert "tasks/plan.md" in run_cli_text(repo, "log", "render") + + +def test_a_blocked_guest_run_can_be_unblocked(repo, split_guest): + start_run(repo) + run_cli(repo, "blocker", "--kind", "plan_defect", "--detail", "x") + + out = run_cli(repo, "resume", "--unblock", "--note", "fixed") + assert out["result"].get("resumed") is True From 00f7b25316769b6f4188c1d0b9aa53e449355ed7 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:18:11 +0100 Subject: [PATCH 23/88] test: an unreachable ledger socket refuses the verb TDD-Run: 1 TDD-Cycle: 15 TDD-Phase: red --- tests/test_ledger_service.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index 1ba4dc3..854a791 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -11,7 +11,7 @@ import tempfile from pathlib import Path -from conftest import run_cli, run_cli_text, write_plan +from conftest import guest_config, run_cli, run_cli_text, write_plan from tddcli import gitutil, service, wire from tddcli import ledger as ledger_mod @@ -78,3 +78,14 @@ def test_a_blocked_guest_run_can_be_unblocked(repo, split_guest): out = run_cli(repo, "resume", "--unblock", "--note", "fixed") assert out["result"].get("resumed") is True + + +def test_an_unreachable_socket_refuses_the_verb(repo, split_guest, ledger_service): + guest_config(split_guest.runner, ledger_service.dir / "absent.sock") + + out = start_run(repo) + local = sorted(str(p) for p in split_guest.guest_home.rglob("*.sqlite3")) + assert {"reason": out["result"].get("reason"), "local_ledgers": local} == { + "reason": "ledger_unreachable", + "local_ledgers": [], + } From 7beb34b4659a09e42736008c1a0cc8d6af9b269d Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:21:43 +0100 Subject: [PATCH 24/88] feat: fail closed when the ledger service is unreachable TDD-Run: 1 TDD-Cycle: 15 TDD-Phase: green --- src/tddcli/cli.py | 5 +++++ src/tddcli/wire.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index dcaa476..f9106a9 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -27,6 +27,7 @@ identity, render, snapshot, + wire, ) from . import ( config as config_mod, @@ -1885,6 +1886,10 @@ def main(argv: list[str] | None = None) -> int: " worktree readable by that user (`chmod -R go+rX` it, or share a group)", reason="worktree_unreadable", ) + except wire.LedgerUnreachable as exc: + # Fail closed: a guest whose ledger service is gone refuses the verb, and never + # falls back to a ledger of its own. + envelope = failure(str(exc), reason="ledger_unreachable") except ( config_mod.ConfigError, gitutil.GitError, diff --git a/src/tddcli/wire.py b/src/tddcli/wire.py index f1cf81a..52250f9 100644 --- a/src/tddcli/wire.py +++ b/src/tddcli/wire.py @@ -15,6 +15,14 @@ from pathlib import Path +class LedgerUnreachable(RuntimeError): + """The ledger service could not be reached, or dropped the connection. + + Nothing on the guest side ever falls back to a local ledger when this is raised: + the verb is refused instead. + """ + + class Row: """A ledger row received over the wire: `row["col"]`, `row[i]`, `keys()`, `dict(row)`.""" @@ -89,12 +97,27 @@ class Connection: def __init__(self, socket_path: Path): self.socket_path = Path(socket_path) self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.sock.connect(str(self.socket_path)) + try: + self.sock.connect(str(self.socket_path)) + except OSError as exc: + self.sock.close() + raise LedgerUnreachable( + f"the ledger service at {self.socket_path} is unreachable: {exc.strerror or exc}" + ) from exc self.stream = self.sock.makefile("rwb") def request(self, method: str, *args, **kwargs): - send(self.stream, {"method": method, "args": list(args), "kwargs": kwargs}) - reply = receive(self.stream) + try: + send(self.stream, {"method": method, "args": list(args), "kwargs": kwargs}) + reply = receive(self.stream) + except OSError as exc: + raise LedgerUnreachable( + f"the ledger service at {self.socket_path} dropped the connection: {exc}" + ) from exc + if reply is None: + raise LedgerUnreachable( + f"the ledger service at {self.socket_path} closed the connection" + ) if not reply.get("ok"): raise RuntimeError(reply.get("error") or f"the ledger service refused {method}") return decode(reply.get("result")) From 1206cb0dbbc6fa4d3258eb88876393b2ccf00bb4 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:22:03 +0100 Subject: [PATCH 25/88] test: a guest cannot reach another source's run TDD-Run: 1 TDD-Cycle: 16 TDD-Phase: red --- tests/test_ledger_service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index 854a791..7b7c47c 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -89,3 +89,15 @@ def test_an_unreachable_socket_refuses_the_verb(repo, split_guest, ledger_servic "reason": "ledger_unreachable", "local_ledgers": [], } + + +def test_a_guest_naming_another_sources_run_is_refused(repo, split_guest, ledger_service): + ledger_service.service.add_source("vm-2", ledger_service.dir / "vm-2.sock", executor=None) + run_id = start_run(repo)["run"]["id"] + guest_config(split_guest.runner, ledger_service.dir / "vm-2.sock") + + out = run_cli(repo, "run", "abandon", "--run", str(run_id), "--reason", "x") + assert {k: out["result"].get(k) for k in ("reason", "refusal")} == { + "reason": "ledger_refused", + "refusal": "foreign_run", + } From afc2f5f93a017f615e75289e1b5140513e998141 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:25:37 +0100 Subject: [PATCH 26/88] feat: the service refuses run ids of other sources TDD-Run: 1 TDD-Cycle: 16 TDD-Phase: green --- src/tddcli/cli.py | 2 ++ src/tddcli/ledger.py | 6 ++++++ src/tddcli/service.py | 26 +++++++++++++++++++++++++- src/tddcli/wire.py | 11 +++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index f9106a9..efc6380 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1886,6 +1886,8 @@ def main(argv: list[str] | None = None) -> int: " worktree readable by that user (`chmod -R go+rX` it, or share a group)", reason="worktree_unreadable", ) + except wire.LedgerRefused as exc: + envelope = failure(exc.error, reason="ledger_refused", refusal=exc.refusal) except wire.LedgerUnreachable as exc: # Fail closed: a guest whose ledger service is gone refuses the verb, and never # falls back to a ledger of its own. diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 7066d12..349c2c3 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -525,6 +525,7 @@ def ledger_path(repo_path: Path) -> Path: "close", "insert", "one", + "run_source", "set_meta", "update", } @@ -1019,6 +1020,11 @@ def register_contract( # -- runs ---------------------------------------------------------------- + def run_source(self, run_id: int) -> str | None: + """The source a run belongs to, whatever this ledger's own; None if no such run.""" + row = self.one("SELECT source FROM run WHERE id = ?", (run_id,)) + return row["source"] if row else None + def run(self, run_id: int) -> sqlite3.Row | None: return self.one("SELECT * FROM run WHERE id = ?", (run_id,)) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 35df96d..c36b69f 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -13,6 +13,7 @@ from __future__ import annotations import contextlib +import inspect import os import socketserver import threading @@ -90,11 +91,30 @@ def answer(self, request: dict) -> dict: return _ok(None) if self.ledger is None: return {"ok": False, "error": "no ledger is open on this connection: send `open`"} + fn = getattr(self.ledger, method) try: - return _ok(getattr(self.ledger, method)(*args, **kwargs)) + bound = inspect.signature(fn).bind(*args, **kwargs) + except TypeError as exc: + return {"ok": False, "error": f"{method}: {exc}"} + refused = self._foreign(bound.arguments) + if refused is not None: + return refused + try: + return _ok(fn(*args, **kwargs)) except Exception as exc: # the client gets the reason, the service keeps serving return {"ok": False, "error": f"{method}: {exc}"} + def _foreign(self, arguments: dict) -> dict | None: + """A refusal when the call names a run of another source. + + A run that does not exist is refused the same way, so that a guest cannot learn + which ids exist in other sources by probing for them. + """ + run_id = arguments.get("run_id") + if run_id is not None and self.ledger.run_source(run_id) != self.source: + return _refusal("foreign_run", f"run {run_id} is not one of this source's runs") + return None + def close(self) -> None: if self.ledger is not None: self.ledger.close() @@ -104,6 +124,10 @@ def _ok(result) -> dict: return {"ok": True, "result": wire.encode(result)} +def _refusal(code: str, error: str) -> dict: + return {"ok": False, "refusal": code, "error": error} + + class _Handler(socketserver.StreamRequestHandler): """One connection: requests answered in order until the client closes it.""" diff --git a/src/tddcli/wire.py b/src/tddcli/wire.py index 52250f9..f1168e9 100644 --- a/src/tddcli/wire.py +++ b/src/tddcli/wire.py @@ -23,6 +23,15 @@ class LedgerUnreachable(RuntimeError): """ +class LedgerRefused(RuntimeError): + """The ledger service answered, and said no. `refusal` is its code for why.""" + + def __init__(self, refusal: str, error: str): + super().__init__(error) + self.refusal = refusal + self.error = error + + class Row: """A ledger row received over the wire: `row["col"]`, `row[i]`, `keys()`, `dict(row)`.""" @@ -118,6 +127,8 @@ def request(self, method: str, *args, **kwargs): raise LedgerUnreachable( f"the ledger service at {self.socket_path} closed the connection" ) + if not reply.get("ok") and reply.get("refusal"): + raise LedgerRefused(reply["refusal"], reply.get("error") or reply["refusal"]) if not reply.get("ok"): raise RuntimeError(reply.get("error") or f"the ledger service refused {method}") return decode(reply.get("result")) From 365762041f684dccbe3da81f272c4961e7ea67c9 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:25:56 +0100 Subject: [PATCH 27/88] test: a guest cannot reach another source's cycle TDD-Run: 1 TDD-Cycle: 17 TDD-Phase: red --- tests/test_ledger_service.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index 7b7c47c..8c668a2 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -11,6 +11,8 @@ import tempfile from pathlib import Path +import pytest + from conftest import guest_config, run_cli, run_cli_text, write_plan from tddcli import gitutil, service, wire from tddcli import ledger as ledger_mod @@ -101,3 +103,19 @@ def test_a_guest_naming_another_sources_run_is_refused(repo, split_guest, ledger "reason": "ledger_refused", "refusal": "foreign_run", } + + +def test_a_guest_naming_another_sources_cycle_is_refused(repo, split_guest, ledger_service): + ledger_service.service.add_source("vm-2", ledger_service.dir / "vm-2.sock", executor=None) + start_run(repo) + host = ledger_mod.open_readonly(host_file(ledger_service, repo), source=None) + cycle_id = host.open_cycle(host.active_runs()[0]["id"])["id"] + + conn = wire.Connection(ledger_service.dir / "vm-2.sock") + try: + conn.request("open", str(gitutil.repo_identity(repo))) + with pytest.raises(wire.LedgerRefused) as refused: + conn.request("cycle", cycle_id) + finally: + conn.close() + assert refused.value.refusal == "foreign_run" From b2b3227f6ed5b73b6a4bec82d01940b5efe5b054 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:31:08 +0100 Subject: [PATCH 28/88] feat: ownership checks cover cycle and contract ids TDD-Run: 1 TDD-Cycle: 17 TDD-Phase: green --- src/tddcli/service.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index c36b69f..106891d 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -105,16 +105,30 @@ def answer(self, request: dict) -> dict: return {"ok": False, "error": f"{method}: {exc}"} def _foreign(self, arguments: dict) -> dict | None: - """A refusal when the call names a run of another source. + """A refusal when the call names a run, cycle or contract of another source. - A run that does not exist is refused the same way, so that a guest cannot learn + One that does not exist is refused the same way, so that a guest cannot learn which ids exist in other sources by probing for them. """ - run_id = arguments.get("run_id") - if run_id is not None and self.ledger.run_source(run_id) != self.source: - return _refusal("foreign_run", f"run {run_id} is not one of this source's runs") + owners = ( + ("run_id", "run", self.ledger.run_source), + ("cycle_id", "cycle", self._cycle_source), + ("contract_id", "plan contract", self._contract_source), + ) + for param, noun, source_of in owners: + value = arguments.get(param) + if value is not None and source_of(value) != self.source: + return _refusal("foreign_run", f"{noun} {value} is not this source's") return None + def _cycle_source(self, cycle_id: int) -> str | None: + cycle = self.ledger.cycle(cycle_id) + return None if cycle is None else self.ledger.run_source(cycle["run_id"]) + + def _contract_source(self, contract_id: int) -> str | None: + contract = self.ledger.contract(contract_id) + return None if contract is None else contract["source"] + def close(self) -> None: if self.ledger is not None: self.ledger.close() From d84a023c056a3361ec600d22ce6c9b7cab831ef4 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:31:39 +0100 Subject: [PATCH 29/88] test: guests cannot call host-only ledger methods TDD-Run: 1 TDD-Cycle: 18 TDD-Phase: red --- tests/test_ledger_service.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index 8c668a2..b1d51e2 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -119,3 +119,23 @@ def test_a_guest_naming_another_sources_cycle_is_refused(repo, split_guest, ledg finally: conn.close() assert refused.value.refusal == "foreign_run" + + +def refusal_of(socket_path, repo_path: str, method: str, *args, **kwargs) -> str: + """What the service says to `method` on a fresh connection: its refusal, or accepted.""" + conn = wire.Connection(socket_path) + try: + conn.request("open", repo_path) + conn.request(method, *args, **kwargs) + except wire.LedgerRefused as exc: + return exc.refusal + finally: + conn.close() + return "accepted" + + +def test_a_guest_cannot_call_a_host_only_method(repo, split_guest): + refusal = refusal_of( + split_guest.socket, str(gitutil.repo_identity(repo)), "insert", "meta", key="k", value="v" + ) + assert refusal == "method_not_allowed" From 4882775f6db742c36b585d014898a811e6df3ba5 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:34:55 +0100 Subject: [PATCH 30/88] feat: the service dispatches only guest-classified methods TDD-Run: 1 TDD-Cycle: 18 TDD-Phase: green --- src/tddcli/ledger.py | 9 +++++---- src/tddcli/service.py | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 349c2c3..75ad797 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -423,6 +423,8 @@ def ledger_path(repo_path: Path) -> Path: GUEST_READS = frozenset( { "abandonment", + "active_advance_claim", + "active_claim", "active_run", "active_runs", "advance_claim_row", @@ -515,12 +517,9 @@ def ledger_path(repo_path: Path) -> Path: ) #: Never answered for a guest. The generic helpers would let it name any row of any -#: source; `active_claim` and `active_advance_claim` judge a pid's liveness, which only -#: the guest that owns the pid can do, so a guest composes them from the raw rows. +#: source. HOST_ONLY = frozenset( { - "active_advance_claim", - "active_claim", "all", "close", "insert", @@ -534,6 +533,8 @@ def ledger_path(repo_path: Path) -> Path: #: Guest methods with no run, cycle or contract id: each filters by the ledger's source. SOURCE_ROOTED = frozenset( { + "active_advance_claim", + "active_claim", "active_run", "active_runs", "advance_claim_row", diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 106891d..f434913 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -91,6 +91,8 @@ def answer(self, request: dict) -> dict: return _ok(None) if self.ledger is None: return {"ok": False, "error": "no ledger is open on this connection: send `open`"} + if method not in ledger_mod.GUEST_READS | ledger_mod.GUEST_WRITES: + return _refusal("method_not_allowed", f"{method} is not answered for a guest") fn = getattr(self.ledger, method) try: bound = inspect.signature(fn).bind(*args, **kwargs) From b8c97aa0bc7dff021cab4a87d7507f7f4b4febfd Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:35:19 +0100 Subject: [PATCH 31/88] test: malformed repo paths are refused TDD-Run: 1 TDD-Cycle: 19 TDD-Phase: red --- tests/test_ledger_service.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index b1d51e2..fd76adf 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -139,3 +139,26 @@ def test_a_guest_cannot_call_a_host_only_method(repo, split_guest): split_guest.socket, str(gitutil.repo_identity(repo)), "insert", "meta", key="k", value="v" ) assert refusal == "method_not_allowed" + + +def open_refusal(socket_path, repo_path: str) -> str: + """What the service says to `open`: its refusal, or accepted.""" + try: + conn = wire.Connection(socket_path) + except wire.LedgerUnreachable: + return "unreachable" + try: + conn.request("open", repo_path) + except wire.LedgerRefused as exc: + return exc.refusal + except wire.LedgerUnreachable: + return "unreachable" + finally: + conn.close() + return "accepted" + + +def test_a_malformed_repo_path_is_refused(split_guest): + malformed = ["relative/repo", "", "/a\x00b", "/a\nb"] + + assert [open_refusal(split_guest.socket, p) for p in malformed] == ["bad_repo"] * 4 From 200784916a0bcdde174e71787f3e5b620a868fab Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:38:37 +0100 Subject: [PATCH 32/88] feat: the service validates the guest's repo path TDD-Run: 1 TDD-Cycle: 19 TDD-Phase: green --- src/tddcli/service.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index f434913..ab8c5b1 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -82,7 +82,9 @@ def answer(self, request: dict) -> dict: if method == "ping": return _ok({"source": self.source, "executor": self.service.executors.get(self.source)}) if method == "open": - repo = str(args[0]) + repo = args[0] if args else None + if not _well_formed_repo(repo): + return _refusal("bad_repo", f"{repo!r} is not an absolute repository path") self.ledger = ledger_mod.Ledger( Path(repo), source=self.source, @@ -136,6 +138,16 @@ def close(self) -> None: self.ledger.close() +def _well_formed_repo(repo) -> bool: + """An absolute path with no NUL or newline: it names the host's ledger file.""" + return ( + isinstance(repo, str) + and Path(repo).is_absolute() + and "\x00" not in repo + and "\n" not in repo + ) + + def _ok(result) -> dict: return {"ok": True, "result": wire.encode(result)} From 26c090b94f205bcb2cba30213a62ea4a6dfd7021 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:38:54 +0100 Subject: [PATCH 33/88] test: a completed run is closed to guest writes TDD-Run: 1 TDD-Cycle: 20 TDD-Phase: red --- tests/test_ledger_service.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index fd76adf..28b0310 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -162,3 +162,21 @@ def test_a_malformed_repo_path_is_refused(split_guest): malformed = ["relative/repo", "", "/a\x00b", "/a\nb"] assert [open_refusal(split_guest.socket, p) for p in malformed] == ["bad_repo"] * 4 + + +def complete_on_host(ledger_service, repo) -> int: + """Start a guest run, then mark it complete through the host's unscoped view.""" + run_id = start_run(repo)["run"]["id"] + host = ledger_mod.Ledger(repo, source=None, path=host_file(ledger_service, repo)) + host.update("run", run_id, ended_at=ledger_mod.now(), outcome="complete") + host.close() + return run_id + + +def test_a_guest_cannot_write_to_a_completed_run(repo, split_guest, ledger_service): + run_id = complete_on_host(ledger_service, repo) + + refusal = refusal_of( + split_guest.socket, str(gitutil.repo_identity(repo)), "event", run_id, None, "x", "" + ) + assert refusal == "run_closed" From 535c92b96caffd5bed0fd6bc81e799432f820e07 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:42:21 +0100 Subject: [PATCH 34/88] feat: the service refuses writes to complete or abandoned runs TDD-Run: 1 TDD-Cycle: 20 TDD-Phase: green --- src/tddcli/service.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index ab8c5b1..97e23a5 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -25,6 +25,9 @@ from . import runner, wire CONFIG_PATH = Path("/etc/tdd-cli/ledger.toml") + +#: A run with one of these outcomes takes no more writes from a guest. +CLOSED_OUTCOMES = ("complete", "abandoned") CONFIG_ENV = "TDD_LEDGER_SERVICE_CONFIG" @@ -101,6 +104,8 @@ def answer(self, request: dict) -> dict: except TypeError as exc: return {"ok": False, "error": f"{method}: {exc}"} refused = self._foreign(bound.arguments) + if refused is None and method in ledger_mod.GUEST_WRITES: + refused = self._closed(bound.arguments) if refused is not None: return refused try: @@ -125,6 +130,21 @@ def _foreign(self, arguments: dict) -> dict | None: return _refusal("foreign_run", f"{noun} {value} is not this source's") return None + def _closed(self, arguments: dict) -> dict | None: + """A refusal when the call writes to a run that is complete or abandoned. + + Judged before the method runs. `blocked` is not closed: `resume --unblock` and + `--accept-failures` write to a blocked run by design. + """ + run_id = arguments.get("run_id") + if run_id is None and arguments.get("cycle_id") is not None: + cycle = self.ledger.cycle(arguments["cycle_id"]) + run_id = cycle["run_id"] if cycle else None + run = self.ledger.run(run_id) if run_id is not None else None + if run is not None and run["outcome"] in CLOSED_OUTCOMES: + return _refusal("run_closed", f"run {run_id} is {run['outcome']}: it takes no writes") + return None + def _cycle_source(self, cycle_id: int) -> str | None: cycle = self.ledger.cycle(cycle_id) return None if cycle is None else self.ledger.run_source(cycle["run_id"]) From 0511a140c191ccfaab19be43e182db08a24b756e Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:42:38 +0100 Subject: [PATCH 35/88] test: notes on a completed run are still accepted TDD-Run: 1 TDD-Cycle: 21 TDD-Phase: red --- tests/test_ledger_service.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index 28b0310..b1c7173 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -180,3 +180,10 @@ def test_a_guest_cannot_write_to_a_completed_run(repo, split_guest, ledger_servi split_guest.socket, str(gitutil.repo_identity(repo)), "event", run_id, None, "x", "" ) assert refusal == "run_closed" + + +def test_a_guest_may_still_note_a_completed_run(repo, split_guest, ledger_service): + complete_on_host(ledger_service, repo) + + out = run_cli(repo, "note", "after the fact") + assert out["result"].get("noted") is True From 173e4ab8398a3aeeb1682ddc83ef928777c7978e Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:45:56 +0100 Subject: [PATCH 36/88] feat: appending a note is exempt from the closed-run rule TDD-Run: 1 TDD-Cycle: 21 TDD-Phase: green --- src/tddcli/service.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 97e23a5..76f2a22 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -28,6 +28,10 @@ #: A run with one of these outcomes takes no more writes from a guest. CLOSED_OUTCOMES = ("complete", "abandoned") + +#: The one write a closed run still takes: `tdd note` after the run is documented use, +#: and is how an executor leaves its closing narrative. +CLOSED_RUN_EXEMPT = frozenset({"add_note"}) CONFIG_ENV = "TDD_LEDGER_SERVICE_CONFIG" @@ -104,7 +108,7 @@ def answer(self, request: dict) -> dict: except TypeError as exc: return {"ok": False, "error": f"{method}: {exc}"} refused = self._foreign(bound.arguments) - if refused is None and method in ledger_mod.GUEST_WRITES: + if refused is None and method in ledger_mod.GUEST_WRITES - CLOSED_RUN_EXEMPT: refused = self._closed(bound.arguments) if refused is not None: return refused From c204eaabfef03e4e15e9ca6acab09f1b22b52251 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:46:15 +0100 Subject: [PATCH 37/88] test: a bound source records the operator's executor TDD-Run: 1 TDD-Cycle: 22 TDD-Phase: red --- tests/test_ledger_service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index b1c7173..b68d0cb 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -187,3 +187,15 @@ def test_a_guest_may_still_note_a_completed_run(repo, split_guest, ledger_servic out = run_cli(repo, "note", "after the fact") assert out["result"].get("noted") is True + + +def host_run(ledger_service, repo, run_id: int): + return ledger_mod.open_readonly(host_file(ledger_service, repo), source=None).run(run_id) + + +def test_a_bound_source_records_the_operator_assigned_executor(repo, split_guest, ledger_service): + ledger_service.service.add_source("vm-b", ledger_service.dir / "vm-b.sock", executor="model-x") + guest_config(split_guest.runner, ledger_service.dir / "vm-b.sock") + + run = host_run(ledger_service, repo, start_run(repo)["run"]["id"]) + assert (run["executor_model"], run["executor_source"]) == ("model-x", "operator") From 0ecac99bb6f1f02abe77cb908eae5d6bba48ce2e Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:49:41 +0100 Subject: [PATCH 38/88] feat: the service stamps a bound source's executor on run start TDD-Run: 1 TDD-Cycle: 22 TDD-Phase: green --- src/tddcli/service.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 76f2a22..c29d7a3 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -29,6 +29,9 @@ #: A run with one of these outcomes takes no more writes from a guest. CLOSED_OUTCOMES = ("complete", "abandoned") +#: Writes that record who executed a run: the service sets the executor on them. +STAMPED = frozenset({"start_run"}) + #: The one write a closed run still takes: `tdd note` after the run is documented use, #: and is how an executor leaves its closing narrative. CLOSED_RUN_EXEMPT = frozenset({"add_note"}) @@ -112,6 +115,9 @@ def answer(self, request: dict) -> dict: refused = self._closed(bound.arguments) if refused is not None: return refused + if method in STAMPED: + kwargs = self._stamp_executor(bound.arguments) + args = [] try: return _ok(fn(*args, **kwargs)) except Exception as exc: # the client gets the reason, the service keeps serving @@ -134,6 +140,20 @@ def _foreign(self, arguments: dict) -> dict | None: return _refusal("foreign_run", f"{noun} {value} is not this source's") return None + def _stamp_executor(self, arguments: dict) -> dict: + """The call's arguments, with the executor the operator bound to this source. + + A guest's own identity is guest root's to edit, so when the operator has said + which model a source runs, that is what the run records. + """ + stamped = dict(arguments) + binding = self.service.executors.get(self.source) + if binding is not None: + stamped.update( + executor_model=binding, executor_session=None, executor_source="operator" + ) + return stamped + def _closed(self, arguments: dict) -> dict | None: """A refusal when the call writes to a run that is complete or abandoned. From 4bc0ca86c7d2c722b0b9926d97eaf29950977e87 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:50:02 +0100 Subject: [PATCH 39/88] test: a guest cannot claim operator identity TDD-Run: 1 TDD-Cycle: 23 TDD-Phase: red --- tests/test_ledger_service.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index b68d0cb..24300fb 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -13,7 +13,7 @@ import pytest -from conftest import guest_config, run_cli, run_cli_text, write_plan +from conftest import current_user, guest_config, run_cli, run_cli_text, write_plan from tddcli import gitutil, service, wire from tddcli import ledger as ledger_mod @@ -199,3 +199,14 @@ def test_a_bound_source_records_the_operator_assigned_executor(repo, split_guest run = host_run(ledger_service, repo, start_run(repo)["run"]["id"]) assert (run["executor_model"], run["executor_source"]) == ("model-x", "operator") + + +def test_an_unbound_source_downgrades_a_guest_operator_claim(repo, split_guest, ledger_service): + guest_config( + split_guest.runner, + split_guest.socket, + extra=f'[executor]\n"{current_user()}" = "guest-says"\n', + ) + + run = host_run(ledger_service, repo, start_run(repo)["run"]["id"]) + assert (run["executor_model"], run["executor_source"]) == ("guest-says", "claimed") From 42259e74ec9ab9909b14523c1a7ba1a9ae927670 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:55:05 +0100 Subject: [PATCH 40/88] feat: an unbound source records guest identity as claimed TDD-Run: 1 TDD-Cycle: 23 TDD-Phase: green --- src/tddcli/service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index c29d7a3..9d7908d 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -144,7 +144,8 @@ def _stamp_executor(self, arguments: dict) -> dict: """The call's arguments, with the executor the operator bound to this source. A guest's own identity is guest root's to edit, so when the operator has said - which model a source runs, that is what the run records. + which model a source runs, that is what the run records. With no binding, the + guest's word passes through, except that it cannot speak for the operator. """ stamped = dict(arguments) binding = self.service.executors.get(self.source) @@ -152,6 +153,8 @@ def _stamp_executor(self, arguments: dict) -> dict: stamped.update( executor_model=binding, executor_session=None, executor_source="operator" ) + elif stamped.get("executor_source") == "operator": + stamped["executor_source"] = "claimed" return stamped def _closed(self, arguments: dict) -> dict | None: From ac5a7bd781b568877a984229a5a2bbade17659f4 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:55:25 +0100 Subject: [PATCH 41/88] test: abandonment records the bound executor TDD-Run: 1 TDD-Cycle: 24 TDD-Phase: red --- tests/test_ledger_service.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index 24300fb..e174af5 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -210,3 +210,14 @@ def test_an_unbound_source_downgrades_a_guest_operator_claim(repo, split_guest, run = host_run(ledger_service, repo, start_run(repo)["run"]["id"]) assert (run["executor_model"], run["executor_source"]) == ("guest-says", "claimed") + + +def test_a_bound_source_stamps_the_abandoning_executor(repo, split_guest, ledger_service): + ledger_service.service.add_source("vm-b", ledger_service.dir / "vm-b.sock", executor="model-x") + guest_config(split_guest.runner, ledger_service.dir / "vm-b.sock") + run_id = start_run(repo)["run"]["id"] + + run_cli(repo, "run", "abandon", "--reason", "x") + host = ledger_mod.open_readonly(host_file(ledger_service, repo), source=None) + row = host.abandonment(run_id) + assert (row["executor_model"], row["executor_source"]) == ("model-x", "operator") From 1055ac06e9ef56648a333759765c050bc48539a8 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:58:46 +0100 Subject: [PATCH 42/88] feat: the service stamps the executor on abandonment TDD-Run: 1 TDD-Cycle: 24 TDD-Phase: green --- src/tddcli/service.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 9d7908d..36fe32a 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -30,7 +30,7 @@ CLOSED_OUTCOMES = ("complete", "abandoned") #: Writes that record who executed a run: the service sets the executor on them. -STAMPED = frozenset({"start_run"}) +STAMPED = frozenset({"start_run", "abandon_run"}) #: The one write a closed run still takes: `tdd note` after the run is documented use, #: and is how an executor leaves its closing narrative. @@ -116,7 +116,7 @@ def answer(self, request: dict) -> dict: if refused is not None: return refused if method in STAMPED: - kwargs = self._stamp_executor(bound.arguments) + kwargs = self._stamp_executor(bound.arguments, set(bound.signature.parameters)) args = [] try: return _ok(fn(*args, **kwargs)) @@ -140,7 +140,7 @@ def _foreign(self, arguments: dict) -> dict | None: return _refusal("foreign_run", f"{noun} {value} is not this source's") return None - def _stamp_executor(self, arguments: dict) -> dict: + def _stamp_executor(self, arguments: dict, params: set[str]) -> dict: """The call's arguments, with the executor the operator bound to this source. A guest's own identity is guest root's to edit, so when the operator has said @@ -150,9 +150,9 @@ def _stamp_executor(self, arguments: dict) -> dict: stamped = dict(arguments) binding = self.service.executors.get(self.source) if binding is not None: - stamped.update( - executor_model=binding, executor_session=None, executor_source="operator" - ) + stamped.update(executor_model=binding, executor_source="operator") + if "executor_session" in params: + stamped["executor_session"] = None elif stamped.get("executor_source") == "operator": stamped["executor_source"] = "claimed" return stamped From b328906f5509ea84d6028d9c8c9d3451311e428c Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:58:46 +0100 Subject: [PATCH 43/88] refactor: tidy abandonment stamping TDD-Run: 1 TDD-Cycle: 24 TDD-Phase: refactor --- src/tddcli/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 36fe32a..6aad009 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -25,6 +25,7 @@ from . import runner, wire CONFIG_PATH = Path("/etc/tdd-cli/ledger.toml") +CONFIG_ENV = "TDD_LEDGER_SERVICE_CONFIG" #: A run with one of these outcomes takes no more writes from a guest. CLOSED_OUTCOMES = ("complete", "abandoned") @@ -35,7 +36,6 @@ #: The one write a closed run still takes: `tdd note` after the run is documented use, #: and is how an executor leaves its closing narrative. CLOSED_RUN_EXEMPT = frozenset({"add_note"}) -CONFIG_ENV = "TDD_LEDGER_SERVICE_CONFIG" class ServiceConfigError(RuntimeError): From 0a456b793fdfe619bfd0b15d092c4f7397597923 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:00:49 +0100 Subject: [PATCH 44/88] test: claim staleness is judged where the pid lives TDD-Run: 1 TDD-Cycle: 25 TDD-Phase: red --- tests/test_ledger_service.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index e174af5..febbe54 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -8,6 +8,8 @@ from __future__ import annotations import shutil +import socket +import subprocess import tempfile from pathlib import Path @@ -221,3 +223,16 @@ def test_a_bound_source_stamps_the_abandoning_executor(repo, split_guest, ledger host = ledger_mod.open_readonly(host_file(ledger_service, repo), source=None) row = host.abandonment(run_id) assert (row["executor_model"], row["executor_source"]) == ("model-x", "operator") + + +def test_claim_staleness_is_judged_in_the_guest(repo, split_guest, ledger_service, monkeypatch): + # The host cannot tell whether a guest's pid lives: stand that in for its judgement. + monkeypatch.setattr(ledger_mod.Ledger, "_claim_is_stale", staticmethod(lambda *a: False)) + proc = subprocess.Popen(["true"]) + proc.wait() + host = ledger_mod.Ledger(repo, source="vm-1", path=host_file(ledger_service, repo)) + host.claim(str(repo), socket.gethostname(), proc.pid, projects_total=2) + host.close() + + out = run_cli(repo, "progress", "--json") + assert out["result"].get("stale") is True From d7aad4b387510422e0bbe0f882c34d6185fd5624 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:04:23 +0100 Subject: [PATCH 45/88] feat: the remote ledger judges claim liveness locally TDD-Run: 1 TDD-Cycle: 25 TDD-Phase: green --- src/tddcli/ledger.py | 40 ++++++++++++++++++++-------------------- src/tddcli/remote.py | 10 ++++++++++ 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 75ad797..0850be1 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -423,8 +423,6 @@ def ledger_path(repo_path: Path) -> Path: GUEST_READS = frozenset( { "abandonment", - "active_advance_claim", - "active_claim", "active_run", "active_runs", "advance_claim_row", @@ -517,9 +515,12 @@ def ledger_path(repo_path: Path) -> Path: ) #: Never answered for a guest. The generic helpers would let it name any row of any -#: source. +#: source; `active_claim` and `active_advance_claim` judge a pid's liveness, which only +#: the guest that owns the pid can do, so a guest composes them from the raw rows. HOST_ONLY = frozenset( { + "active_advance_claim", + "active_claim", "all", "close", "insert", @@ -533,8 +534,6 @@ def ledger_path(repo_path: Path) -> Path: #: Guest methods with no run, cycle or contract id: each filters by the ledger's source. SOURCE_ROOTED = frozenset( { - "active_advance_claim", - "active_claim", "active_run", "active_runs", "advance_claim_row", @@ -607,6 +606,20 @@ def claim_is_stale(hostname: str, pid: int, started_at: str) -> bool: return datetime.now(timezone.utc) - started > timedelta(minutes=60) +def judged_claim(row, is_stale=claim_is_stale) -> dict | None: + """A raw claim row, with whether its owner is gone. Read-only: nothing is deleted. + + A pid is meaningless from another host, and reused pids would make a host-crossing + liveness check actively wrong. `claim_is_stale` falls back to age there: a false + "alive" bricks the worktree, a false "dead" reopens the bug (Decisions). + """ + if row is None: + return None + claim = dict(row) + claim["stale"] = is_stale(claim["hostname"], claim["pid"], claim["started_at"]) + return claim + + class Ledger: _claim_is_stale = staticmethod(claim_is_stale) @@ -944,26 +957,13 @@ def release_advance_claim(self, worktree: str) -> None: def active_advance_claim(self, worktree: str) -> dict | None: """Read-only observer — staleness computed but no row deleted.""" - row = self.advance_claim_row(worktree) - if row is None: - return None - claim = dict(row) - claim["stale"] = self._claim_is_stale(claim["hostname"], claim["pid"], claim["started_at"]) - return claim + return judged_claim(self.advance_claim_row(worktree), self._claim_is_stale) def active_claim(self, worktree: str) -> dict | None: """Read-only, per the store's append-only contract — `cmd_progress` and `cmd_status` call this as pure observers. Only `cmd_run_start` acts on the computed `stale` flag (release + reclaim); nothing here deletes a row.""" - row = self.claim_row(worktree) - if row is None: - return None - claim = dict(row) - # A pid is meaningless from another host, and reused pids would make a - # host-crossing liveness check actively wrong. Fall back to age: a false - # "alive" bricks the worktree, a false "dead" reopens the bug (Decisions). - claim["stale"] = self._claim_is_stale(claim["hostname"], claim["pid"], claim["started_at"]) - return claim + return judged_claim(self.claim_row(worktree), self._claim_is_stale) def claim_row(self, worktree: str) -> sqlite3.Row | None: """The raw baseline claim, with no judgement of whether its owner lives.""" diff --git a/src/tddcli/remote.py b/src/tddcli/remote.py index 824148f..cd5f427 100644 --- a/src/tddcli/remote.py +++ b/src/tddcli/remote.py @@ -11,6 +11,7 @@ from pathlib import Path from . import wire +from .ledger import claim_is_stale, judged_claim class RemoteLedger: @@ -31,5 +32,14 @@ def call(*args, **kwargs): return call + # Claim liveness is judged here, where the claiming pid lives. On the host the + # guest's hostname is foreign, so the host could only fall back to the claim's age. + + def active_claim(self, worktree: str) -> dict | None: + return judged_claim(self.claim_row(worktree), claim_is_stale) + + def active_advance_claim(self, worktree: str) -> dict | None: + return judged_claim(self.advance_claim_row(worktree), claim_is_stale) + def close(self) -> None: self._conn.close() From e2bbcfbd2278e59031de6f30b97f5e19e885d6bb Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:04:43 +0100 Subject: [PATCH 46/88] test: runner import is refused when the ledger is remote TDD-Run: 1 TDD-Cycle: 26 TDD-Phase: red --- tests/test_runner_import.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_runner_import.py b/tests/test_runner_import.py index 53c4d11..37d682e 100644 --- a/tests/test_runner_import.py +++ b/tests/test_runner_import.py @@ -146,3 +146,16 @@ def test_root_may_import_and_is_told_what_was_marked( told = (out["ok"], out["result"]["pre_split_import"]["last_run_id"]) assert told == (True, 1) + + +def test_import_is_refused_when_the_ledger_is_on_the_host(repo, tmp_path, split_guest, monkeypatch): + from tddcli.ledger import Ledger + + legacy = tmp_path / "legacy.sqlite3" + Ledger(tmp_path, path=legacy).close() + # Logged in as the runner: no sudo in between. + monkeypatch.delenv("SUDO_USER") + monkeypatch.delenv("SUDO_UID") + + out = run_cli(repo, "runner", "import", str(legacy)) + assert out["result"].get("reason") == "ledger_remote" From 886914225e999124dbffda8bff8e3c5a41175d43 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:08:08 +0100 Subject: [PATCH 47/88] feat: runner import refuses a socket-mode runner TDD-Run: 1 TDD-Cycle: 26 TDD-Phase: green --- src/tddcli/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index efc6380..4f5296d 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1582,6 +1582,13 @@ def cmd_runner_import(args) -> Envelope: "this machine is not split, or this is not its runner: `tdd docs split` sets one up", reason="not_split", ) + if split.ledger_socket is not None: + # This runner keeps no ledger: it is the host's, behind the socket. + return failure( + "this runner keeps its ledger on the host's ledger service: import there, with" + " `tdd ledger import --source `", + reason="ledger_remote", + ) # sudo sets SUDO_UID itself. Unset means someone logged in as the runner; "0" means # root. Anything else reached us the way an agent does, and an agent that could # import could hand the runner a history it wrote itself. From 51610993c20ea19b011a75ee9bfc46f430da3d78 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:08:22 +0100 Subject: [PATCH 48/88] test: ledger_socket and ledger_home are exclusive TDD-Run: 1 TDD-Cycle: 27 TDD-Phase: red --- tests/test_split_config.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_split_config.py b/tests/test_split_config.py index d4cfc1b..e066828 100644 --- a/tests/test_split_config.py +++ b/tests/test_split_config.py @@ -5,6 +5,8 @@ import os import pwd +import pytest + from tddcli import runner @@ -56,3 +58,15 @@ def refused(mode: int, euid: int) -> bool: "foreign owner": refused(0o644, real_euid + 1), } assert forgeries == dict.fromkeys(forgeries, True) + + +def test_ledger_socket_and_ledger_home_together_are_refused(tmp_path, monkeypatch): + cfg = tmp_path / "runner.toml" + monkeypatch.setenv("TDD_RUNNER_CONFIG", str(cfg)) + cfg.write_text( + f'[runner]\nuser = "{_current_user()}"\ncommand = "/opt/tdd-cli/bin/tdd"\n' + 'ledger_home = "/var/lib/tdd"\nledger_socket = "/run/tdd-cli/ledger.sock"\n' + ) + + with pytest.raises(runner.RunnerConfigError, match="ledger_socket"): + runner.load() From f17d37487e4a4345391dbc1c9a6a4f8b5bcf0313 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:11:52 +0100 Subject: [PATCH 49/88] feat: refuse a runner config naming both ledger locations TDD-Run: 1 TDD-Cycle: 27 TDD-Phase: green --- src/tddcli/runner.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tddcli/runner.py b/src/tddcli/runner.py index a3ffeda..38fe030 100644 --- a/src/tddcli/runner.py +++ b/src/tddcli/runner.py @@ -103,6 +103,13 @@ def load() -> RunnerConfig | None: ) from exc ledger_home = section.get("ledger_home") ledger_socket = section.get("ledger_socket") + if ledger_home and ledger_socket: + # Two places for the ledger is one too many: a runner that kept a local one as + # well would be falling back to it, which socket mode must never do. + raise RunnerConfigError( + f"{path}: [runner] names both `ledger_socket` and `ledger_home`; a runner keeps" + " its ledger in one place" + ) return RunnerConfig( user=section["user"], command=section["command"], From dacedc7c9d6337c4f8b000a84950a1693a3da81c Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:14:48 +0100 Subject: [PATCH 50/88] refactor: tidy socket-mode doctor TDD-Run: 1 TDD-Cycle: 28 TDD-Phase: refactor --- src/tddcli/cli.py | 36 +++++++++++++++++++++++------------- src/tddcli/remote.py | 4 ++++ tests/test_split_doctor.py | 7 +++++++ 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 4f5296d..4458882 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -46,6 +46,7 @@ from .envelope import Envelope, NextAction, Verb, failure, heartbeat from .ledger import Ledger, LedgerVersionError, ledger_path, now, open_ledger from .machine import CLOSED, SKIPPED, Engine +from .remote import RemoteLedger BASELINE_MAX_FAILURE_RATIO_DEFAULT = 0.5 BASELINE_MIN_COLLECTED = 10 @@ -311,12 +312,15 @@ def listed(paths: list[str]) -> str: return "" -def _split_checks(check: Callable, split: runner_mod.RunnerConfig, ledger_file: Path) -> None: +def _split_checks( + check: Callable, split: runner_mod.RunnerConfig, ledger_file: Path | None +) -> None: """What split mode rests on, probed live and as the agent. Each probe goes through the installed actor, so on a real machine it is answered by the real sudo and the real accounts: that is the one place the uid change - itself is ever verified. + itself is ever verified. `ledger_file` is None when the ledger is on the host's + ledger service: there is then no file here for the agent to reach. """ agent = os.environ.get("SUDO_USER", "") asked = actor.current().run_argv(["id", "-u"]) @@ -333,15 +337,16 @@ def _split_checks(check: Callable, split: runner_mod.RunnerConfig, ledger_file: f" `{split.user} ALL=({agent}) NOPASSWD:SETENV: ALL`", ) - readable = actor.current().run_argv(["test", "-r", str(ledger_file)]).returncode == 0 - check( - "ledger out of the agent's reach", - not readable, - f"{agent!r} can read {ledger_file}. Its directory must belong to {split.user}" - " alone, at mode 700, somewhere the agent's account cannot otherwise reach" - if readable - else "", - ) + if ledger_file is not None: + readable = actor.current().run_argv(["test", "-r", str(ledger_file)]).returncode == 0 + check( + "ledger out of the agent's reach", + not readable, + f"{agent!r} can read {ledger_file}. Its directory must belong to {split.user}" + " alone, at mode 700, somewhere the agent's account cannot otherwise reach" + if readable + else "", + ) # A runner whose code the agent can edit is the agent. install = Path(__file__).resolve().parent @@ -370,7 +375,12 @@ def cmd_doctor(args) -> Envelope: repo = gitutil.repo_identity(worktree) ledger = open_ledger(repo) - check("ledger reachable", True, str(ledger.path)) + remote = isinstance(ledger, RemoteLedger) + if remote: + # Reaching the service is not enough: say which source it records this guest as. + check("ledger reachable", True, f"socket {ledger.path} (source {ledger.ping()['source']})") + else: + check("ledger reachable", True, str(ledger.path)) check( "ledger outside worktree", not str(ledger.path).startswith(str(worktree)), str(ledger.path) ) @@ -387,7 +397,7 @@ def cmd_doctor(args) -> Envelope: " cannot reach.", ) else: - _split_checks(check, split, ledger.path) + _split_checks(check, split, None if remote else ledger.path) ex = identity.resolve(worktree) ex_detail = f"{ex.source}: {ex.model}" diff --git a/src/tddcli/remote.py b/src/tddcli/remote.py index cd5f427..8946c08 100644 --- a/src/tddcli/remote.py +++ b/src/tddcli/remote.py @@ -32,6 +32,10 @@ def call(*args, **kwargs): return call + def ping(self) -> dict: + """The source this guest is recorded under, and the executor bound to it.""" + return self._conn.request("ping") + # Claim liveness is judged here, where the claiming pid lives. On the host the # guest's hostname is foreign, so the host could only fall back to the claim's age. diff --git a/tests/test_split_doctor.py b/tests/test_split_doctor.py index 33e515b..aa777fd 100644 --- a/tests/test_split_doctor.py +++ b/tests/test_split_doctor.py @@ -59,3 +59,10 @@ def test_split_doctor_install_check_follows_what_the_agent_can_write( verdicts, _ = _probe(repo, monkeypatch, "install not writable by the agent", deny="test") assert verdicts == [False, True] + + +def test_socket_mode_doctor_names_the_source_it_reaches(repo, split_guest): + reachable = _check(run_cli(repo, "doctor"), "ledger reachable") + + # `source vm-1`, not just `vm-1`: the socket's own file name already says that. + assert (reachable["ok"], "source vm-1" in reachable["detail"]) == (True, True) From 28b66005a12d4f3870d186ff12b810949ea1eed4 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:16:38 +0100 Subject: [PATCH 51/88] test: ledger verbs stay local and need a service config TDD-Run: 1 TDD-Cycle: 29 TDD-Phase: red --- tests/test_ledger_admin.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/test_ledger_admin.py diff --git a/tests/test_ledger_admin.py b/tests/test_ledger_admin.py new file mode 100644 index 0000000..227c859 --- /dev/null +++ b/tests/test_ledger_admin.py @@ -0,0 +1,16 @@ +"""`tdd ledger …`: the operator's verbs, answered on the host over the admin socket. + +The operator adds a source per guest, binds the executor it runs, and reads every +source's history in one place. None of these verbs is ever forwarded to a runner. +""" + +from __future__ import annotations + +from conftest import run_cli + + +def test_ledger_verbs_are_never_forwarded_to_a_runner(repo, split_client, tmp_path, monkeypatch): + monkeypatch.setenv("TDD_LEDGER_SERVICE_CONFIG", str(tmp_path / "no-service.toml")) + + out = run_cli(repo, "ledger", "sources") + assert out.get("result", {}).get("reason") == "no_service" From c3bf0568c517e1f25f6aa795957ffb32ab55e0dd Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:22:27 +0100 Subject: [PATCH 52/88] feat: tdd ledger verb group, answered locally TDD-Run: 1 TDD-Cycle: 29 TDD-Phase: green --- src/tddcli/cli.py | 77 +++++++++++++++++++++++++++++++++++--- tests/test_split_client.py | 6 ++- 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 4458882..6f05a43 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -40,6 +40,7 @@ ) from . import ledger as ledger_mod from . import runner as runner_mod +from . import service as service_mod from . import target_lint as target_lint_mod from .adapters.base import FAILED, NOT_COLLECTED from .advance import advance as do_advance @@ -1630,6 +1631,33 @@ def cmd_runner_import(args) -> Envelope: ) +# -- the ledger service: the operator's verbs ---------------------------------------- + + +def _ledger_verb(answer: Callable) -> Callable: + """A `tdd ledger` handler: it needs this machine's ledger service config first. + + These verbs are the host operator's. They are answered here, never by a runner, + and only on a machine that runs a ledger service. + """ + + def handler(args) -> Envelope: + path = service_mod.config_path() + if not path.is_file(): + return failure( + f"no ledger service is configured on this machine ({path} is absent):" + " `tdd docs split` describes one", + reason="no_service", + ) + return answer(args, service_mod.load_config(path)) + + return handler + + +def _not_implemented(args, cfg) -> Envelope: + return failure("not implemented", reason="not_implemented") + + def cmd_docs(args) -> Envelope: """The shipped documentation, printed. No network, and versioned with the binary. @@ -1819,13 +1847,48 @@ def build_parser() -> argparse.ArgumentParser: s = runner_p.add_parser("import") s.add_argument("ledger") s.set_defaults(fn=cmd_runner_import) + + ledger_p = sub.add_parser( + "ledger", help="the host's ledger service: its sources and every source's history" + ).add_subparsers(dest="ledger_command", required=True) + s = ledger_p.add_parser("serve", help="run the ledger service until SIGTERM") + s.set_defaults(fn=_ledger_verb(_not_implemented)) + s = ledger_p.add_parser("sources", help="every source, its socket and its executor") + s.set_defaults(fn=_ledger_verb(_not_implemented)) + s = ledger_p.add_parser("add-source", help="open a socket for a new guest") + s.add_argument("name") + s.add_argument("--socket", required=True) + s.add_argument("--executor", help="the model this source runs, recorded on its runs") + s.set_defaults(fn=_ledger_verb(_not_implemented)) + s = ledger_p.add_parser("remove-source", help="close a guest's socket") + s.add_argument("name") + s.set_defaults(fn=_ledger_verb(_not_implemented)) + s = ledger_p.add_parser("bind", help="set the executor a source's next runs record") + s.add_argument("name") + s.add_argument("--executor", required=True) + s.set_defaults(fn=_ledger_verb(_not_implemented)) + s = ledger_p.add_parser("fleet", help="active runs of every source") + s.set_defaults(fn=_ledger_verb(_not_implemented)) + s = ledger_p.add_parser("metrics", help="metrics over every source's runs") + s.add_argument("--repo", help="one repository's ledger, by its slug") + s.set_defaults(fn=_ledger_verb(_not_implemented)) + s = ledger_p.add_parser("log", help="render any source's run as a friction log") + s.add_argument("--repo", required=True, help="the repository's slug") + s.add_argument("--run", type=int, required=True) + s.add_argument("--out") + s.set_defaults(fn=_ledger_verb(_not_implemented)) + s = ledger_p.add_parser("import", help="bring an existing ledger in as one source") + s.add_argument("file") + s.add_argument("--source", required=True) + s.set_defaults(fn=_ledger_verb(_not_implemented)) return p -#: Verbs an agent's own `tdd` answers on a split machine. Neither reads or writes the -#: ledger: `docs` prints what shipped with the binary, and `init` writes `tdd.toml` -#: into the agent's own worktree. Everything else is the runner's to do. -LOCAL_VERBS = {"docs", "init"} +#: Verbs an agent's own `tdd` answers on a split machine. `docs` prints what shipped +#: with the binary, and `init` writes `tdd.toml` into the agent's own worktree; neither +#: touches the ledger. `ledger` is the host operator's, and talks to the ledger service +#: on this machine. Everything else is the runner's to do. +LOCAL_VERBS = {"docs", "init", "ledger"} def _actor_for( @@ -1852,11 +1915,12 @@ def _refusal(split: runner_mod.RunnerConfig | None, args) -> Envelope | None: It must not fall back to acting as itself: that would run the agent's suites, and the agent's git hooks, as the uid that owns the ledger. `runner` verbs are the - exception: they are the operator's, touch no worktree and spawn nothing. + exception, and so are `ledger` verbs: they are the operator's, touch no worktree + and spawn nothing. """ if split is None or split.role != "runner" or os.environ.get("SUDO_USER"): return None - if args.command == "runner": + if args.command in ("runner", "ledger"): return None return failure( f"this is the split-mode runner ({split.user}) and no agent called it: run `tdd`" @@ -1914,6 +1978,7 @@ def main(argv: list[str] | None = None) -> int: gitutil.GitError, LedgerVersionError, runner_mod.RunnerConfigError, + service_mod.ServiceConfigError, ) as exc: envelope = failure(str(exc)) except SystemExit as exc: diff --git a/tests/test_split_client.py b/tests/test_split_client.py index b360422..17d7a14 100644 --- a/tests/test_split_client.py +++ b/tests/test_split_client.py @@ -70,6 +70,7 @@ def test_the_client_sends_its_environment_on_stdin(repo, split_client, monkeypat "target": ["target", "x"], "log": ["log", "render"], "runner": ["runner", "import", "x"], + "ledger": ["ledger", "sources"], } @@ -83,7 +84,8 @@ def _verbs() -> list[str]: return sorted(subparsers.choices) -def test_only_docs_and_init_stay_local(repo, split_client): +def test_only_docs_init_and_ledger_stay_local(repo, split_client): + # `ledger` is the host operator's: it talks to this machine's ledger service. stayed_local = set() for verb in _verbs(): before = len(split_client.shim.lines()) @@ -91,4 +93,4 @@ def test_only_docs_and_init_stay_local(repo, split_client): if len(split_client.shim.lines()) == before: stayed_local.add(verb) - assert stayed_local == {"docs", "init"} + assert stayed_local == {"docs", "init", "ledger"} From 5ef9abfc10fb45450dac596c5d1973266e09f40d Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:22:46 +0100 Subject: [PATCH 53/88] test: add-source opens a guest socket TDD-Run: 1 TDD-Cycle: 30 TDD-Phase: red --- tests/test_ledger_admin.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_ledger_admin.py b/tests/test_ledger_admin.py index 227c859..ba21787 100644 --- a/tests/test_ledger_admin.py +++ b/tests/test_ledger_admin.py @@ -7,6 +7,7 @@ from __future__ import annotations from conftest import run_cli +from tddcli import wire def test_ledger_verbs_are_never_forwarded_to_a_runner(repo, split_client, tmp_path, monkeypatch): @@ -14,3 +15,10 @@ def test_ledger_verbs_are_never_forwarded_to_a_runner(repo, split_client, tmp_pa out = run_cli(repo, "ledger", "sources") assert out.get("result", {}).get("reason") == "no_service" + + +def test_add_source_opens_a_guest_socket(repo, ledger_service): + socket_path = ledger_service.dir / "vm-2.sock" + run_cli(repo, "ledger", "add-source", "vm-2", "--socket", str(socket_path)) + + assert wire.call(socket_path, "ping") == {"source": "vm-2", "executor": None} From f9f10ab9ea7876473bfff6d82cfa741655c0b790 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:26:28 +0100 Subject: [PATCH 54/88] feat: tdd ledger add-source TDD-Run: 1 TDD-Cycle: 30 TDD-Phase: green --- src/tddcli/cli.py | 16 +++++++++++++++- src/tddcli/service.py | 11 +++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 6f05a43..722c3ab 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1658,6 +1658,20 @@ def _not_implemented(args, cfg) -> Envelope: return failure("not implemented", reason="not_implemented") +def _ledger_add_source(args, cfg) -> Envelope: + added = wire.call( + cfg.admin_socket, "add_source", args.name, args.socket, executor=args.executor + ) + return Envelope( + result=added, + next_action=NextAction( + Verb.COMPLETE, + f"Source {args.name} listens on {args.socket}. Forward that socket into the" + " guest and name it as the guest runner's `ledger_socket`.", + ), + ) + + def cmd_docs(args) -> Envelope: """The shipped documentation, printed. No network, and versioned with the binary. @@ -1859,7 +1873,7 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("name") s.add_argument("--socket", required=True) s.add_argument("--executor", help="the model this source runs, recorded on its runs") - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_add_source)) s = ledger_p.add_parser("remove-source", help="close a guest's socket") s.add_argument("name") s.set_defaults(fn=_ledger_verb(_not_implemented)) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 6aad009..5bafe37 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -91,6 +91,9 @@ def answer(self, request: dict) -> dict: kwargs = request.get("kwargs") or {} if method == "ping": return _ok({"source": self.source, "executor": self.service.executors.get(self.source)}) + if self.source is None: + # The admin socket: the operator's verbs, never a ledger method. + return self.service.admin(method, args, kwargs) if method == "open": repo = args[0] if args else None if not _well_formed_repo(repo): @@ -257,3 +260,11 @@ def stop(self) -> None: def add_source(self, name: str, socket_path: Path, executor: str | None = None) -> None: self.executors[name] = executor self.listeners[name] = _Listener(Path(socket_path), self, name) + + def admin(self, method: str, args: list, kwargs: dict) -> dict: + """An operator's request, from the admin socket.""" + if method == "add_source": + name, socket_path = args + self.add_source(name, Path(socket_path), executor=kwargs.get("executor")) + return _ok({"name": name, "socket": str(socket_path), "executor": self.executors[name]}) + return _refusal("method_not_allowed", f"{method} is not an admin request") From 9a235e432779a6d102c15b483e33b2550b885e4f Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:26:40 +0100 Subject: [PATCH 55/88] test: ledger sources lists sources TDD-Run: 1 TDD-Cycle: 31 TDD-Phase: red --- tests/test_ledger_admin.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_ledger_admin.py b/tests/test_ledger_admin.py index ba21787..8a07d1c 100644 --- a/tests/test_ledger_admin.py +++ b/tests/test_ledger_admin.py @@ -22,3 +22,15 @@ def test_add_source_opens_a_guest_socket(repo, ledger_service): run_cli(repo, "ledger", "add-source", "vm-2", "--socket", str(socket_path)) assert wire.call(socket_path, "ping") == {"source": "vm-2", "executor": None} + + +def test_sources_lists_each_source_and_its_binding(repo, ledger_service): + svc, sockets = ledger_service.service, ledger_service.dir + svc.add_source("vm-2", sockets / "vm-2.sock", executor="m") + svc.add_source("vm-1", sockets / "vm-1.sock", executor=None) + + out = run_cli(repo, "ledger", "sources") + assert out["result"].get("sources") == [ + {"name": "vm-1", "socket": str(sockets / "vm-1.sock"), "executor": None}, + {"name": "vm-2", "socket": str(sockets / "vm-2.sock"), "executor": "m"}, + ] From 21c2ec6c4566dffb91cd47adee5085d108904673 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:30:15 +0100 Subject: [PATCH 56/88] feat: tdd ledger sources TDD-Run: 1 TDD-Cycle: 31 TDD-Phase: green --- src/tddcli/cli.py | 10 +++++++++- src/tddcli/service.py | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 722c3ab..0ba3dac 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1658,6 +1658,14 @@ def _not_implemented(args, cfg) -> Envelope: return failure("not implemented", reason="not_implemented") +def _ledger_sources(args, cfg) -> Envelope: + sources = wire.call(cfg.admin_socket, "sources") + return Envelope( + result={"sources": sources}, + next_action=NextAction(Verb.COMPLETE, f"{len(sources)} source(s)."), + ) + + def _ledger_add_source(args, cfg) -> Envelope: added = wire.call( cfg.admin_socket, "add_source", args.name, args.socket, executor=args.executor @@ -1868,7 +1876,7 @@ def build_parser() -> argparse.ArgumentParser: s = ledger_p.add_parser("serve", help="run the ledger service until SIGTERM") s.set_defaults(fn=_ledger_verb(_not_implemented)) s = ledger_p.add_parser("sources", help="every source, its socket and its executor") - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_sources)) s = ledger_p.add_parser("add-source", help="open a socket for a new guest") s.add_argument("name") s.add_argument("--socket", required=True) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 5bafe37..88575aa 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -261,8 +261,17 @@ def add_source(self, name: str, socket_path: Path, executor: str | None = None) self.executors[name] = executor self.listeners[name] = _Listener(Path(socket_path), self, name) + def sources(self) -> list[dict]: + """Every source, by name: the socket it listens on and the executor bound to it.""" + return [ + {"name": name, "socket": str(self.listeners[name].path), "executor": executor} + for name, executor in sorted(self.executors.items()) + ] + def admin(self, method: str, args: list, kwargs: dict) -> dict: """An operator's request, from the admin socket.""" + if method == "sources": + return _ok(self.sources()) if method == "add_source": name, socket_path = args self.add_source(name, Path(socket_path), executor=kwargs.get("executor")) From 3d2078169ee6314cf31e3ab384eb95e56717da2f Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:30:24 +0100 Subject: [PATCH 57/88] test: bind changes a source's executor TDD-Run: 1 TDD-Cycle: 32 TDD-Phase: red --- tests/test_ledger_admin.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_ledger_admin.py b/tests/test_ledger_admin.py index 8a07d1c..088b50a 100644 --- a/tests/test_ledger_admin.py +++ b/tests/test_ledger_admin.py @@ -34,3 +34,11 @@ def test_sources_lists_each_source_and_its_binding(repo, ledger_service): {"name": "vm-1", "socket": str(sockets / "vm-1.sock"), "executor": None}, {"name": "vm-2", "socket": str(sockets / "vm-2.sock"), "executor": "m"}, ] + + +def test_bind_changes_the_executor_for_the_next_run(repo, ledger_service): + socket_path = ledger_service.dir / "vm-1.sock" + ledger_service.service.add_source("vm-1", socket_path, executor=None) + + run_cli(repo, "ledger", "bind", "vm-1", "--executor", "m2") + assert wire.call(socket_path, "ping")["executor"] == "m2" From 5679302e5efe49785bbad83e76938a99d3a3137e Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:33:51 +0100 Subject: [PATCH 58/88] feat: tdd ledger bind TDD-Run: 1 TDD-Cycle: 32 TDD-Phase: green --- src/tddcli/cli.py | 13 ++++++++++++- src/tddcli/service.py | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 0ba3dac..9f24eb9 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1666,6 +1666,17 @@ def _ledger_sources(args, cfg) -> Envelope: ) +def _ledger_bind(args, cfg) -> Envelope: + bound = wire.call(cfg.admin_socket, "bind", args.name, args.executor) + return Envelope( + result=bound, + next_action=NextAction( + Verb.COMPLETE, + f"Source {args.name}'s next runs record {args.executor}, as said by the operator.", + ), + ) + + def _ledger_add_source(args, cfg) -> Envelope: added = wire.call( cfg.admin_socket, "add_source", args.name, args.socket, executor=args.executor @@ -1888,7 +1899,7 @@ def build_parser() -> argparse.ArgumentParser: s = ledger_p.add_parser("bind", help="set the executor a source's next runs record") s.add_argument("name") s.add_argument("--executor", required=True) - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_bind)) s = ledger_p.add_parser("fleet", help="active runs of every source") s.set_defaults(fn=_ledger_verb(_not_implemented)) s = ledger_p.add_parser("metrics", help="metrics over every source's runs") diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 88575aa..73ced0a 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -272,6 +272,12 @@ def admin(self, method: str, args: list, kwargs: dict) -> dict: """An operator's request, from the admin socket.""" if method == "sources": return _ok(self.sources()) + if method == "bind": + name, executor = args + if name not in self.executors: + return _refusal("unknown_source", f"no source named {name!r}") + self.executors[name] = executor + return _ok({"name": name, "executor": executor}) if method == "add_source": name, socket_path = args self.add_source(name, Path(socket_path), executor=kwargs.get("executor")) From dbe2d5cee5a35be11f04e28e3b64af904bab3a79 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:34:01 +0100 Subject: [PATCH 59/88] test: remove-source closes the guest socket TDD-Run: 1 TDD-Cycle: 33 TDD-Phase: red --- tests/test_ledger_admin.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_ledger_admin.py b/tests/test_ledger_admin.py index 088b50a..d61b38d 100644 --- a/tests/test_ledger_admin.py +++ b/tests/test_ledger_admin.py @@ -42,3 +42,19 @@ def test_bind_changes_the_executor_for_the_next_run(repo, ledger_service): run_cli(repo, "ledger", "bind", "vm-1", "--executor", "m2") assert wire.call(socket_path, "ping")["executor"] == "m2" + + +def unreachable(socket_path) -> bool: + try: + wire.call(socket_path, "ping") + except wire.LedgerUnreachable: + return True + return False + + +def test_remove_source_closes_its_socket(repo, ledger_service): + socket_path = ledger_service.dir / "vm-1.sock" + ledger_service.service.add_source("vm-1", socket_path, executor=None) + + run_cli(repo, "ledger", "remove-source", "vm-1") + assert (socket_path.exists(), unreachable(socket_path)) == (False, True) From 4a68d6891d9089dd2e26fe89f8813fddfeba9abf Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:37:29 +0100 Subject: [PATCH 60/88] feat: tdd ledger remove-source TDD-Run: 1 TDD-Cycle: 33 TDD-Phase: green --- src/tddcli/cli.py | 12 +++++++++++- src/tddcli/service.py | 11 +++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 9f24eb9..6e2332e 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1677,6 +1677,16 @@ def _ledger_bind(args, cfg) -> Envelope: ) +def _ledger_remove_source(args, cfg) -> Envelope: + removed = wire.call(cfg.admin_socket, "remove_source", args.name) + return Envelope( + result=removed, + next_action=NextAction( + Verb.COMPLETE, f"Source {args.name}'s socket is closed; its runs stay recorded." + ), + ) + + def _ledger_add_source(args, cfg) -> Envelope: added = wire.call( cfg.admin_socket, "add_source", args.name, args.socket, executor=args.executor @@ -1895,7 +1905,7 @@ def build_parser() -> argparse.ArgumentParser: s.set_defaults(fn=_ledger_verb(_ledger_add_source)) s = ledger_p.add_parser("remove-source", help="close a guest's socket") s.add_argument("name") - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_remove_source)) s = ledger_p.add_parser("bind", help="set the executor a source's next runs record") s.add_argument("name") s.add_argument("--executor", required=True) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 73ced0a..54097e0 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -261,6 +261,11 @@ def add_source(self, name: str, socket_path: Path, executor: str | None = None) self.executors[name] = executor self.listeners[name] = _Listener(Path(socket_path), self, name) + def remove_source(self, name: str) -> None: + """Close the source's socket. Its runs stay in the host's ledgers.""" + self.listeners.pop(name).close() + del self.executors[name] + def sources(self) -> list[dict]: """Every source, by name: the socket it listens on and the executor bound to it.""" return [ @@ -278,6 +283,12 @@ def admin(self, method: str, args: list, kwargs: dict) -> dict: return _refusal("unknown_source", f"no source named {name!r}") self.executors[name] = executor return _ok({"name": name, "executor": executor}) + if method == "remove_source": + (name,) = args + if name not in self.executors: + return _refusal("unknown_source", f"no source named {name!r}") + self.remove_source(name) + return _ok({"name": name}) if method == "add_source": name, socket_path = args self.add_source(name, Path(socket_path), executor=kwargs.get("executor")) From 758a9c762269aaf94eb239afd16a117900347400 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:37:44 +0100 Subject: [PATCH 61/88] test: sources survive a service restart TDD-Run: 1 TDD-Cycle: 34 TDD-Phase: red --- tests/test_ledger_admin.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/test_ledger_admin.py b/tests/test_ledger_admin.py index d61b38d..b57c15b 100644 --- a/tests/test_ledger_admin.py +++ b/tests/test_ledger_admin.py @@ -7,7 +7,7 @@ from __future__ import annotations from conftest import run_cli -from tddcli import wire +from tddcli import service, wire def test_ledger_verbs_are_never_forwarded_to_a_runner(repo, split_client, tmp_path, monkeypatch): @@ -58,3 +58,24 @@ def test_remove_source_closes_its_socket(repo, ledger_service): run_cli(repo, "ledger", "remove-source", "vm-1") assert (socket_path.exists(), unreachable(socket_path)) == (False, True) + + +def ping_or_unreachable(socket_path): + try: + return wire.call(socket_path, "ping") + except wire.LedgerUnreachable: + return "unreachable" + + +def test_sources_survive_a_service_restart(ledger_service): + socket_path = ledger_service.dir / "vm-1.sock" + ledger_service.service.add_source("vm-1", socket_path, executor="m") + ledger_service.service.stop() + + restarted = service.Service(service.load_config(ledger_service.config)) + restarted.start() + try: + reply = ping_or_unreachable(socket_path) + finally: + restarted.stop() + assert reply == {"source": "vm-1", "executor": "m"} From 7452f6485beec2e3963849d0ddf815e605e72417 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:41:22 +0100 Subject: [PATCH 62/88] feat: the service persists its sources TDD-Run: 1 TDD-Cycle: 34 TDD-Phase: green --- src/tddcli/service.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 54097e0..913187a 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -14,6 +14,7 @@ import contextlib import inspect +import json import os import socketserver import threading @@ -33,6 +34,9 @@ #: Writes that record who executed a run: the service sets the executor on them. STAMPED = frozenset({"start_run", "abandon_run"}) +#: Where the service keeps its sources, under its home, so a restart listens on them again. +SOURCES_FILE = "sources.json" + #: The one write a closed run still takes: `tdd note` after the run is documented use, #: and is how an executor leaves its closing narrative. CLOSED_RUN_EXEMPT = frozenset({"add_note"}) @@ -251,6 +255,23 @@ def __init__(self, config: ServiceConfig) -> None: def start(self) -> None: self.config.home.mkdir(parents=True, exist_ok=True) self.listeners[None] = _Listener(self.config.admin_socket, self, None) + for name, entry in sorted(self._saved().items()): + self.add_source(name, Path(entry["socket"]), entry.get("executor")) + + def _saved(self) -> dict: + path = self.config.home / SOURCES_FILE + return json.loads(path.read_text()) if path.is_file() else {} + + def _save(self) -> None: + """Persist every source, replacing the file whole so a crash never halves it.""" + path = self.config.home / SOURCES_FILE + sources = { + name: {"socket": str(self.listeners[name].path), "executor": executor} + for name, executor in sorted(self.executors.items()) + } + partial = path.with_name(path.name + ".partial") + partial.write_text(json.dumps(sources, indent=2) + "\n") + partial.replace(path) def stop(self) -> None: for listener in list(self.listeners.values()): @@ -260,11 +281,17 @@ def stop(self) -> None: def add_source(self, name: str, socket_path: Path, executor: str | None = None) -> None: self.executors[name] = executor self.listeners[name] = _Listener(Path(socket_path), self, name) + self._save() + + def bind(self, name: str, executor: str) -> None: + self.executors[name] = executor + self._save() def remove_source(self, name: str) -> None: """Close the source's socket. Its runs stay in the host's ledgers.""" self.listeners.pop(name).close() del self.executors[name] + self._save() def sources(self) -> list[dict]: """Every source, by name: the socket it listens on and the executor bound to it.""" @@ -281,7 +308,7 @@ def admin(self, method: str, args: list, kwargs: dict) -> dict: name, executor = args if name not in self.executors: return _refusal("unknown_source", f"no source named {name!r}") - self.executors[name] = executor + self.bind(name, executor) return _ok({"name": name, "executor": executor}) if method == "remove_source": (name,) = args From 38658a02ae24971c49b060f2c9c8b07381545578 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:41:35 +0100 Subject: [PATCH 63/88] test: admin and guest socket modes TDD-Run: 1 TDD-Cycle: 35 TDD-Phase: red --- tests/test_ledger_admin.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_ledger_admin.py b/tests/test_ledger_admin.py index b57c15b..32cb979 100644 --- a/tests/test_ledger_admin.py +++ b/tests/test_ledger_admin.py @@ -6,6 +6,8 @@ from __future__ import annotations +import stat + from conftest import run_cli from tddcli import service, wire @@ -79,3 +81,14 @@ def test_sources_survive_a_service_restart(ledger_service): finally: restarted.stop() assert reply == {"source": "vm-1", "executor": "m"} + + +def test_socket_modes(ledger_service): + guest = ledger_service.dir / "vm-1.sock" + ledger_service.service.add_source("vm-1", guest, executor=None) + + modes = { + "admin": stat.S_IMODE((ledger_service.dir / "admin.sock").stat().st_mode), + "vm-1": stat.S_IMODE(guest.stat().st_mode), + } + assert modes == {"admin": 0o600, "vm-1": 0o660} From 516724fa7e6533ee053dc76b847c5fb7ce8cc949 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:45:05 +0100 Subject: [PATCH 64/88] feat: the service sets its sockets' modes TDD-Run: 1 TDD-Cycle: 35 TDD-Phase: green --- src/tddcli/service.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 913187a..10ee690 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -34,6 +34,9 @@ #: Writes that record who executed a run: the service sets the executor on them. STAMPED = frozenset({"start_run", "abandon_run"}) +ADMIN_MODE = 0o600 +GUEST_MODE = 0o660 + #: Where the service keeps its sources, under its home, so a restart listens on them again. SOURCES_FILE = "sources.json" @@ -236,6 +239,9 @@ def __init__(self, path: Path, service: Service, source: str | None): with contextlib.suppress(FileNotFoundError): path.unlink() super().__init__(str(path), _Handler) + # A socket's mode follows the umask unless set. The admin socket is the service + # account's alone; a guest's is shared with the group the operator forwards it to. + path.chmod(ADMIN_MODE if source is None else GUEST_MODE) self.thread = threading.Thread(target=self.serve_forever, daemon=True) self.thread.start() From 2d80f49c146cb457502ac6728541be9f492fe557 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:45:25 +0100 Subject: [PATCH 65/88] test: ledger serve runs until SIGTERM TDD-Run: 1 TDD-Cycle: 36 TDD-Phase: red --- tests/test_ledger_admin.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_ledger_admin.py b/tests/test_ledger_admin.py index 32cb979..0acd070 100644 --- a/tests/test_ledger_admin.py +++ b/tests/test_ledger_admin.py @@ -6,7 +6,15 @@ from __future__ import annotations +import os +import shutil +import signal import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path from conftest import run_cli from tddcli import service, wire @@ -92,3 +100,32 @@ def test_socket_modes(ledger_service): "vm-1": stat.S_IMODE(guest.stat().st_mode), } assert modes == {"admin": 0o600, "vm-1": 0o660} + + +def test_ledger_serve_runs_until_terminated(tmp_path): + sockets = Path(tempfile.mkdtemp(dir="/tmp", prefix="tdd-")) + cfg = tmp_path / "ledger.toml" + cfg.write_text( + f'[service]\nadmin_socket = "{sockets / "admin.sock"}"\nhome = "{tmp_path / "host"}"\n' + ) + # No `python -m tddcli` entry point: call `main` as the installed script does. + entry = "import sys; from tddcli.cli import main; sys.exit(main(sys.argv[1:]))" + proc = subprocess.Popen( + [sys.executable, "-c", entry, "ledger", "serve"], + env={**os.environ, "TDD_LEDGER_SERVICE_CONFIG": str(cfg)}, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + answered = False + try: + deadline = time.monotonic() + 10 + while not answered and proc.poll() is None and time.monotonic() < deadline: + answered = ping_or_unreachable(sockets / "admin.sock") != "unreachable" + time.sleep(0.1) + proc.send_signal(signal.SIGTERM) + code = proc.wait(timeout=10) + finally: + if proc.poll() is None: + proc.kill() + shutil.rmtree(sockets, ignore_errors=True) + assert (answered, code) == (True, 0) From 522ee33a2a44b6092622f71b1e174c675acb6ced Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:49:05 +0100 Subject: [PATCH 66/88] feat: tdd ledger serve TDD-Run: 1 TDD-Cycle: 36 TDD-Phase: green --- src/tddcli/cli.py | 11 ++++++++++- src/tddcli/service.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 6e2332e..5e8048c 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1658,6 +1658,15 @@ def _not_implemented(args, cfg) -> Envelope: return failure("not implemented", reason="not_implemented") +def _ledger_serve(args, cfg) -> Envelope: + heartbeat(event="ledger_service_listening", admin_socket=str(cfg.admin_socket)) + service_mod.serve(cfg) + return Envelope( + result={"stopped": True}, + next_action=NextAction(Verb.COMPLETE, "The ledger service stopped."), + ) + + def _ledger_sources(args, cfg) -> Envelope: sources = wire.call(cfg.admin_socket, "sources") return Envelope( @@ -1895,7 +1904,7 @@ def build_parser() -> argparse.ArgumentParser: "ledger", help="the host's ledger service: its sources and every source's history" ).add_subparsers(dest="ledger_command", required=True) s = ledger_p.add_parser("serve", help="run the ledger service until SIGTERM") - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_serve)) s = ledger_p.add_parser("sources", help="every source, its socket and its executor") s.set_defaults(fn=_ledger_verb(_ledger_sources)) s = ledger_p.add_parser("add-source", help="open a socket for a new guest") diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 10ee690..6c19e9b 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -16,6 +16,7 @@ import inspect import json import os +import signal import socketserver import threading import tomllib @@ -80,6 +81,25 @@ def load_config(path: Path | None = None) -> ServiceConfig: ) +def serve(config: ServiceConfig) -> Service: + """Run the service until SIGTERM or SIGINT, then close every socket and return it.""" + svc = Service(config) + svc.start() + stopped = threading.Event() + previous = { + sig: signal.signal(sig, lambda *_: stopped.set()) for sig in (signal.SIGTERM, signal.SIGINT) + } + try: + # A timed wait, so the main thread returns to the interpreter to run the handler. + while not stopped.wait(0.5): + pass + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + svc.stop() + return svc + + class _Session: """One client connection, and the ledger it opened. From 4042ca9fb1282728d49b85f6fef822ea1ce235f7 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:49:06 +0100 Subject: [PATCH 67/88] refactor: tidy serve TDD-Run: 1 TDD-Cycle: 36 TDD-Phase: refactor --- src/tddcli/service.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 6c19e9b..c5297ae 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -29,22 +29,24 @@ CONFIG_PATH = Path("/etc/tdd-cli/ledger.toml") CONFIG_ENV = "TDD_LEDGER_SERVICE_CONFIG" -#: A run with one of these outcomes takes no more writes from a guest. -CLOSED_OUTCOMES = ("complete", "abandoned") - -#: Writes that record who executed a run: the service sets the executor on them. -STAMPED = frozenset({"start_run", "abandon_run"}) +#: Where the service keeps its sources, under its home, so a restart listens on them again. +SOURCES_FILE = "sources.json" +#: The admin socket is the service account's alone; a guest's is shared with the group +#: the operator forwards it through. ADMIN_MODE = 0o600 GUEST_MODE = 0o660 -#: Where the service keeps its sources, under its home, so a restart listens on them again. -SOURCES_FILE = "sources.json" +#: A run with one of these outcomes takes no more writes from a guest. +CLOSED_OUTCOMES = ("complete", "abandoned") #: The one write a closed run still takes: `tdd note` after the run is documented use, #: and is how an executor leaves its closing narrative. CLOSED_RUN_EXEMPT = frozenset({"add_note"}) +#: Writes that record who executed a run: the service sets the executor on them. +STAMPED = frozenset({"start_run", "abandon_run"}) + class ServiceConfigError(RuntimeError): pass @@ -259,8 +261,7 @@ def __init__(self, path: Path, service: Service, source: str | None): with contextlib.suppress(FileNotFoundError): path.unlink() super().__init__(str(path), _Handler) - # A socket's mode follows the umask unless set. The admin socket is the service - # account's alone; a guest's is shared with the group the operator forwards it to. + # A socket's mode follows the umask unless it is set. path.chmod(ADMIN_MODE if source is None else GUEST_MODE) self.thread = threading.Thread(target=self.serve_forever, daemon=True) self.thread.start() From 09d6a5256f891274abd99d3476703cd8aaab79f3 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:51:09 +0100 Subject: [PATCH 68/88] test: worker leases are counted across guests TDD-Run: 1 TDD-Cycle: 37 TDD-Phase: red --- tests/test_remote_leases.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/test_remote_leases.py diff --git a/tests/test_remote_leases.py b/tests/test_remote_leases.py new file mode 100644 index 0000000..5dfa8d4 --- /dev/null +++ b/tests/test_remote_leases.py @@ -0,0 +1,39 @@ +"""One worker budget for every guest on a host. + +Each guest's suites share the host's cores, so a socket-mode runner takes its worker +lease from the ledger service, which counts leases across every source. A lease ends +when its connection closes, which is also what a guest that dies does. +""" + +from __future__ import annotations + +import pytest + +from conftest import guest_config +from tddcli import leases + + +@pytest.fixture(autouse=True) +def _twelve_cores(monkeypatch): + # The service runs in this process, so it sees the same budget. + monkeypatch.setenv("TDD_CORE_BUDGET", "12") + + +def as_second_guest(split_guest, ledger_service, monkeypatch, tmp_path) -> None: + """Become guest `vm-2`, which shares no filesystem with `vm-1`: not even leases.""" + socket_path = ledger_service.dir / "vm-2.sock" + if "vm-2" not in ledger_service.service.executors: + ledger_service.service.add_source("vm-2", socket_path, executor=None) + guest_config(split_guest.runner, socket_path) + monkeypatch.setenv("TDD_LEASE_DIR", str(tmp_path / "vm-2-leases")) + + +def test_the_service_counts_leases_across_sources( + split_guest, ledger_service, monkeypatch, tmp_path +): + with leases.worker_lease(): + as_second_guest(split_guest, ledger_service, monkeypatch, tmp_path) + with leases.worker_lease() as workers: + second = workers + + assert second == 6 From 269670518f2c0e28d14e5f2090978a8980526915 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:54:46 +0100 Subject: [PATCH 69/88] feat: a socket-mode runner takes its worker lease from the service TDD-Run: 1 TDD-Cycle: 37 TDD-Phase: green --- src/tddcli/leases.py | 21 +++++++++++++++++++++ src/tddcli/remote.py | 16 ++++++++++++++++ src/tddcli/service.py | 13 ++++++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/tddcli/leases.py b/src/tddcli/leases.py index c52c3ad..9cc3678 100644 --- a/src/tddcli/leases.py +++ b/src/tddcli/leases.py @@ -152,9 +152,30 @@ def named_lease(name: str) -> Iterator[None]: lock_path.unlink() +def _ledger_socket() -> Path | None: + """The ledger service's socket, when this process is a runner in a guest. + + Such a runner shares the host's cores with every other guest's, so the budget is + the service's to split, not a lease directory's on this guest. + """ + from . import runner + + split = runner.load() + if split is None or split.role != "runner": + return None + return split.ledger_socket + + @contextlib.contextmanager def worker_lease(total_cores: int | None = None) -> Iterator[int]: """Hold a lease for one suite invocation; yields the worker count to use.""" + socket_path = _ledger_socket() + if socket_path is not None: + from . import remote + + with remote.worker_lease(socket_path) as workers: + yield workers + return directory = lease_dir() directory.mkdir(parents=True, exist_ok=True) total = total_cores or _total_cores() diff --git a/src/tddcli/remote.py b/src/tddcli/remote.py index 8946c08..713b649 100644 --- a/src/tddcli/remote.py +++ b/src/tddcli/remote.py @@ -8,12 +8,28 @@ from __future__ import annotations +import contextlib +from collections.abc import Iterator from pathlib import Path from . import wire from .ledger import claim_is_stale, judged_claim +@contextlib.contextmanager +def worker_lease(socket_path: Path) -> Iterator[int]: + """A worker lease from the ledger service, held for as long as its connection is. + + There is no release message: the lease ends when the connection closes, which is + also what happens when the guest holding it dies. + """ + conn = wire.Connection(socket_path) + try: + yield conn.request("lease")["workers"] + finally: + conn.close() + + class RemoteLedger: def __init__(self, socket_path: Path, repo_path: Path): self.repo_path = repo_path diff --git a/src/tddcli/service.py b/src/tddcli/service.py index c5297ae..6b6e909 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -23,8 +23,8 @@ from dataclasses import dataclass from pathlib import Path +from . import leases, runner, wire from . import ledger as ledger_mod -from . import runner, wire CONFIG_PATH = Path("/etc/tdd-cli/ledger.toml") CONFIG_ENV = "TDD_LEDGER_SERVICE_CONFIG" @@ -123,6 +123,8 @@ def answer(self, request: dict) -> dict: if self.source is None: # The admin socket: the operator's verbs, never a ledger method. return self.service.admin(method, args, kwargs) + if method == "lease": + return _ok({"workers": self.service.take_lease()}) if method == "open": repo = args[0] if args else None if not _well_formed_repo(repo): @@ -278,6 +280,9 @@ def __init__(self, config: ServiceConfig) -> None: self.config = config self.listeners: dict[str | None, _Listener] = {} self.executors: dict[str, str | None] = {} + # Worker leases held by guests, across every source: one budget per host. + self.lock = threading.Lock() + self.leases = 0 def start(self) -> None: self.config.home.mkdir(parents=True, exist_ok=True) @@ -314,6 +319,12 @@ def bind(self, name: str, executor: str) -> None: self.executors[name] = executor self._save() + def take_lease(self) -> int: + """Count one more suite running on this host; the worker count it should use.""" + with self.lock: + self.leases += 1 + return max(1, leases._total_cores() // self.leases) + def remove_source(self, name: str) -> None: """Close the source's socket. Its runs stay in the host's ledgers.""" self.listeners.pop(name).close() From dc957bb3651ca5f5ff08b24a434f35bb2f9be70f Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:55:04 +0100 Subject: [PATCH 70/88] test: a closed lease connection frees its share TDD-Run: 1 TDD-Cycle: 38 TDD-Phase: red --- tests/test_remote_leases.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_remote_leases.py b/tests/test_remote_leases.py index 5dfa8d4..16c698d 100644 --- a/tests/test_remote_leases.py +++ b/tests/test_remote_leases.py @@ -37,3 +37,14 @@ def test_the_service_counts_leases_across_sources( second = workers assert second == 6 + + +def test_a_lease_ends_when_its_connection_closes( + split_guest, ledger_service, monkeypatch, tmp_path +): + with leases.worker_lease(): + pass + as_second_guest(split_guest, ledger_service, monkeypatch, tmp_path) + + with leases.worker_lease() as workers: + assert workers == 12 From c6d25fe8f1d095753bbf931206db4b5d15b615a0 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:59:31 +0100 Subject: [PATCH 71/88] feat: the service forgets a lease when its connection closes TDD-Run: 1 TDD-Cycle: 38 TDD-Phase: green --- src/tddcli/service.py | 47 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 6b6e909..c4cbcf9 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -17,6 +17,7 @@ import json import os import signal +import socket import socketserver import threading import tomllib @@ -109,9 +110,10 @@ class _Session: connection must not cross threads. """ - def __init__(self, service: Service, source: str | None): + def __init__(self, service: Service, source: str | None, conn: socket.socket | None = None): self.service = service self.source = source + self.conn = conn self.ledger: ledger_mod.Ledger | None = None def answer(self, request: dict) -> dict: @@ -124,7 +126,7 @@ def answer(self, request: dict) -> dict: # The admin socket: the operator's verbs, never a ledger method. return self.service.admin(method, args, kwargs) if method == "lease": - return _ok({"workers": self.service.take_lease()}) + return _ok({"workers": self.service.take_lease(self, self.conn)}) if method == "open": repo = args[0] if args else None if not _well_formed_repo(repo): @@ -215,10 +217,23 @@ def _contract_source(self, contract_id: int) -> str | None: return None if contract is None else contract["source"] def close(self) -> None: + self.service.drop_lease(self) if self.ledger is not None: self.ledger.close() +def _hung_up(conn: socket.socket | None) -> bool: + """Whether the other end of a connection has closed it. Reads nothing.""" + if conn is None: + return False + try: + return conn.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) == b"" + except BlockingIOError: + return False + except OSError: + return True + + def _well_formed_repo(repo) -> bool: """An absolute path with no NUL or newline: it names the host's ledger file.""" return ( @@ -242,7 +257,7 @@ class _Handler(socketserver.StreamRequestHandler): def handle(self) -> None: listener: _Listener = self.server # type: ignore[assignment] - session = _Session(listener.service, listener.source) + session = _Session(listener.service, listener.source, self.request) try: while (request := wire.receive(self.rfile)) is not None: wire.send(self.wfile, session.answer(request)) @@ -280,9 +295,10 @@ def __init__(self, config: ServiceConfig) -> None: self.config = config self.listeners: dict[str | None, _Listener] = {} self.executors: dict[str, str | None] = {} - # Worker leases held by guests, across every source: one budget per host. + # Worker leases held by guests, across every source: one budget per host. A + # lease is its connection, held open for as long as the suite runs. self.lock = threading.Lock() - self.leases = 0 + self.leases: dict[_Session, socket.socket | None] = {} def start(self) -> None: self.config.home.mkdir(parents=True, exist_ok=True) @@ -319,11 +335,24 @@ def bind(self, name: str, executor: str) -> None: self.executors[name] = executor self._save() - def take_lease(self) -> int: - """Count one more suite running on this host; the worker count it should use.""" + def take_lease(self, holder: _Session, conn: socket.socket | None) -> int: + """Count one more suite running on this host; the worker count it should use. + + A lease whose connection has already hung up is forgotten first, so a guest + that just finished never counts against the next, however its handler thread + is scheduled. + """ + with self.lock: + for other, other_conn in list(self.leases.items()): + if _hung_up(other_conn): + del self.leases[other] + self.leases[holder] = conn + return max(1, leases._total_cores() // len(self.leases)) + + def drop_lease(self, holder: _Session) -> None: + """Forget a lease: its connection reached EOF. There is no release message.""" with self.lock: - self.leases += 1 - return max(1, leases._total_cores() // self.leases) + self.leases.pop(holder, None) def remove_source(self, name: str) -> None: """Close the source's socket. Its runs stay in the host's ledgers.""" From 4760a05ce5cbe942ac30d714e94a7a4a6ca79af4 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:59:43 +0100 Subject: [PATCH 72/88] test: stale leases do not throttle the host TDD-Run: 1 TDD-Cycle: 39 TDD-Phase: red --- tests/test_remote_leases.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_remote_leases.py b/tests/test_remote_leases.py index 16c698d..dd5a6b7 100644 --- a/tests/test_remote_leases.py +++ b/tests/test_remote_leases.py @@ -48,3 +48,13 @@ def test_a_lease_ends_when_its_connection_closes( with leases.worker_lease() as workers: assert workers == 12 + + +def test_a_stale_lease_is_not_counted(split_guest, ledger_service, monkeypatch, tmp_path): + with leases.worker_lease(): + monkeypatch.setattr(leases, "STALE_AFTER_S", 0) + as_second_guest(split_guest, ledger_service, monkeypatch, tmp_path) + with leases.worker_lease() as workers: + second = workers + + assert second == 12 From 7f7fe984ba7db8ce88408aaacdcabf2d7c6ee791 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:03:05 +0100 Subject: [PATCH 73/88] feat: the service ignores leases past the stale limit TDD-Run: 1 TDD-Cycle: 39 TDD-Phase: green --- src/tddcli/service.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index c4cbcf9..c67ceff 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -20,6 +20,7 @@ import socket import socketserver import threading +import time import tomllib from dataclasses import dataclass from pathlib import Path @@ -298,7 +299,7 @@ def __init__(self, config: ServiceConfig) -> None: # Worker leases held by guests, across every source: one budget per host. A # lease is its connection, held open for as long as the suite runs. self.lock = threading.Lock() - self.leases: dict[_Session, socket.socket | None] = {} + self.leases: dict[_Session, tuple[socket.socket | None, float]] = {} def start(self) -> None: self.config.home.mkdir(parents=True, exist_ok=True) @@ -340,14 +341,19 @@ def take_lease(self, holder: _Session, conn: socket.socket | None) -> int: A lease whose connection has already hung up is forgotten first, so a guest that just finished never counts against the next, however its handler thread - is scheduled. + is scheduled. One older than `leases.STALE_AFTER_S` is not counted: no suite + runs that long, and a guest whose VM froze must not throttle the host forever. """ + now = time.monotonic() with self.lock: - for other, other_conn in list(self.leases.items()): + for other, (other_conn, _) in list(self.leases.items()): if _hung_up(other_conn): del self.leases[other] - self.leases[holder] = conn - return max(1, leases._total_cores() // len(self.leases)) + live = sum( + 1 for _, taken in self.leases.values() if now - taken <= leases.STALE_AFTER_S + ) + self.leases[holder] = (conn, now) + return max(1, leases._total_cores() // (live + 1)) def drop_lease(self, holder: _Session) -> None: """Forget a lease: its connection reached EOF. There is no release message.""" From eb13b0cb58584a71b9c04fe8a3c9327dc2d73853 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:03:19 +0100 Subject: [PATCH 74/88] test: guest fleet shows host-wide suites TDD-Run: 1 TDD-Cycle: 40 TDD-Phase: red --- tests/test_remote_leases.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_remote_leases.py b/tests/test_remote_leases.py index dd5a6b7..542de45 100644 --- a/tests/test_remote_leases.py +++ b/tests/test_remote_leases.py @@ -9,7 +9,7 @@ import pytest -from conftest import guest_config +from conftest import guest_config, run_cli from tddcli import leases @@ -58,3 +58,15 @@ def test_a_stale_lease_is_not_counted(split_guest, ledger_service, monkeypatch, second = workers assert second == 12 + + +def test_guest_fleet_reports_the_hosts_lease_count( + repo, split_guest, ledger_service, monkeypatch, tmp_path +): + as_second_guest(split_guest, ledger_service, monkeypatch, tmp_path) + with leases.worker_lease(): + guest_config(split_guest.runner, split_guest.socket) + monkeypatch.setenv("TDD_LEASE_DIR", str(tmp_path / "vm-1-leases")) + out = run_cli(repo, "fleet", "--json") + + assert out["result"]["suites"]["active"] == 1 From 4b373ec41f9fd8830bdf8ffbbb129a02e1aa25c8 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:06:58 +0100 Subject: [PATCH 75/88] feat: a socket-mode lease snapshot comes from the service TDD-Run: 1 TDD-Cycle: 40 TDD-Phase: green --- src/tddcli/leases.py | 8 +++++++- src/tddcli/remote.py | 5 +++++ src/tddcli/service.py | 23 +++++++++++++++++------ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/tddcli/leases.py b/src/tddcli/leases.py index 9cc3678..4acce56 100644 --- a/src/tddcli/leases.py +++ b/src/tddcli/leases.py @@ -92,7 +92,13 @@ def _live_count(directory: Path) -> int: def snapshot() -> dict: """Observe the budget without participating in it — counts live leases but - sweeps nothing and takes nothing, so a fleet view never perturbs the split.""" + sweeps nothing and takes nothing, so a fleet view never perturbs the split. + In a guest, the budget is the host's: the ledger service counts it.""" + socket_path = _ledger_socket() + if socket_path is not None: + from . import remote + + return remote.lease_snapshot(socket_path) directory = lease_dir() total = _total_cores() live = 0 diff --git a/src/tddcli/remote.py b/src/tddcli/remote.py index 713b649..fc9ffa7 100644 --- a/src/tddcli/remote.py +++ b/src/tddcli/remote.py @@ -30,6 +30,11 @@ def worker_lease(socket_path: Path) -> Iterator[int]: conn.close() +def lease_snapshot(socket_path: Path) -> dict: + """The host's worker budget, as the ledger service counts it across every guest.""" + return wire.call(socket_path, "lease_snapshot") + + class RemoteLedger: def __init__(self, socket_path: Path, repo_path: Path): self.repo_path = repo_path diff --git a/src/tddcli/service.py b/src/tddcli/service.py index c67ceff..fc02612 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -128,6 +128,8 @@ def answer(self, request: dict) -> dict: return self.service.admin(method, args, kwargs) if method == "lease": return _ok({"workers": self.service.take_lease(self, self.conn)}) + if method == "lease_snapshot": + return _ok(self.service.lease_snapshot()) if method == "open": repo = args[0] if args else None if not _well_formed_repo(repo): @@ -346,15 +348,24 @@ def take_lease(self, holder: _Session, conn: socket.socket | None) -> int: """ now = time.monotonic() with self.lock: - for other, (other_conn, _) in list(self.leases.items()): - if _hung_up(other_conn): - del self.leases[other] - live = sum( - 1 for _, taken in self.leases.values() if now - taken <= leases.STALE_AFTER_S - ) + live = self._live_leases(now) self.leases[holder] = (conn, now) return max(1, leases._total_cores() // (live + 1)) + def _live_leases(self, now: float) -> int: + """Leases still held and not past the stale limit. Call with the lock held.""" + for other, (other_conn, _) in list(self.leases.items()): + if _hung_up(other_conn): + del self.leases[other] + return sum(1 for _, taken in self.leases.values() if now - taken <= leases.STALE_AFTER_S) + + def lease_snapshot(self) -> dict: + """The host's budget, observed without taking a lease: `leases.snapshot`'s shape.""" + total = leases._total_cores() + with self.lock: + live = self._live_leases(time.monotonic()) + return {"active": live, "total_cores": total, "workers_each": max(1, total // max(1, live))} + def drop_lease(self, holder: _Session) -> None: """Forget a lease: its connection reached EOF. There is no release message.""" with self.lock: From f4a2d90cf31f8360b20e5b053e60a853cfa7ef3c Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:07:23 +0100 Subject: [PATCH 76/88] test: host metrics cover every source TDD-Run: 1 TDD-Cycle: 41 TDD-Phase: red --- tests/test_ledger_host_views.py | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_ledger_host_views.py diff --git a/tests/test_ledger_host_views.py b/tests/test_ledger_host_views.py new file mode 100644 index 0000000..a3963e3 --- /dev/null +++ b/tests/test_ledger_host_views.py @@ -0,0 +1,50 @@ +"""One history per host: the operator reads every source's runs in one place. + +Guests only ever see their own source. On the host, `tdd ledger metrics|fleet|log` +ask the ledger service, which reads every repository's ledger across all sources. +""" + +from __future__ import annotations + +import pytest + +from conftest import guest_config, run_cli, write_plan +from tddcli import gitutil +from tddcli import ledger as ledger_mod + +MINIMAL_PLAN = """\ +--- +cycles: + - n: 1 + project: backend + title: "placeholder" + test: "tests/test_smoke.py::test_smoke" + commit_red: "test: placeholder" + commit_green: "feat: placeholder" +--- +# Minimal plan for host view tests +""" + + +@pytest.fixture +def two_sources(repo, split_guest, ledger_service): + """A run as `vm-1`, then one in the same repository as `vm-2`: {source: run id}.""" + ledger_service.service.add_source("vm-2", ledger_service.dir / "vm-2.sock", executor=None) + plan = write_plan(repo, MINIMAL_PLAN) + runs = {} + for source in ("vm-1", "vm-2"): + guest_config(split_guest.runner, ledger_service.dir / f"{source}.sock") + run_cli(repo, "plan", "register", plan) + runs[source] = run_cli(repo, "run", "start", "--plan", plan)["run"]["id"] + return runs + + +def slug(repo) -> str: + return ledger_mod.slug(gitutil.repo_identity(repo)) + + +def test_ledger_metrics_covers_every_source(repo, two_sources): + out = run_cli(repo, "ledger", "metrics") + + runs = (out["result"].get("repos") or {}).get(slug(repo), {}).get("runs", []) + assert sorted(r["source"] for r in runs) == ["vm-1", "vm-2"] From 163e5c648d4bbcddc5c4518c451d1aa900f02bfe Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:11:07 +0100 Subject: [PATCH 77/88] feat: tdd ledger metrics TDD-Run: 1 TDD-Cycle: 41 TDD-Phase: green --- src/tddcli/cli.py | 9 ++++++++- src/tddcli/ledger.py | 5 ++++- src/tddcli/render.py | 3 ++- src/tddcli/service.py | 20 +++++++++++++++++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 5e8048c..3ea6c08 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1667,6 +1667,13 @@ def _ledger_serve(args, cfg) -> Envelope: ) +def _ledger_metrics(args, cfg) -> Envelope: + return Envelope( + result=wire.call(cfg.admin_socket, "metrics", args.repo), + next_action=NextAction(Verb.COMPLETE, "Metrics computed over every source."), + ) + + def _ledger_sources(args, cfg) -> Envelope: sources = wire.call(cfg.admin_socket, "sources") return Envelope( @@ -1923,7 +1930,7 @@ def build_parser() -> argparse.ArgumentParser: s.set_defaults(fn=_ledger_verb(_not_implemented)) s = ledger_p.add_parser("metrics", help="metrics over every source's runs") s.add_argument("--repo", help="one repository's ledger, by its slug") - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_metrics)) s = ledger_p.add_parser("log", help="render any source's run as a friction log") s.add_argument("--repo", required=True, help="the repository's slug") s.add_argument("--run", type=int, required=True) diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 0850be1..22af7e8 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -1466,8 +1466,11 @@ def advance_claims(self) -> list[sqlite3.Row]: # -- projections: the friction log, progress and metrics ------------------- - def runs_in(self, worktree: str) -> list[sqlite3.Row]: + def runs_in(self, worktree: str | None) -> list[sqlite3.Row]: + """The runs recorded in one worktree, or in every worktree when it is None.""" scope, extra = self._scoped() + if worktree is None: + return self.all(f"SELECT * FROM run WHERE 1 = 1{scope} ORDER BY id", extra) return self.all( f"SELECT * FROM run WHERE worktree_path = ?{scope} ORDER BY id", (worktree, *extra) ) diff --git a/src/tddcli/render.py b/src/tddcli/render.py index e384d0d..18a77bd 100644 --- a/src/tddcli/render.py +++ b/src/tddcli/render.py @@ -335,7 +335,8 @@ def _impl_attempts(rows) -> int: return len(target_only) if target_only else len(rows) -def metrics(ledger: Ledger, worktree: str) -> dict: +def metrics(ledger: Ledger, worktree: str | None) -> dict: + """Per-run metrics for one worktree's runs, or every worktree's when it is None.""" runs = ledger.runs_in(worktree) out = { "runs": [], diff --git a/src/tddcli/service.py b/src/tddcli/service.py index fc02612..cb3b2fa 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -25,7 +25,7 @@ from dataclasses import dataclass from pathlib import Path -from . import leases, runner, wire +from . import leases, render, runner, wire from . import ledger as ledger_mod CONFIG_PATH = Path("/etc/tdd-cli/ledger.toml") @@ -384,8 +384,26 @@ def sources(self) -> list[dict]: for name, executor in sorted(self.executors.items()) ] + def host_files(self, repo: str | None = None) -> dict[str, Path]: + """Every repository's ledger on this host, by slug; only `repo`'s when named.""" + files = {p.stem: p for p in sorted(self.config.home.glob("*.sqlite3"))} + return files if repo is None else {k: v for k, v in files.items() if k == repo} + + def metrics(self, repo: str | None = None) -> dict: + """`tdd metrics`, over every source's runs, for each repository on the host.""" + out = {} + for slug, path in self.host_files(repo).items(): + view = ledger_mod.Ledger(Path(slug), path=path, source=None) + try: + out[slug] = render.metrics(view, None) + finally: + view.close() + return {"repos": out} + def admin(self, method: str, args: list, kwargs: dict) -> dict: """An operator's request, from the admin socket.""" + if method == "metrics": + return _ok(self.metrics(*args)) if method == "sources": return _ok(self.sources()) if method == "bind": From 737992113d109ece95b7686d327b011a8e6ed2af Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:11:22 +0100 Subject: [PATCH 78/88] test: host fleet covers every source TDD-Run: 1 TDD-Cycle: 42 TDD-Phase: red --- tests/test_ledger_host_views.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_ledger_host_views.py b/tests/test_ledger_host_views.py index a3963e3..dff2a3a 100644 --- a/tests/test_ledger_host_views.py +++ b/tests/test_ledger_host_views.py @@ -48,3 +48,10 @@ def test_ledger_metrics_covers_every_source(repo, two_sources): runs = (out["result"].get("repos") or {}).get(slug(repo), {}).get("runs", []) assert sorted(r["source"] for r in runs) == ["vm-1", "vm-2"] + + +def test_ledger_fleet_lists_active_runs_of_every_source(repo, two_sources): + out = run_cli(repo, "ledger", "fleet") + + runs = out["result"].get("runs") or [] + assert sorted((r["source"], r["run_id"]) for r in runs) == sorted(two_sources.items()) From 7eb43838329f2a88bc8674cbb1e9e85c278a9065 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:15:14 +0100 Subject: [PATCH 79/88] feat: tdd ledger fleet TDD-Run: 1 TDD-Cycle: 42 TDD-Phase: green --- src/tddcli/cli.py | 9 ++++++++- src/tddcli/fleet.py | 17 +++++++++++------ src/tddcli/service.py | 19 ++++++++++++++++++- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 3ea6c08..9998b33 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1667,6 +1667,13 @@ def _ledger_serve(args, cfg) -> Envelope: ) +def _ledger_fleet(args, cfg) -> Envelope: + return Envelope( + result=wire.call(cfg.admin_socket, "fleet"), + next_action=NextAction(Verb.COMPLETE, "Fleet reported across every source."), + ) + + def _ledger_metrics(args, cfg) -> Envelope: return Envelope( result=wire.call(cfg.admin_socket, "metrics", args.repo), @@ -1927,7 +1934,7 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--executor", required=True) s.set_defaults(fn=_ledger_verb(_ledger_bind)) s = ledger_p.add_parser("fleet", help="active runs of every source") - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_fleet)) s = ledger_p.add_parser("metrics", help="metrics over every source's runs") s.add_argument("--repo", help="one repository's ledger, by its slug") s.set_defaults(fn=_ledger_verb(_ledger_metrics)) diff --git a/src/tddcli/fleet.py b/src/tddcli/fleet.py index 2246729..f3063bd 100644 --- a/src/tddcli/fleet.py +++ b/src/tddcli/fleet.py @@ -34,6 +34,7 @@ def _runs(reader: Ledger) -> list[dict]: out.append( { "run_id": row["id"], + "source": row["source"], "worktree": row["worktree_path"], "plan": row["plan_path"], "executor": row["executor_model"], @@ -81,17 +82,21 @@ def _advance_claims(reader: Ledger) -> list[dict]: ] +def entries(reader: Ledger) -> dict: + """One ledger's active runs, baselines being collected and advances in flight.""" + return { + "runs": _runs(reader), + "collecting": _claims(reader), + "advancing": _advance_claims(reader), + } + + def summarise(reader: Ledger | None) -> dict: """`reader` is from `ledger.open_readonly`: None when no ledger exists yet.""" if reader is None: return {"runs": [], "collecting": [], "advancing": [], "suites": leases.snapshot()} try: - return { - "runs": _runs(reader), - "collecting": _claims(reader), - "advancing": _advance_claims(reader), - "suites": leases.snapshot(), - } + return {**entries(reader), "suites": leases.snapshot()} finally: reader.close() diff --git a/src/tddcli/service.py b/src/tddcli/service.py index cb3b2fa..cb0ec65 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -25,7 +25,7 @@ from dataclasses import dataclass from pathlib import Path -from . import leases, render, runner, wire +from . import fleet, leases, render, runner, wire from . import ledger as ledger_mod CONFIG_PATH = Path("/etc/tdd-cli/ledger.toml") @@ -389,6 +389,21 @@ def host_files(self, repo: str | None = None) -> dict[str, Path]: files = {p.stem: p for p in sorted(self.config.home.glob("*.sqlite3"))} return files if repo is None else {k: v for k, v in files.items() if k == repo} + def fleet(self) -> dict: + """`tdd fleet` across the host: every repository, every source, one lease budget.""" + out: dict = {"runs": [], "collecting": [], "advancing": []} + for slug, path in self.host_files().items(): + reader = ledger_mod.open_readonly(path, source=None) + if reader is None: + continue + try: + for key, rows in fleet.entries(reader).items(): + out[key].extend({**row, "repo": slug} for row in rows) + finally: + reader.close() + out["suites"] = self.lease_snapshot() + return out + def metrics(self, repo: str | None = None) -> dict: """`tdd metrics`, over every source's runs, for each repository on the host.""" out = {} @@ -404,6 +419,8 @@ def admin(self, method: str, args: list, kwargs: dict) -> dict: """An operator's request, from the admin socket.""" if method == "metrics": return _ok(self.metrics(*args)) + if method == "fleet": + return _ok(self.fleet()) if method == "sources": return _ok(self.sources()) if method == "bind": From 11e8e19a8e7f7e608c63150bfb51897fdb8180d5 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:15:34 +0100 Subject: [PATCH 80/88] test: host log renders any source's run TDD-Run: 1 TDD-Cycle: 43 TDD-Phase: red --- tests/test_ledger_host_views.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_ledger_host_views.py b/tests/test_ledger_host_views.py index dff2a3a..99a8941 100644 --- a/tests/test_ledger_host_views.py +++ b/tests/test_ledger_host_views.py @@ -8,7 +8,7 @@ import pytest -from conftest import guest_config, run_cli, write_plan +from conftest import guest_config, run_cli, run_cli_text, write_plan from tddcli import gitutil from tddcli import ledger as ledger_mod @@ -55,3 +55,11 @@ def test_ledger_fleet_lists_active_runs_of_every_source(repo, two_sources): runs = out["result"].get("runs") or [] assert sorted((r["source"], r["run_id"]) for r in runs) == sorted(two_sources.items()) + + +def test_ledger_log_renders_any_sources_run(repo, two_sources): + run_id = two_sources["vm-2"] + + text = run_cli_text(repo, "ledger", "log", "--repo", slug(repo), "--run", str(run_id)) + # `render.friction_log` heads a run's log with its plan, then `- Run: `. + assert ("tasks/plan.md" in text, f"- Run: {run_id}\n" in text) == (True, True) From 62fe48172b807c0be9369c894d2485bf693509ac Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:19:18 +0100 Subject: [PATCH 81/88] feat: tdd ledger log TDD-Run: 1 TDD-Cycle: 43 TDD-Phase: green --- src/tddcli/cli.py | 19 ++++++++++++++++++- src/tddcli/service.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 9998b33..3d79d22 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1674,6 +1674,23 @@ def _ledger_fleet(args, cfg) -> Envelope: ) +def _ledger_log(args, cfg) -> Envelope: + rendered = wire.call(cfg.admin_socket, "log", args.repo, args.run) + if args.out: + out = Path(args.out) + actor.current().write_file(out, rendered["text"].encode()) + return Envelope( + result={"written": str(out), "source": rendered["source"]}, + next_action=NextAction(Verb.COMPLETE, f"Friction log written to {out}."), + ) + sys.stdout.write(rendered["text"]) + return Envelope( + result={"rendered": True, "source": rendered["source"]}, + next_action=NextAction(Verb.COMPLETE, "Rendered."), + silent=True, + ) + + def _ledger_metrics(args, cfg) -> Envelope: return Envelope( result=wire.call(cfg.admin_socket, "metrics", args.repo), @@ -1942,7 +1959,7 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--repo", required=True, help="the repository's slug") s.add_argument("--run", type=int, required=True) s.add_argument("--out") - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_log)) s = ledger_p.add_parser("import", help="bring an existing ledger in as one source") s.add_argument("file") s.add_argument("--source", required=True) diff --git a/src/tddcli/service.py b/src/tddcli/service.py index cb0ec65..8c0a7fc 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -404,6 +404,20 @@ def fleet(self) -> dict: out["suites"] = self.lease_snapshot() return out + def log(self, repo: str, run_id: int) -> dict: + """Any source's run, rendered as the friction log its guest would render.""" + path = self.host_files(repo).get(repo) + if path is None: + return _refusal("unknown_repo", f"no ledger for {repo!r} on this host") + view = ledger_mod.Ledger(Path(repo), path=path, source=None) + try: + run = view.run(run_id) + if run is None: + return _refusal("unknown_run", f"no run {run_id} in {repo}'s ledger") + return _ok({"text": render.friction_log(view, run), "source": run["source"]}) + finally: + view.close() + def metrics(self, repo: str | None = None) -> dict: """`tdd metrics`, over every source's runs, for each repository on the host.""" out = {} @@ -421,6 +435,8 @@ def admin(self, method: str, args: list, kwargs: dict) -> dict: return _ok(self.metrics(*args)) if method == "fleet": return _ok(self.fleet()) + if method == "log": + return self.log(*args) if method == "sources": return _ok(self.sources()) if method == "bind": From 50c66ac1f2466ed70eca9578823654a85c22463a Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:19:33 +0100 Subject: [PATCH 82/88] test: ledger import retags an existing ledger TDD-Run: 1 TDD-Cycle: 44 TDD-Phase: red --- tests/test_ledger_host_views.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_ledger_host_views.py b/tests/test_ledger_host_views.py index 99a8941..86f41f4 100644 --- a/tests/test_ledger_host_views.py +++ b/tests/test_ledger_host_views.py @@ -63,3 +63,20 @@ def test_ledger_log_renders_any_sources_run(repo, two_sources): text = run_cli_text(repo, "ledger", "log", "--repo", slug(repo), "--run", str(run_id)) # `render.friction_log` heads a run's log with its plan, then `- Run: `. assert ("tasks/plan.md" in text, f"- Run: {run_id}\n" in text) == (True, True) + + +def single_user_ledger(repo, ledger_home): + """A ledger with one run, from a machine with no ledger service: all of it `local`.""" + plan = write_plan(repo, MINIMAL_PLAN) + run_cli(repo, "plan", "register", plan) + run_cli(repo, "run", "start", "--plan", plan) + (path,) = ledger_home.glob("*.sqlite3") + return path + + +def test_ledger_import_retags_a_ledger_as_one_source(repo, ledger_home, ledger_service): + legacy = single_user_ledger(repo, ledger_home) + + run_cli(repo, "ledger", "import", str(legacy), "--source", "legacy") + host = ledger_mod.open_readonly(ledger_service.home / legacy.name, source=None) + assert host is not None and [r["source"] for r in host.runs_in(None)] == ["legacy"] From e5c94e2486d2252ab4ca12026dd2037cb50967ee Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:23:32 +0100 Subject: [PATCH 83/88] feat: tdd ledger import TDD-Run: 1 TDD-Cycle: 44 TDD-Phase: green --- src/tddcli/cli.py | 18 +++++++++++++++- src/tddcli/ledger.py | 49 +++++++++++++++++++++++++++++++++++++------ src/tddcli/service.py | 9 ++++++++ 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 3d79d22..5c07076 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1691,6 +1691,22 @@ def _ledger_log(args, cfg) -> Envelope: ) +def _ledger_import(args, cfg) -> Envelope: + source = Path(args.file).resolve() + if not source.is_file(): + return failure(f"{source} is not a file") + imported = wire.call(cfg.admin_socket, "import", str(source), args.source) + return Envelope( + result=imported, + next_action=NextAction( + Verb.COMPLETE, + f"Imported to {imported['imported']} as source {args.source}. Runs up to" + f" {imported['pre_split_import']['last_run_id']} were recorded where the agent" + " could reach the ledger, and are marked as such.", + ), + ) + + def _ledger_metrics(args, cfg) -> Envelope: return Envelope( result=wire.call(cfg.admin_socket, "metrics", args.repo), @@ -1963,7 +1979,7 @@ def build_parser() -> argparse.ArgumentParser: s = ledger_p.add_parser("import", help="bring an existing ledger in as one source") s.add_argument("file") s.add_argument("--source", required=True) - s.set_defaults(fn=_ledger_verb(_not_implemented)) + s.set_defaults(fn=_ledger_verb(_ledger_import)) return p diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index 22af7e8..c5e54b9 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -568,6 +568,18 @@ def import_legacy(source: Path) -> tuple[Path, dict]: Returns where it landed and the marker written into it. """ target = _ensured_home() / source.name + _copy(source, target) + imported = Ledger(target.parent, path=target) # opening it migrates it forward + # MAX over no rows is one row holding NULL: an empty ledger imports with `None`. + last = imported.db.execute("SELECT MAX(id) FROM run").fetchone()[0] + marker = {"source": str(source), "imported_at": now(), "last_run_id": last} + imported.set_meta(PRE_SPLIT_IMPORT, json.dumps(marker)) + return target, marker + + +def _copy(source: Path, target: Path) -> None: + """Copy a ledger with SQLite's backup API: a plain copy of a WAL-mode ledger's main + file would drop the newest rows, which are still in its sidecar.""" src = sqlite3.connect(f"file:{source}?mode=ro", uri=True) dst = sqlite3.connect(target) try: @@ -575,12 +587,37 @@ def import_legacy(source: Path) -> tuple[Path, dict]: finally: src.close() dst.close() - imported = Ledger(target.parent, path=target) # opening it migrates it forward - # MAX over no rows is one row holding NULL: an empty ledger imports with `None`. - last = imported.db.execute("SELECT MAX(id) FROM run").fetchone()[0] - marker = {"source": str(source), "imported_at": now(), "last_run_id": last} - imported.set_meta(PRE_SPLIT_IMPORT, json.dumps(marker)) - return target, marker + + +def import_as_source(source: Path, target: Path, name: str) -> dict: + """Bring an existing ledger onto a ledger service's host as one named source. + + Every run and contract it recorded as `local` becomes `name`'s. Its claims are + dropped: they belonged to processes on another machine. It is marked as history + recorded where the agent could reach it, as `import_legacy` marks one. There is + no merge: the target is a new file. + """ + _copy(source, target) + imported = Ledger(target.parent, path=target, source=None) # opening it migrates it + try: + with imported.db: + for table in ("run", "plan_contract"): + imported.db.execute( + f"UPDATE {table} SET source = ? WHERE source = ?", (name, LOCAL_SOURCE) + ) + imported.db.execute("DELETE FROM baseline_claim") + imported.db.execute("DELETE FROM advance_claim") + last = imported.db.execute("SELECT MAX(id) FROM run").fetchone()[0] + marker = { + "source": str(source), + "as_source": name, + "imported_at": now(), + "last_run_id": last, + } + imported.set_meta(PRE_SPLIT_IMPORT, json.dumps(marker)) + finally: + imported.close() + return marker def claim_is_stale(hostname: str, pid: int, started_at: str) -> bool: diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 8c0a7fc..bd725df 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -418,6 +418,13 @@ def log(self, repo: str, run_id: int) -> dict: finally: view.close() + def import_ledger(self, file: str, name: str) -> dict: + """An existing ledger, brought onto this host as the source `name`.""" + source = Path(file) + target = self.config.home / source.name + marker = ledger_mod.import_as_source(source, target, name) + return _ok({"imported": str(target), "pre_split_import": marker}) + def metrics(self, repo: str | None = None) -> dict: """`tdd metrics`, over every source's runs, for each repository on the host.""" out = {} @@ -437,6 +444,8 @@ def admin(self, method: str, args: list, kwargs: dict) -> dict: return _ok(self.fleet()) if method == "log": return self.log(*args) + if method == "import": + return self.import_ledger(*args) if method == "sources": return _ok(self.sources()) if method == "bind": From 5b637e2c53129b23dedbc31acf2f62e6507b71fd Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:23:53 +0100 Subject: [PATCH 84/88] test: ledger import never overwrites TDD-Run: 1 TDD-Cycle: 45 TDD-Phase: red --- tests/test_ledger_host_views.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_ledger_host_views.py b/tests/test_ledger_host_views.py index 86f41f4..4b7fb43 100644 --- a/tests/test_ledger_host_views.py +++ b/tests/test_ledger_host_views.py @@ -80,3 +80,11 @@ def test_ledger_import_retags_a_ledger_as_one_source(repo, ledger_home, ledger_s run_cli(repo, "ledger", "import", str(legacy), "--source", "legacy") host = ledger_mod.open_readonly(ledger_service.home / legacy.name, source=None) assert host is not None and [r["source"] for r in host.runs_in(None)] == ["legacy"] + + +def test_ledger_import_refuses_an_existing_host_ledger(repo, ledger_home, ledger_service): + legacy = single_user_ledger(repo, ledger_home) + run_cli(repo, "ledger", "import", str(legacy), "--source", "legacy") + + again = run_cli(repo, "ledger", "import", str(legacy), "--source", "legacy") + assert again["result"].get("reason") == "ledger_exists" From 9070d6edb69c034124ca64678b21647ca141ffd9 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:27:28 +0100 Subject: [PATCH 85/88] feat: ledger import refuses an existing host ledger TDD-Run: 1 TDD-Cycle: 45 TDD-Phase: green --- src/tddcli/cli.py | 6 +++++- src/tddcli/service.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 5c07076..29ed26f 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1649,7 +1649,11 @@ def handler(args) -> Envelope: " `tdd docs split` describes one", reason="no_service", ) - return answer(args, service_mod.load_config(path)) + try: + return answer(args, service_mod.load_config(path)) + except wire.LedgerRefused as exc: + # The operator's own service said no: its code is the reason. + return failure(exc.error, reason=exc.refusal) return handler diff --git a/src/tddcli/service.py b/src/tddcli/service.py index bd725df..954d726 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -422,6 +422,13 @@ def import_ledger(self, file: str, name: str) -> dict: """An existing ledger, brought onto this host as the source `name`.""" source = Path(file) target = self.config.home / source.name + if target.exists(): + # No merge: the host's copy may hold runs no guest could reach, and an import + # over it would trade them for ones recorded where an agent could. + return _refusal( + "ledger_exists", + f"the host already holds {target}; importing again would overwrite it", + ) marker = ledger_mod.import_as_source(source, target, name) return _ok({"imported": str(target), "pre_split_import": marker}) From 65e822235795ca98a6a72b01f97aaa454c20ba9a Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:30:05 +0100 Subject: [PATCH 86/88] docs: split mode across a VM boundary TDD-Run: 1 TDD-Cycle: 46 TDD-Phase: refactor --- CHANGELOG.md | 36 ++++++++-- README.md | 6 +- docs/split-runner.md | 163 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 196 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df42913..f47205d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,38 @@ and the project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- **Split mode across a VM boundary: a ledger service on the host** (#169). A runner + configured with `ledger_socket` (instead of `ledger_home`) keeps no ledger: it reads and + writes the host's through a unix socket served by `tdd ledger serve`, which owns the SQLite + files and is their only writer. Each run records the source of the socket it arrived on. + The service refuses a guest another source's runs, cycles and contracts (`foreign_run`), + writes to its own complete or abandoned runs (`run_closed`, except `tdd note`), host-only + methods (`method_not_allowed`) and malformed repository paths (`bad_repo`). An unreachable + socket fails the verb with `reason: "ledger_unreachable"`; nothing falls back to a local + ledger. `tdd doctor` in a guest names the source it reaches. +- **Operator verbs, `tdd ledger …`** (#169): `serve`, `add-source`, `remove-source`, `bind` + (the executor a source's runs record, as `executor_source: "operator"`; an unbound guest's + `operator` claim is recorded as `claimed`), `sources`, and host-wide `metrics`, `fleet` and + `log` across every source. `tdd ledger import --source ` brings an existing + ledger in as one source. `tdd runner import` on a socket-mode runner is refused with + `reason: "ledger_remote"`. +- **Host-wide worker leases** (#169). A socket-mode runner takes its worker lease from the + ledger service, so one core budget covers every guest; a lease ends when its connection + closes, and `tdd fleet` in a guest reports the host's count. +- **`per_file_collects` in the `baseline_captured` heartbeat** (#168): the number of files that + fell to the per-file loop, one runner start each, next to `collect_s`. Reported by serial and + `--baseline-jobs` probes alike. + ### Changed +- **Ledger schema v13: every run has a source** (#169). `run` and `plan_contract` gain a + `source` column, and the baseline/advance claims and the baseline cache are rebuilt keyed + by source, so two guests at the same path neither block each other nor share a cached + baseline. Existing rows migrate as `local`. Every ledger access now goes through named + `Ledger` methods; `fleet` reads through `ledger.open_readonly`. `tdd metrics` reports each + run's `source`. - **A successful batch listing is authoritative for its suite's files** (#168). `collect()` started the runner once more for every `test_paths` file a successful whole-suite listing did not mention — another project's test file on a shared root, a fixture, a helper — and @@ -19,12 +49,6 @@ and the project adheres to [Semantic Versioning](https://semver.org/). `(command, env)` pairs keeps the old loop, and can opt in with a third element naming the suite (`None` for the default, else the override). -### Added - -- **`per_file_collects` in the `baseline_captured` heartbeat** (#168): the number of files that - fell to the per-file loop, one runner start each, next to `collect_s`. Reported by serial and - `--baseline-jobs` probes alike. - ## [0.14.1] - 2026-09-25 ### Fixed diff --git a/README.md b/README.md index 5e82701..66bf735 100644 --- a/README.md +++ b/README.md @@ -689,7 +689,11 @@ cannot become: - executor identity is `operator` when the runner's config assigns the calling account a model, and `claimed` otherwise; - `tdd runner import ` brings an existing ledger under the runner, marked as - pre-split history that `tdd metrics` reports. + pre-split history that `tdd metrics` reports; +- across a VM boundary, the runner in each guest keeps no ledger at all: it reads and writes + the host's through a socket served by `tdd ledger serve`, which records every run under the + source of the socket it came in on, refuses a guest anything but its own open runs, and + gives the operator one history and one worker budget for every guest. `tdd docs split` ([docs/split-runner.md](docs/split-runner.md)) has the setup, the verification checklist, and what split mode does not protect against. diff --git a/docs/split-runner.md b/docs/split-runner.md index 8965e71..362c559 100644 --- a/docs/split-runner.md +++ b/docs/split-runner.md @@ -183,5 +183,164 @@ Afterwards, remove the agent's copy so that nothing reads it by mistake. - `tdd fleet`, `tdd metrics` and `tdd log render` are forwarded like everything else and read the runner's ledger. - A runner invoked directly, with no sudo in between, acts for nobody: every verb - except `runner import` is refused with `reason: "no_agent"`. It never falls back to - running suites as itself. + except `runner import` and `ledger` is refused with `reason: "no_agent"`. It never + falls back to running suites as itself. + +## Across a VM boundary + +Everything above is one machine. Once agents run in disposable VMs or containers, the +runner has to be in the guest, because that is where the suites run. Its ledger must not +be: it would be deleted with the guest, two guests' files could never be merged, and an +agent with root in the guest (a `docker` group membership is enough) could rewrite it +whatever the file modes say. + +So the runner in the guest keeps **no ledger**. It reads and writes the host's through a +unix socket, served by a **ledger service** on the host. The service owns the SQLite +files and is their only writer. The runner still runs every suite, gate and git command +as the agent, in the agent's worktree, exactly as above. + + host guest (one per run) + ┌──────────────────────────────┐ ┌──────────────────────────────┐ + │ tdd ledger serve │ │ agent ─sudo─▶ runner │ + │ admin.sock (operator) │ │ ledger_socket │ + │ vm-1.sock ───forwarded──────────────────▶│ /run/tdd-cli/ledger.sock │ + │ /.sqlite3 │ │ (no ledger file here) │ + └──────────────────────────────┘ └──────────────────────────────┘ + +A run's **source** is the socket it arrived on. The operator names it when they create +the socket, and the protocol has no field in which a guest could name it instead. + +### The service + +`/etc/tdd-cli/ledger.toml` (or `TDD_LEDGER_SERVICE_CONFIG`), owned by root or the +service account and not group- or world-writable, the same trust rule as `runner.toml`: + + [service] + admin_socket = "/run/tdd-cli/admin.sock" + # home = "/var/lib/tdd-cli" # default: the service account's ~/.local/share/tdd-cli + +Run `tdd ledger serve` under a dedicated account that owns `home`. It runs until SIGTERM +and exits 0. A systemd unit: + + [Unit] + Description=tdd-cli ledger service + + [Service] + User=tdd-ledger + Group=tdd-guests + RuntimeDirectory=tdd-cli + ExecStart=/opt/tdd-cli/bin/tdd ledger serve + KillSignal=SIGTERM + + [Install] + WantedBy=multi-user.target + +Host ledgers are `/.sqlite3`, one per repository, where the slug is the +guest's repository path with `/` replaced by `-`, as on a single machine. Every guest +that checks a repository out at the same path shares that file, kept apart by source. + +### One source per guest + +The operator adds a source for each guest, from the host: + + tdd ledger add-source vm-1 --socket /run/tdd-cli/vm-1.sock --executor claude-fable-5-1 + tdd ledger bind vm-1 --executor claude-opus-5-5 # the model for its next runs + tdd ledger sources # every source, socket and binding + tdd ledger remove-source vm-1 # closes the socket; its runs stay + +Sources are kept in `/sources.json`, so a restarted service listens on all of them +again. The admin socket is mode 600, the service account's alone. A guest's socket is +mode 660: give its group to whatever forwards it into the guest, and nothing else. + +`tdd ledger` verbs are always answered on the machine you run them on, never forwarded to +a runner. With no service config they fail with `reason: "no_service"`. + +### Forwarding the socket into the guest + +The forwarding itself is the operator's, and tdd-cli does not set it up: + +- **ssh**: `ssh -R /run/tdd-cli/ledger.sock:/run/tdd-cli/vm-1.sock guest`, with + `StreamLocalBindUnlink yes` in the guest's `sshd_config` so a reconnect can replace a + stale socket, and `StreamLocalBindMask 0117` so the forwarded socket is not world-open. +- **containers**: bind-mount the source's socket, `-v /run/tdd-cli/vm-1.sock:/run/tdd-cli/ledger.sock`. + Mount the one socket, never the host's ledger directory. +- **vsock**: forward the source's socket to a vsock port with a relay such as `socat`, + and relay it back to a unix socket in the guest. + +### The guest + +The guest's `runner.toml` names the forwarded socket instead of a ledger home: + + [runner] + user = "tdd-runner" + command = "/opt/tdd-cli/bin/tdd" + ledger_socket = "/run/tdd-cli/ledger.sock" # instead of ledger_home + +Naming both `ledger_socket` and `ledger_home` is refused. In the guest, `tdd doctor`'s +`ledger reachable` check pings the service and names the source it reached: +`socket /run/tdd-cli/ledger.sock (source vm-1)`. + +**It fails closed.** If the socket is missing or the service hangs up, every verb that +needs the ledger fails with `reason: "ledger_unreachable"`. Nothing in the guest ever +falls back to a ledger of its own. + +### What it does and does not protect + +The service enforces this for every request, so a modified client cannot get around it: + +- **A guest reads and writes only its own source's runs.** A run, cycle or plan contract + of another source is refused with `refusal: "foreign_run"`, and so is one that does not + exist, so a guest cannot probe for other sources' ids. Claims and the baseline cache + are per source: one guest can neither block another's `run start` at the same path nor + hand it a forged cached baseline. +- **A closed run stays closed.** Writes to a `complete` or `abandoned` run are refused with + `refusal: "run_closed"`. `tdd note` is the one exception, so a closing narrative can still + be added. A `blocked` run is not closed: `resume --unblock` still works. +- **A guest reaches only named domain methods.** The wire carries a method name and its + arguments, never SQL. Methods meant for the host alone (`set_meta`, the generic row + helpers) are refused with `refusal: "method_not_allowed"`, and a repository path that is + not absolute is refused with `refusal: "bad_repo"`. + +Refusals reach the guest's envelope as `reason: "ledger_refused"` with the `refusal` code. + +What a compromised guest can still do is record runs under its own source, and lie in +them, which is no worse than a forged test report on a single machine. + +**Executor identity is set on the host.** A guest's `[executor]` table is guest root's to +edit, so it is a claim. When the operator binds an executor to a source, the service +records that model with `executor_source: "operator"` on every run the source starts or +abandons, whatever the guest sent. With no binding, a guest that says `operator` is +recorded as `claimed`. + +### One history per host + +On the host, the operator reads every source together through the admin socket: + + tdd ledger metrics [--repo ] # `tdd metrics` over every source, per repository + tdd ledger fleet # active runs of every source, and the host's suites + tdd ledger log --repo --run [--out FILE] + +Every run in them carries its `source`. Inside a guest, `tdd metrics`, `tdd fleet` and +`tdd log render` still work, and see only that guest's source. + +**Worker leases are the host's.** A socket-mode runner takes each suite's worker lease +from the service, which splits the host's cores across every guest's running suites. A +lease ends when its connection closes, which is also what a dead VM does, and one older +than the stale limit is not counted. Named leases stay in the guest: they guard +resources the guest's suites run against. + +### Existing ledgers + +`tdd runner import` is refused on a socket-mode runner (`reason: "ledger_remote"`): it has +no ledger to import into. Bring an existing ledger in on the host instead, as one source: + + tdd ledger import /home/agent1/.local/share/tdd-cli/.sqlite3 --source legacy + +It copies the file into the service's home under the same name, migrates it, retags every +run and plan contract it recorded to the named source, drops its claims, and marks it as +`pre_split_import`, which `tdd ledger metrics` reports. It refuses a file the host already +holds (`reason: "ledger_exists"`). There is no merge. + +Ledgers from before schema v13 gain a source on their first open: every existing run, +contract, claim and cached baseline becomes `local`, the source of single-user mode and of +a split runner on one machine. From 33892a992b689d4f1ea90545045a0a6035896a37 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:32:03 +0100 Subject: [PATCH 87/88] docs: friction log for issue-169-ledger-service --- .../issue-169-ledger-service-friction.md | 499 ++++++++++++++++++ 1 file changed, 499 insertions(+) create mode 100644 tasks/friction-logs/issue-169-ledger-service-friction.md diff --git a/tasks/friction-logs/issue-169-ledger-service-friction.md b/tasks/friction-logs/issue-169-ledger-service-friction.md new file mode 100644 index 0000000..7210736 --- /dev/null +++ b/tasks/friction-logs/issue-169-ledger-service-friction.md @@ -0,0 +1,499 @@ +# Implementation Friction Log: tasks/issue-169-ledger-service.md + +- Run: 1 +- Executor: claude-opus-5-5 (source: transcript) +- Plan blob: `6e02083cbc917277e9009daaf8a499dcb10a78e3` (declared) +- Started: 2026-10-03T00:25:37.656330+00:00 Ended: 2026-10-03T03:31:49.860055+00:00 Outcome: complete +- Baseline failures at start: tddcli=0 + +## Plan fidelity + +- Declared cycles: 46 +- Delivered: 46 Skipped: 0 +- Never reached: none +- Human interventions: 0 + +### Cycle 46: document split mode across a VM boundary _(refactor)_ +- **Target:** none +- **Projects:** `tddcli` +- **Suite runs by phase:** {'CLOSE_SWEEP': 1} +- **Commits:** + - `dd6a61155` [refactor] docs: split mode across a VM boundary (3 files) + +### Cycle 45: ledger import refuses a ledger the host already holds _(standard)_ +- **Target:** `tddcli::tests/test_ledger_host_views.py::test_ledger_import_refuses_an_existing_host_ledger` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `83dd475fc` [red] test: ledger import never overwrites (1 files) + - `9d2036c90` [green] feat: ledger import refuses an existing host ledger (2 files) +> **note** _(during AWAITING_REFACTOR)_: cycle 45 also touched cli.py (declared: service.py only): a service refusal reaches the CLI as reason=ledger_refused, so 'reason == ledger_exists' needed the ledger verb wrapper to report an admin refusal's own code as its reason. Done generally in _ledger_verb, so every operator verb reports its service's refusal codes (unknown_source, unknown_repo, ledger_exists) directly. + +### Cycle 44: ledger import brings an existing ledger in as one named source _(standard)_ +- **Target:** `tddcli::tests/test_ledger_host_views.py::test_ledger_import_retags_a_ledger_as_one_source` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `18a99e9d3` [red] test: ledger import retags an existing ledger (1 files) + - `5f0386a31` [green] feat: tdd ledger import (3 files) + +### Cycle 43: ledger log renders any source's run on the host _(standard)_ +- **Target:** `tddcli::tests/test_ledger_host_views.py::test_ledger_log_renders_any_sources_run` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `8936fe7e5` [red] test: host log renders any source's run (1 files) + - `24ee03b22` [green] feat: tdd ledger log (2 files) + +### Cycle 42: ledger fleet on the host lists active runs of every source _(standard)_ +- **Target:** `tddcli::tests/test_ledger_host_views.py::test_ledger_fleet_lists_active_runs_of_every_source` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `392dea256` [red] test: host fleet covers every source (1 files) + - `859f111be` [green] feat: tdd ledger fleet (3 files) + +### Cycle 41: ledger metrics on the host covers every source _(standard)_ +- **Target:** `tddcli::tests/test_ledger_host_views.py::test_ledger_metrics_covers_every_source` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `c7cf65c90` [red] test: host metrics cover every source (1 files) + - `0ebcb9a74` [green] feat: tdd ledger metrics (4 files) + +### Cycle 40: a guest's fleet view reports the host's live lease count _(standard)_ +- **Target:** `tddcli::tests/test_remote_leases.py::test_guest_fleet_reports_the_hosts_lease_count` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `a85a76a06` [red] test: guest fleet shows host-wide suites (1 files) + - `47f6dcd2d` [green] feat: a socket-mode lease snapshot comes from the service (3 files) + +### Cycle 39: a lease older than the stale limit is not counted _(standard)_ +- **Target:** `tddcli::tests/test_remote_leases.py::test_a_stale_lease_is_not_counted` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `023c90b93` [red] test: stale leases do not throttle the host (1 files) + - `104976269` [green] feat: the service ignores leases past the stale limit (1 files) + +### Cycle 38: a lease ends when its connection closes _(standard)_ +- **Target:** `tddcli::tests/test_remote_leases.py::test_a_lease_ends_when_its_connection_closes` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `51e91ae95` [red] test: a closed lease connection frees its share (1 files) + - `4d9cbdc71` [green] feat: the service forgets a lease when its connection closes (1 files) + +### Cycle 37: the service counts worker leases across sources _(standard)_ +- **Target:** `tddcli::tests/test_remote_leases.py::test_the_service_counts_leases_across_sources` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `f52f6743a` [red] test: worker leases are counted across guests (1 files) + - `aa5737883` [green] feat: a socket-mode runner takes its worker lease from the service (3 files) + +### Cycle 36: ledger serve runs the service until SIGTERM and exits 0 _(standard)_ +- **Target:** `tddcli::tests/test_ledger_admin.py::test_ledger_serve_runs_until_terminated` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2, 'CLOSE_SWEEP': 1} +- **First run outcome:** failed (as expected) +- **Commits:** + - `1485bf0fa` [red] test: ledger serve runs until SIGTERM (1 files) + - `345a13f71` [green] feat: tdd ledger serve (2 files) + - `b0e86aa8e` [refactor] refactor: tidy serve (1 files) + +### Cycle 35: the admin socket is mode 600 and guest sockets are mode 660 _(standard)_ +- **Target:** `tddcli::tests/test_ledger_admin.py::test_socket_modes` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `356c568fc` [red] test: admin and guest socket modes (1 files) + - `38838b2f4` [green] feat: the service sets its sockets' modes (1 files) + +### Cycle 34: sources and bindings survive a service restart _(standard)_ +- **Target:** `tddcli::tests/test_ledger_admin.py::test_sources_survive_a_service_restart` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `314cb3d9d` [red] test: sources survive a service restart (1 files) + - `24b050ec2` [green] feat: the service persists its sources (1 files) + +### Cycle 33: ledger remove-source closes and removes its socket _(standard)_ +- **Target:** `tddcli::tests/test_ledger_admin.py::test_remove_source_closes_its_socket` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `5ffde934b` [red] test: remove-source closes the guest socket (1 files) + - `07e2920e0` [green] feat: tdd ledger remove-source (2 files) + +### Cycle 32: ledger bind changes a source's executor for its next run _(standard)_ +- **Target:** `tddcli::tests/test_ledger_admin.py::test_bind_changes_the_executor_for_the_next_run` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `3d628b7aa` [red] test: bind changes a source's executor (1 files) + - `029473356` [green] feat: tdd ledger bind (2 files) + +### Cycle 31: ledger sources lists each source with its socket and binding _(standard)_ +- **Target:** `tddcli::tests/test_ledger_admin.py::test_sources_lists_each_source_and_its_binding` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `49d0bfb72` [red] test: ledger sources lists sources (1 files) + - `f8ad925d5` [green] feat: tdd ledger sources (2 files) + +### Cycle 30: ledger add-source opens a guest socket through the admin socket _(standard)_ +- **Target:** `tddcli::tests/test_ledger_admin.py::test_add_source_opens_a_guest_socket` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `77743fd52` [red] test: add-source opens a guest socket (1 files) + - `f9f818cb2` [green] feat: tdd ledger add-source (2 files) + +### Cycle 29: ledger verbs run locally and refuse without a service config _(standard)_ +- **Target:** `tddcli::tests/test_ledger_admin.py::test_ledger_verbs_are_never_forwarded_to_a_runner` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `023b5776d` [red] test: ledger verbs stay local and need a service config (1 files) + - `e27e18916` [green] feat: tdd ledger verb group, answered locally (2 files) +> **note** _(during AWAITING_REFACTOR)_: cycle 29 also changed tests/test_split_client.py (not in modifies_tests): test_only_docs_and_init_stay_local pinned the local verb set to {docs, init}, and decision 16 adds ledger by design. Renamed it test_only_docs_init_and_ledger_stay_local, widened the set, and gave ledger an ARGV entry so the verb actually parses rather than staying local by failing to. + +### Cycle 28: doctor on a socket-mode runner names the source it reaches _(standard)_ +- **Target:** `tddcli::tests/test_split_doctor.py::test_socket_mode_doctor_names_the_source_it_reaches` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'SENSITIVITY': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** passed (**passed**) +- **Sensitivity check:** verified, restore byte-identical + - observed: `assert (True, False) == (True, True)` +- **Commits:** + - `41adac5ef` [refactor] refactor: tidy socket-mode doctor (3 files) +- **Event — red_first_violation:** ["tddcli::tests/test_split_doctor.py::test_socket_mode_doctor_names_the_source_it_reaches"] +> **note** _(during AWAITING_REFACTOR)_: cycle 28 pre-passed: the plan's assertion ('vm-1' in the ledger-reachable detail) was already true because the guest socket's file name is vm-1.sock. Ran the sensitivity check against the test as written (it bites), then in the refactor phase tightened it to 'source vm-1' (fails on the old detail, which was the socket path alone) and implemented the planned GREEN: doctor pings the service and renders 'socket (source )', skipping the local ledger-readability probe in socket mode. + +### Cycle 27: a runner config naming both ledger_socket and ledger_home is refused _(standard)_ +- **Target:** `tddcli::tests/test_split_config.py::test_ledger_socket_and_ledger_home_together_are_refused` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `a1567f3b4` [red] test: ledger_socket and ledger_home are exclusive (1 files) + - `14f26f8c1` [green] feat: refuse a runner config naming both ledger locations (1 files) + +### Cycle 26: runner import is refused on a socket-mode runner _(standard)_ +- **Target:** `tddcli::tests/test_runner_import.py::test_import_is_refused_when_the_ledger_is_on_the_host` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `82c3ecc9f` [red] test: runner import is refused when the ledger is remote (1 files) + - `e1ecf8733` [green] feat: runner import refuses a socket-mode runner (1 files) + +### Cycle 25: claim staleness is judged in the guest, not on the host _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_claim_staleness_is_judged_in_the_guest` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `f8b298245` [red] test: claim staleness is judged where the pid lives (1 files) + - `52f1874f3` [green] feat: the remote ledger judges claim liveness locally (2 files) + +### Cycle 24: a bound source stamps the abandoning executor too _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_bound_source_stamps_the_abandoning_executor` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2, 'CLOSE_SWEEP': 1} +- **First run outcome:** failed (as expected) +- **Commits:** + - `e3410288e` [red] test: abandonment records the bound executor (1 files) + - `574cd5e33` [green] feat: the service stamps the executor on abandonment (1 files) + - `9aa53c0f9` [refactor] refactor: tidy abandonment stamping (1 files) + +### Cycle 23: an unbound source records a guest-claimed operator executor as claimed _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_an_unbound_source_downgrades_a_guest_operator_claim` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `826d5b102` [red] test: a guest cannot claim operator identity (1 files) + - `8ec2def10` [green] feat: an unbound source records guest identity as claimed (1 files) + +### Cycle 22: a bound source records the operator-assigned executor _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_bound_source_records_the_operator_assigned_executor` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `a62d3c1f5` [red] test: a bound source records the operator's executor (1 files) + - `0e851b688` [green] feat: the service stamps a bound source's executor on run start (1 files) + +### Cycle 21: a guest may still note a completed run _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_guest_may_still_note_a_completed_run` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `55fc021a4` [red] test: notes on a completed run are still accepted (1 files) + - `058d55b94` [green] feat: appending a note is exempt from the closed-run rule (1 files) + +### Cycle 20: a guest cannot write to its own completed run _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_guest_cannot_write_to_a_completed_run` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `1ca514029` [red] test: a completed run is closed to guest writes (1 files) + - `ff8368bbe` [green] feat: the service refuses writes to complete or abandoned runs (1 files) + +### Cycle 19: a guest repo path that is not absolute is refused _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_malformed_repo_path_is_refused` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `d1ba5a035` [red] test: malformed repo paths are refused (1 files) + - `db58c5139` [green] feat: the service validates the guest's repo path (1 files) + +### Cycle 18: a guest cannot call a host-only method _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_guest_cannot_call_a_host_only_method` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `a04d449c6` [red] test: guests cannot call host-only ledger methods (1 files) + - `54f77258f` [green] feat: the service dispatches only guest-classified methods (2 files) +> **note** _(during AWAITING_REFACTOR)_: cycle 18 also touched ledger.py (not declared): cycle 11 had classified active_claim/active_advance_claim as HOST_ONLY, but guest commands still call them through the socket until cycle 25 composes them locally; the allow-list would have broken every guest run start. Reclassified them as SOURCE_ROOTED guest reads here; cycle 25 moves them to HOST_ONLY, which is what its expected failure assumes. + +### Cycle 17: a guest naming another source's cycle or contract is refused _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_guest_naming_another_sources_cycle_is_refused` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `dcbd8d72c` [red] test: a guest cannot reach another source's cycle (1 files) + - `1309371bb` [green] feat: ownership checks cover cycle and contract ids (1 files) + +### Cycle 16: a guest naming another source's run is refused _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_guest_naming_another_sources_run_is_refused` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `98a7eb9ef` [red] test: a guest cannot reach another source's run (1 files) + - `5796174c2` [green] feat: the service refuses run ids of other sources (4 files) + +### Cycle 15: an unreachable ledger socket refuses the verb _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_an_unreachable_socket_refuses_the_verb` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `78c276d0e` [red] test: an unreachable ledger socket refuses the verb (1 files) + - `99ea8d598` [green] feat: fail closed when the ledger service is unreachable (2 files) + +### Cycle 14: a blocked guest run can still be unblocked _(pin)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_blocked_guest_run_can_be_unblocked` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_PIN': 1, 'SENSITIVITY': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** passed (as expected) +- **Sensitivity check:** verified, restore byte-identical + - observed: `RuntimeError: mutated` +- **Commits:** + - `871ed1497` [pin] test: pin a blocked guest run can still be unblocked (1 files) +> **note** _(during SENSITIVITY_REQUIRED)_: cycle 14: the plan's pin uses `tdd blocker --kind environment`, which is not in BLOCKER_KINDS; the pin uses plan_defect instead. Same path: blocker then resume --unblock. + +### Cycle 13: a guest renders its own friction log through the socket _(pin)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_guest_renders_its_own_friction_log` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_PIN': 1, 'SENSITIVITY': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** passed (as expected) +- **Sensitivity check:** verified, restore byte-identical + - observed: `TypeError: 'NoneType' object is not subscriptable` +- **Commits:** + - `1a13225e3` [pin] test: pin a guest renders its own friction log through the socket (1 files) + +### Cycle 12: a guest runner's run lands in the host ledger under its source _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_guest_run_lands_in_the_host_ledger_under_its_source` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `fd9465f41` [red] test: a guest run lands in the host ledger (2 files) + - `baf996a46` [green] feat: a runner with ledger_socket keeps its ledger on the host (6 files) + +### Cycle 11: every Ledger method is classified for guests and scoped _(standard)_ +- **Target:** `tddcli::tests/test_ledger_surface.py::test_every_ledger_method_is_classified_and_scoped` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `cfd2f5b3d` [red] test: every ledger method is classified for guests (1 files) + - `3e94cae95` [green] feat: classify the ledger's methods for guests (1 files) + +### Cycle 10: a source socket answers ping with the source's name and binding _(standard)_ +- **Target:** `tddcli::tests/test_ledger_service.py::test_a_source_socket_answers_ping_with_its_name` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 2, 'AWAITING_IMPL': 2} +- **First run outcome:** not_collected (**not_collected**) +- **Commits:** + - `c2246568b` [red] test: a source socket answers ping (3 files) + - `b9173ef0b` [green] feat: ledger service listens on a socket per source (3 files) +- **Event — stub_directive_issued:** ["tddcli::tests/test_ledger_service.py::test_a_source_socket_answers_ping_with_its_name"] + +### Cycle 9: metrics names each run's source _(standard)_ +- **Target:** `tddcli::tests/test_ledger_sources.py::test_metrics_names_each_runs_source` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `fb5c15a4a` [red] test: metrics names each run's source (1 files) + - `c1c4e1148` [green] feat: metrics names each run's source (1 files) + +### Cycle 8: two sources claim and cache the same worktree independently _(standard)_ +- **Target:** `tddcli::tests/test_ledger_sources.py::test_two_sources_claim_and_cache_the_same_worktree_independently` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `ee6d5b5a9` [red] test: claims and baseline cache are per source (1 files) + - `257f69f96` [green] feat: claims and the baseline cache are keyed by source (1 files) + +### Cycle 7: a Ledger bound to one source never reads another source's runs or contracts _(standard)_ +- **Target:** `tddcli::tests/test_ledger_sources.py::test_a_source_reads_only_its_own_runs_and_contracts` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `6ace2773b` [red] test: a source reads only its own runs and contracts (1 files) + - `d8accbd0b` [green] feat: source-rooted ledger reads filter by source (1 files) +> **note** _(during AWAITING_REFACTOR)_: cycle 7: the declared stub (source kwarg) was not needed as a separate step; the TypeError was accepted as RED directly. max_suite_duration_ms excludes other sources' runs with NOT IN rather than a JOIN, because tests/test_named_leases_and_timeout.py inserts invocations with no run row. + +### Cycle 6: a v12 ledger migrates to v13 with every row tagged source 'local' _(standard)_ +- **Target:** `tddcli::tests/test_ledger_sources.py::test_a_v12_ledger_migrates_with_its_rows_tagged_local` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `7c191e282` [red] test: a v12 ledger migrates with its rows tagged local (1 files) + - `656bfb87d` [green] feat: ledger schema v13 records each run's source (1 files) + +### Cycle 5: no module but ledger.py speaks SQL or imports sqlite3 _(standard)_ +- **Target:** `tddcli::tests/test_ledger_surface.py::test_no_module_but_the_ledger_speaks_sql` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `3e5d56b1d` [red] test: only the ledger module speaks SQL (1 files) + - `0c3817e26` [green] refactor: render reaches the ledger through named domain methods (2 files) + +### Cycle 4: fleet reads through a read-only Ledger instead of its own sqlite connection _(refactor)_ +- **Target:** none +- **Projects:** `tddcli` +- **Suite runs by phase:** {'CLOSE_SWEEP': 1} +- **Commits:** + - `9c55c1b43` [refactor] refactor: fleet reads through a read-only Ledger (4 files) + +### Cycle 3: the final close decides the run's outcome before it ends the run _(refactor)_ +- **Target:** none +- **Projects:** `tddcli` +- **Suite runs by phase:** {'CLOSE_SWEEP': 1} +- **Commits:** + - `ece0dce39` [refactor] refactor: the final close settles blocked-or-complete before ending the run (3 files) + +### Cycle 2: machine.py and advance.py reach the ledger only through named Ledger methods _(refactor)_ +- **Target:** none +- **Projects:** `tddcli` +- **Suite runs by phase:** {'CLOSE_SWEEP': 1} +- **Commits:** + - `c4f648b95` [refactor] refactor: the state machine reaches the ledger through named domain methods (3 files) + +### Cycle 1: cli.py reaches the ledger only through named Ledger methods _(refactor)_ +- **Target:** none +- **Projects:** `tddcli` +- **Suite runs by phase:** {'CLOSE_SWEEP': 1} +- **Commits:** + - `ec923a963` [refactor] refactor: cli reaches the ledger through named domain methods (2 files) + +## Time + +- Wall clock: 186.2 min. Suite: 79.8 min (43%). + +| Phase | Suite runs | Suite (min) | Average (s) | +|---|---|---|---| +| AWAITING_TEST | 40 | 1.1 | 1.7 | +| AWAITING_PIN | 2 | 0.1 | 2.2 | +| SENSITIVITY | 3 | 0.1 | 2.0 | +| AWAITING_IMPL | 76 | 62.2 | 49.1 | +| CLOSE_SWEEP | 10 | 16.3 | 97.9 | + +| Cycle | Wall (min) | Suite (min) | Suite runs | +|---|---|---|---| +| 1 | 5.2 | 2.0 | 1 | +| 2 | 4.7 | 1.6 | 1 | +| 3 | 2.1 | 1.5 | 1 | +| 4 | 2.3 | 1.5 | 1 | +| 5 | 4.2 | 1.5 | 3 | +| 6 | 4.2 | 1.5 | 3 | +| 7 | 4.5 | 1.5 | 3 | +| 8 | 3.8 | 1.6 | 3 | +| 9 | 3.7 | 1.6 | 3 | +| 10 | 4.3 | 1.5 | 4 | +| 11 | 3.9 | 1.6 | 3 | +| 12 | 5.4 | 1.6 | 3 | +| 13 | 2.1 | 1.6 | 3 | +| 14 | 2.0 | 1.6 | 3 | +| 15 | 3.8 | 1.6 | 3 | +| 16 | 3.8 | 1.6 | 3 | +| 17 | 5.5 | 1.7 | 3 | +| 18 | 3.9 | 1.6 | 3 | +| 19 | 3.6 | 1.6 | 3 | +| 20 | 3.7 | 1.6 | 3 | +| 21 | 3.6 | 1.7 | 3 | +| 22 | 3.7 | 1.7 | 3 | +| 23 | 5.4 | 1.6 | 3 | +| 24 | 5.3 | 3.3 | 4 | +| 25 | 4.0 | 1.6 | 3 | +| 26 | 3.7 | 1.7 | 3 | +| 27 | 3.7 | 1.7 | 3 | +| 28 | 4.5 | 1.7 | 3 | +| 29 | 6.1 | 1.7 | 3 | +| 30 | 3.9 | 1.7 | 3 | +| 31 | 3.8 | 1.7 | 3 | +| 32 | 3.6 | 1.7 | 3 | +| 33 | 3.6 | 1.7 | 3 | +| 34 | 3.9 | 1.7 | 3 | +| 35 | 3.7 | 1.7 | 3 | +| 36 | 5.7 | 3.3 | 4 | +| 37 | 4.0 | 1.7 | 3 | +| 38 | 4.7 | 1.7 | 3 | +| 39 | 3.6 | 1.7 | 3 | +| 40 | 3.9 | 1.8 | 3 | +| 41 | 4.1 | 1.8 | 3 | +| 42 | 4.1 | 1.8 | 3 | +| 43 | 4.1 | 1.8 | 3 | +| 44 | 4.2 | 1.8 | 3 | +| 45 | 3.9 | 1.8 | 3 | +| 46 | 4.3 | 1.7 | 1 | + +## Executor narrative + +_Claims from the executor, unverified by design._ + +> hardest cycle: 38, lease release on EOF: the handler thread sees EOF asynchronously, so the next guest's lease could still count a finished one; made deterministic by pruning hung-up connections with a non-blocking MSG_PEEK at take time. plan got wrong: cycle 28's assertion pre-passed (the socket file is named vm-1.sock); cycle 14 named a blocker kind (environment) that does not exist; cycles 18, 29 and 45 needed files the plan did not declare (ledger.py reclassification of active_claim until cycle 25; tests/test_split_client.py's local-verb guard; cli.py to surface ledger_exists as the reason). harness friction: the worktree guard rejects heredocs and chained git, so every multi-line edit went through a scripted exact-match replacement file; ruff format reformatted an already-committed test once and it had to be reverted to keep the cycle's file set clean. + From fea8a9158d326643787fb0f602846a578c390c17 Mon Sep 17 00:00:00 2001 From: "geuben-dispatch[bot]" <326654978+geuben-dispatch[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:38:36 +0100 Subject: [PATCH 88/88] test: cover the service's error replies, closed-run writes by cycle, and dropped connections Found by the pre-PR mutation pass: a Connection closed after a dropped write raised BrokenPipeError from its flush, masking ledger_unreachable. Removes two unreachable branches in _hung_up and the unused Row.__eq__. --- src/tddcli/service.py | 18 ++++---- src/tddcli/wire.py | 9 ++-- tests/test_ledger_service.py | 45 ++++++++++++++++++++ tests/test_ledger_sources.py | 8 ++++ tests/test_wire.py | 80 ++++++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 tests/test_wire.py diff --git a/src/tddcli/service.py b/src/tddcli/service.py index 954d726..63941c1 100644 --- a/src/tddcli/service.py +++ b/src/tddcli/service.py @@ -111,7 +111,7 @@ class _Session: connection must not cross threads. """ - def __init__(self, service: Service, source: str | None, conn: socket.socket | None = None): + def __init__(self, service: Service, source: str | None, conn: socket.socket): self.service = service self.source = source self.conn = conn @@ -225,16 +225,16 @@ def close(self) -> None: self.ledger.close() -def _hung_up(conn: socket.socket | None) -> bool: - """Whether the other end of a connection has closed it. Reads nothing.""" - if conn is None: - return False +def _hung_up(conn: socket.socket) -> bool: + """Whether the other end of a connection has closed it. Reads nothing. + + A lease is dropped before its own end of the connection is closed, so `conn` is + always open here: only the guest's end can have gone. + """ try: return conn.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) == b"" except BlockingIOError: return False - except OSError: - return True def _well_formed_repo(repo) -> bool: @@ -301,7 +301,7 @@ def __init__(self, config: ServiceConfig) -> None: # Worker leases held by guests, across every source: one budget per host. A # lease is its connection, held open for as long as the suite runs. self.lock = threading.Lock() - self.leases: dict[_Session, tuple[socket.socket | None, float]] = {} + self.leases: dict[_Session, tuple[socket.socket, float]] = {} def start(self) -> None: self.config.home.mkdir(parents=True, exist_ok=True) @@ -338,7 +338,7 @@ def bind(self, name: str, executor: str) -> None: self.executors[name] = executor self._save() - def take_lease(self, holder: _Session, conn: socket.socket | None) -> int: + def take_lease(self, holder: _Session, conn: socket.socket) -> int: """Count one more suite running on this host; the worker count it should use. A lease whose connection has already hung up is forgotten first, so a guest diff --git a/src/tddcli/wire.py b/src/tddcli/wire.py index f1168e9..1fac47e 100644 --- a/src/tddcli/wire.py +++ b/src/tddcli/wire.py @@ -10,6 +10,7 @@ from __future__ import annotations +import contextlib import json import socket from pathlib import Path @@ -56,9 +57,6 @@ def __iter__(self): def __len__(self) -> int: return len(self._vals) - def __eq__(self, other) -> bool: - return isinstance(other, Row) and (self._cols, self._vals) == (other._cols, other._vals) - def __repr__(self) -> str: return f"Row({dict(zip(self._cols, self._vals, strict=True))!r})" @@ -134,7 +132,10 @@ def request(self, method: str, *args, **kwargs): return decode(reply.get("result")) def close(self) -> None: - self.stream.close() + # Closing flushes the stream: after a dropped connection, a request still in its + # buffer would raise here, masking the LedgerUnreachable already reported. + with contextlib.suppress(OSError): + self.stream.close() self.sock.close() diff --git a/tests/test_ledger_service.py b/tests/test_ledger_service.py index febbe54..ed482a1 100644 --- a/tests/test_ledger_service.py +++ b/tests/test_ledger_service.py @@ -11,6 +11,7 @@ import socket import subprocess import tempfile +import time from pathlib import Path import pytest @@ -236,3 +237,47 @@ def test_claim_staleness_is_judged_in_the_guest(repo, split_guest, ledger_servic out = run_cli(repo, "progress", "--json") assert out["result"].get("stale") is True + + +def test_a_guest_cannot_write_through_a_cycle_of_a_completed_run(repo, split_guest, ledger_service): + run_id = complete_on_host(ledger_service, repo) + host = ledger_mod.open_readonly(host_file(ledger_service, repo), source=None) + cycle_id = host.cycles(run_id)[0]["id"] + + refusal = refusal_of( + split_guest.socket, str(gitutil.repo_identity(repo)), "set_targets", cycle_id, ["t"] + ) + assert refusal == "run_closed" + + +def error_of(socket_path, repo_path, method: str, *args, **kwargs) -> str: + """How a request the service cannot answer comes back: its error, never a hang-up.""" + conn = wire.Connection(socket_path) + try: + if repo_path is not None: + conn.request("open", repo_path) + conn.request(method, *args, **kwargs) + except RuntimeError as exc: + return type(exc).__name__ + finally: + conn.close() + return "answered" + + +def test_a_request_the_service_cannot_answer_is_an_error_not_a_hang_up(repo, split_guest): + repo_path = str(gitutil.repo_identity(repo)) + errors = [ + error_of(split_guest.socket, None, "cycle", 1), # nothing opened yet + error_of(split_guest.socket, repo_path, "cycle", 1, 2, 3), # arguments do not bind + error_of(split_guest.socket, repo_path, "get_meta", ["k"]), # the method raises + ] + assert errors == ["RuntimeError"] * 3 + + +def test_a_connection_that_never_opened_a_ledger_closes_cleanly(split_guest, capsys): + wire.call(split_guest.socket, "ping") + # The next round trip gives the first connection's handler time to finish. + wire.call(split_guest.socket, "ping") + time.sleep(0.3) + + assert "Traceback" not in capsys.readouterr().err diff --git a/tests/test_ledger_sources.py b/tests/test_ledger_sources.py index ccf556a..ed6e04a 100644 --- a/tests/test_ledger_sources.py +++ b/tests/test_ledger_sources.py @@ -172,6 +172,14 @@ def test_metrics_names_each_runs_source(repo): assert [r.get("source") for r in run_cli(repo, "metrics")["result"]["runs"]] == ["local"] +def test_a_source_reads_back_its_own_cached_baseline(tmp_path, ledger_home): + a = Ledger(tmp_path / "somerepo", source="a") + a.cache_baseline("app", "tree", "cfg", failing=["t::x"], tests=["t::x"], failed_files={}) + + cached = a.cached_baseline("app", "tree", "cfg") + assert (cached is not None, cached and cached["source"]) == (True, "a") + + def test_two_sources_claim_and_cache_the_same_worktree_independently(tmp_path, ledger_home): repo = tmp_path / "somerepo" a = Ledger(repo, source="a") diff --git a/tests/test_wire.py b/tests/test_wire.py new file mode 100644 index 0000000..e5c99d3 --- /dev/null +++ b/tests/test_wire.py @@ -0,0 +1,80 @@ +"""The ledger service's wire: rows keep their shape, and a dropped connection fails closed.""" + +from __future__ import annotations + +import shutil +import socket +import tempfile +import threading +from pathlib import Path + +import pytest + +from tddcli import wire + + +def test_a_row_reads_like_an_sqlite_row(): + row = wire.decode(wire.encode([{"$row": {"cols": ["id", "source"], "vals": [7, "vm-1"]}}]))[0] + with pytest.raises(IndexError): + row["absent"] + + assert (row["source"], row[0], row.keys(), dict(row)) == ( + "vm-1", + 7, + ["id", "source"], + {"id": 7, "source": "vm-1"}, + ) + + +def test_sets_and_nested_rows_survive_the_round_trip(): + sent = {"failing": {"t::b", "t::a"}, "rows": [], "n": 3} + + assert wire.decode(wire.encode(sent)) == {"failing": {"t::a", "t::b"}, "rows": [], "n": 3} + + +@pytest.fixture +def socket_dir(): + # Under /tmp, not tmp_path: a socket path under tmp_path is longer than AF_UNIX allows. + path = Path(tempfile.mkdtemp(dir="/tmp", prefix="tdd-")) + yield path + shutil.rmtree(path, ignore_errors=True) + + +def _server(path: Path, behave) -> threading.Thread: + """A one-connection server that does `behave(conn)` and then hangs up.""" + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + listener.bind(str(path)) + listener.listen(1) + + def serve(): + conn, _ = listener.accept() + try: + behave(conn) + finally: + conn.close() + listener.close() + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + return thread + + +def test_a_service_that_hangs_up_before_answering_is_unreachable(socket_dir): + path = socket_dir / "s.sock" + _server(path, lambda conn: conn.makefile("rb").readline()) + + with pytest.raises(wire.LedgerUnreachable, match="closed the connection"): + wire.call(path, "ping") + + +def test_a_service_that_is_gone_before_the_request_is_unreachable(socket_dir): + path = socket_dir / "s.sock" + _server(path, lambda conn: None) + conn = wire.Connection(path) + # Blocks until the server's end has closed, so the request meets a broken pipe. + assert conn.sock.recv(1, socket.MSG_PEEK) == b"" + try: + with pytest.raises(wire.LedgerUnreachable, match="dropped the connection"): + conn.request("ping") + finally: + conn.close()