From a1c7a8a779480c384b30af582025c390275d34b0 Mon Sep 17 00:00:00 2001 From: sourrrish Date: Sat, 13 Jun 2026 23:28:27 +0530 Subject: [PATCH 1/8] feat(plugins): capture Claude and Codex harness events as producer envelopes (#997) Signed-off-by: sourrrish --- plugins/claude-code/DESIGN.md | 132 +++++++++ plugins/claude-code/hooks/pre-compact.sh | 70 ++++- plugins/claude-code/hooks/session-start.sh | 48 +++- plugins/claude-code/schemas/tool-ledger.md | 49 ++++ plugins/claude-code/settings.example.json | 4 + plugins/codex/hooks/pre-compact.py | 66 +++++ plugins/codex/hooks/session-start.py | 39 +++ plugins/codex/schemas/tool-ledger.md | 49 ++++ plugins/shared/harness_envelope.py | 302 +++++++++++++++++++++ 9 files changed, 757 insertions(+), 2 deletions(-) create mode 100644 plugins/claude-code/schemas/tool-ledger.md create mode 100644 plugins/codex/schemas/tool-ledger.md create mode 100644 plugins/shared/harness_envelope.py diff --git a/plugins/claude-code/DESIGN.md b/plugins/claude-code/DESIGN.md index b759a4ad8..6e6f4473d 100644 --- a/plugins/claude-code/DESIGN.md +++ b/plugins/claude-code/DESIGN.md @@ -623,3 +623,135 @@ Docs done 2026-05-28; dogfood is the remaining (human) step. - **Subagent memory bundling** — explore `memory: project|user` on dedicated BM subagents. - **Statusline** — small visible presence (active project, last write). - **`/basic-memory:bm-promote`** — review auto-memory MEMORY.md, graduate observations into BM with proper schema. + +## 14. Harness WAL — event capture as producer envelopes + +**Status:** v0 shipped (issue [#997](https://github.com/basicmachines-co/basic-memory/issues/997)) +**Related:** SPEC-55 (Agent Memory Pipeline — Producer Framework), SPEC-61 (Event-Driven Memory Routines) + +### 14.1 What it is + +A shared, opt-in "harness WAL" path that normalizes supported hook events into +Basic Memory **producer envelopes** — a structured event record that stamps each +checkpoint with its source, session, hook, and an idempotency key. This is the +producer side of SPEC-55; it feeds SPEC-61 memory routines without turning Basic +Memory into an agent runtime. + +### 14.2 The envelope schema + +```python +@dataclass(frozen=True) +class HarnessEnvelope: + event_type: str # "session_started", "compaction_imminent", "session_ended" + source: str # "claude-code" or "codex" + session_id: str # harness session identifier + turn_id: str | None # turn identifier when available (Codex) + timestamp: str # ISO 8601 + cwd: str # working directory + project_hint: str # basicMemory.primaryProject + hook_name: str # "SessionStart", "PreCompact" + idempotency_key: str # sha256(source:session_id:hook:timestamp_minute)[:16] + payload_summary: dict # safe, redacted payload excerpt +``` + +The module lives at `plugins/shared/harness_envelope.py` — stdlib-only, no install +step. Both plugins import it via `sys.path.insert`. + +### 14.3 V0 event types + +V0 captures only events exposed through existing hooks: + +| Event | Hook | Plugin | What it does | +| ---------------------- | ------------- | ------------ | ---------------------------------------------- | +| `session_started` | SessionStart | Both | Logs to local event log (read-only hook) | +| `compaction_imminent` | PreCompact | Both | Stamps provenance onto the SessionNote/CodexSession | +| `session_ended` | (future) | (future) | Defined but not captured in v0 | + +Events like `tool_called`, `file_changed`, `test_ran` require `PostToolUse` hooks +that don't currently exist — deferred to v1. + +### 14.4 What gets stamped on checkpoints + +PreCompact checkpoints gain two additions: + +1. **Frontmatter fields** — `envelope_source`, `envelope_event`, `envelope_hook`, + `idempotency_key` (and `envelope_turn_id` when available). These make checkpoints + queryable by source and dedup-safe. + +2. **Provenance observations** — appended to the `## Observations` section: + ```markdown + - [source] claude-code/abc-123-def + - [hook] PreCompact + - [event] compaction_imminent at 2026-06-13T16:48:00+00:00 + - [idempotency] b7ff76ee5df8cc0a + ``` + +### 14.5 Local event log + +When `captureEvents: true` is set, both SessionStart and PreCompact append a JSONL +record to `/.basic-memory/events.jsonl`. This log: + +- Is append-only (no reads during hook execution) +- Is capped at 1000 lines (configurable via `eventRetention`); oldest half rotates out +- Feeds future SPEC-61 memory routines (nightly coalescing, session summaries) +- Never blocks the hook — write failures are silently swallowed + +### 14.6 Configuration + +New keys in `basicMemory` (both `.claude/settings.json` and `.codex/basic-memory.json`): + +| Key | Type | Default | Description | +| ---------------- | ---------- | ------- | ---------------------------------------------- | +| `captureEvents` | `boolean` | `false` | Opt-in for local event log (events.jsonl) | +| `redactKeys` | `string[]` | `[]` | Extra key patterns to redact from payloads | +| `redactPaths` | `string[]` | `[]` | Extra path prefixes to redact | +| `eventRetention` | `number` | `1000` | Max lines in the local event log before rotation | + +### 14.7 Privacy and redaction + +The envelope module applies layered redaction before any payload summary is stored: + +1. **Key-pattern deny list** — keys matching `SECRET`, `TOKEN`, `KEY`, `PASSWORD`, + `CREDENTIAL`, `AUTH` (case-insensitive) are replaced with `[REDACTED]` +2. **Secret-value detection** — values matching `[A-Za-z0-9_]+=.{20,}` (environment + secret pattern) are replaced with `[REDACTED]` +3. **Path deny list** — values starting with `~/.ssh/`, `~/.aws/`, `~/.gnupg/` are + replaced with `[REDACTED_PATH]`. Extended via `redactPaths` config. +4. **Truncation** — any single value over 500 chars is truncated + +Constraints from the issue: +- Capture is opt-in (tied to plugin installation + `captureEvents` config) +- Never captures hidden chain-of-thought or private model reasoning +- Prefers summaries and metadata over raw transcript dumps +- Fails fast on missing project mapping (no `primaryProject` → no capture) + +### 14.8 Idempotency + +The `idempotency_key` is a 16-char hex string derived from +`sha256(source:session_id:hook:timestamp_minute)`. Minute granularity means: + +- Repeated hooks within the same minute for the same session produce the same key +- A hook one minute later produces a distinct key (new event) +- No persistent state is required for dedup + +The key is written into the note's frontmatter so downstream consumers can detect +and skip duplicates. + +### 14.9 ToolLedger schema (forward compatibility) + +Both plugins ship a `schemas/tool-ledger.md` picoschema defining the ToolLedger +note type. V0 does not produce ToolLedger notes — the schema exists so that when +`PostToolUse` hooks become available (v1), the note shape is already defined and +schema validation works immediately. + +### 14.10 Forward compatibility + +- **SPEC-55 Producer Framework** — the envelope shape aligns with SPEC-55's producer + envelope contract. When the Producer SDK lands, the shared module becomes a thin + adapter over the SDK rather than a standalone implementation. +- **SPEC-61 Memory Routines** — the local event log (`events.jsonl`) is the input + feed for event-driven routines. The nightly coalescing routine reads the log, + groups events by session, and produces enriched artifacts. +- **SPEC-56 Consolidation** — provenance observations enable consolidation to trace + which sessions produced which notes, supporting the dream-mode merge. + diff --git a/plugins/claude-code/hooks/pre-compact.sh b/plugins/claude-code/hooks/pre-compact.sh index b904e9d8a..95117d437 100755 --- a/plugins/claude-code/hooks/pre-compact.sh +++ b/plugins/claude-code/hooks/pre-compact.sh @@ -37,7 +37,12 @@ else exit 0 fi -BM_HOOK_INPUT="$input" BM_BIN="$BM" python3 <<'PY' 2>/dev/null || exit 0 +# Resolve the hook script's own directory so the inline Python can find the +# shared envelope module. __file__ is '' inside a heredoc, so the Python +# code can't locate itself — we pass the real path. +hook_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" + +BM_HOOK_INPUT="$input" BM_BIN="$BM" BM_HOOK_DIR="$hook_dir" python3 <<'PY' 2>/dev/null || exit 0 import json import os import re @@ -46,6 +51,28 @@ import subprocess import sys from datetime import datetime +# --- Load the shared envelope module (lives two directories up in plugins/shared/) --- +# Trigger: this hook wants to stamp provenance and idempotency on the checkpoint. +# Why: the envelope normalizes hook events so downstream consumers (recall, +# consolidation, memory routines) can trace where each note came from. +# Constraint: __file__ is '' inside a bash heredoc, so the hook script's +# real directory is passed in via the BM_HOOK_DIR environment variable. +_hook_dir = os.environ.get("BM_HOOK_DIR", "") +if _hook_dir: + _shared_dir = os.path.join(_hook_dir, "..", "..", "shared") + sys.path.insert(0, os.path.normpath(_shared_dir)) +try: + from harness_envelope import ( + COMPACTION_IMMINENT, + append_to_event_log, + create_envelope, + to_frontmatter_fields, + to_provenance_observations, + ) + _HAS_ENVELOPE = True +except ImportError: + _HAS_ENVELOPE = False + def command_argv(configured): """Preserve one literal executable path, otherwise parse a shell-style command.""" # Trigger: Windows paths commonly contain spaces and backslashes. @@ -122,6 +149,9 @@ def load_settings(directory): cfg = load_settings(cwd) primary_project = (cfg.get("primaryProject") or "").strip() capture_folder = (cfg.get("captureFolder") or "sessions").strip() +capture_events = bool(cfg.get("captureEvents", False)) +redact_keys = cfg.get("redactKeys") or [] +redact_paths = cfg.get("redactPaths") or [] # Trigger: no project pinned for this Claude Code project. # Why: a checkpoint must land somewhere intentional. Writing to the default graph @@ -223,6 +253,32 @@ frontmatter = [ ] if session_id: frontmatter.append(f"claude_session_id: {session_id}") + +# --- Harness envelope: stamp provenance and idempotency onto the checkpoint --- +# Trigger: the shared envelope module is available (always, unless the shared/ +# directory is missing). Why: provenance makes each checkpoint traceable +# to its source hook, session, and exact event. Idempotency prevents +# duplicate notes when the hook fires more than once in the same minute. +envelope = None +if _HAS_ENVELOPE: + try: + envelope = create_envelope( + event_type=COMPACTION_IMMINENT, + source="claude-code", + session_id=session_id or "unknown", + cwd=cwd, + project_hint=primary_project, + hook_name="PreCompact", + timestamp=iso, + payload_summary={"opening": clip(opening, 200)} if opening else {}, + redact_keys=redact_keys, + redact_paths=redact_paths, + ) + for key, value in to_frontmatter_fields(envelope).items(): + frontmatter.append(f"{key}: {value}") + except Exception: + pass # envelope creation failure is non-fatal + frontmatter += ["capture: extractive", "---"] body = [ @@ -247,6 +303,18 @@ body += [ "- [next_step] Review this checkpoint and continue where the thread left off", ] +# --- Append envelope provenance observations --- +# These stamp the note with its producer source so downstream consumers can +# trace provenance without storing the full raw event. +if _HAS_ENVELOPE and envelope: + body += to_provenance_observations(envelope) + +# --- Log the event locally for coalescing --- +# Trigger: captureEvents is enabled. Why: the local event log feeds future +# memory routines (SPEC-61) without requiring the note to carry every detail. +if _HAS_ENVELOPE and envelope and capture_events: + append_to_event_log(envelope, cwd) + content = "\n".join(frontmatter + body) # --- Write the checkpoint (best-effort) --- diff --git a/plugins/claude-code/hooks/session-start.sh b/plugins/claude-code/hooks/session-start.sh index dcaf08f8b..1ee504c26 100755 --- a/plugins/claude-code/hooks/session-start.sh +++ b/plugins/claude-code/hooks/session-start.sh @@ -48,7 +48,12 @@ fi # the brief. Python is a guaranteed dependency (basic-memory requires it) and # avoids brittle shell JSON wrangling. The payload and binary path cross over via # the environment to sidestep argument-quoting issues. -BM_HOOK_INPUT="$input" BM_BIN="$BM" python3 <<'PY' 2>/dev/null || exit 0 +# Resolve the hook script's own directory so the inline Python can find the +# shared envelope module. __file__ is '' inside a heredoc, so the Python +# code can't locate itself — we pass the real path. +hook_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" + +BM_HOOK_INPUT="$input" BM_BIN="$BM" BM_HOOK_DIR="$hook_dir" python3 <<'PY' 2>/dev/null || exit 0 import json import os import re @@ -57,6 +62,25 @@ import subprocess import sys from concurrent.futures import ThreadPoolExecutor +# --- Load the shared envelope module (lives two directories up in plugins/shared/) --- +# SessionStart is read-only (no note writes), so the envelope is only used for +# local event logging when captureEvents is enabled. +# Constraint: __file__ is '' inside a bash heredoc, so the hook script's +# real directory is passed in via the BM_HOOK_DIR environment variable. +_hook_dir = os.environ.get("BM_HOOK_DIR", "") +if _hook_dir: + _shared_dir = os.path.join(_hook_dir, "..", "..", "shared") + sys.path.insert(0, os.path.normpath(_shared_dir)) +try: + from harness_envelope import ( + SESSION_STARTED, + append_to_event_log, + create_envelope, + ) + _HAS_ENVELOPE = True +except ImportError: + _HAS_ENVELOPE = False + def command_argv(configured): """Preserve one literal executable path, otherwise parse a shell-style command.""" # Trigger: Windows paths commonly contain spaces and backslashes. @@ -154,6 +178,8 @@ recall_prompt = cfg.get("recallPrompt") or default_prompt # Without this, setup writes them but they never reach Claude (dead config). placement_conventions = (cfg.get("placementConventions") or "").strip() capture_folder = (cfg.get("captureFolder") or "sessions").strip() +capture_events = bool(cfg.get("captureEvents", False)) +session_id = payload.get("session_id") or "" # --- Resolve the shared/team read set --- # secondaryProjects (read-only recall sources) + teamProjects keys (share targets, @@ -330,4 +356,24 @@ elif not primary_project: lines += ["", "---", recall_prompt] print("\n".join(lines)) + +# --- Log the session_started event locally (opt-in) --- +# Trigger: captureEvents is enabled and the envelope module is available. +# Why: the local event log records session starts for later coalescing by memory +# routines (SPEC-61). This is separate from the brief printed above — it's +# durable metadata, not context for the current session. +# Outcome: a JSONL line is appended to /.basic-memory/events.jsonl. +if _HAS_ENVELOPE and capture_events and primary_project: + try: + envelope = create_envelope( + event_type=SESSION_STARTED, + source="claude-code", + session_id=session_id or "unknown", + cwd=cwd, + project_hint=primary_project, + hook_name="SessionStart", + ) + append_to_event_log(envelope, cwd) + except Exception: + pass # event logging failure is non-fatal PY diff --git a/plugins/claude-code/schemas/tool-ledger.md b/plugins/claude-code/schemas/tool-ledger.md new file mode 100644 index 000000000..0d7bae210 --- /dev/null +++ b/plugins/claude-code/schemas/tool-ledger.md @@ -0,0 +1,49 @@ +--- +title: Tool Ledger +type: schema +entity: ToolLedger +version: 1 +schema: + summary?: string, what tools were used and their overall outcome + tool_call?(array): string, tool name with abbreviated args summary + tool_result?(array): string, tool outcome summary (success or failure reason) + file_changed?(array): string, paths created or modified by tool calls + decision?(array): string, decisions made based on tool results +settings: + validation: warn + frontmatter: + project: string, the Basic Memory project this ledger belongs to + session_id?: string, harness session identifier + started: string, when the first tool call was recorded + ended?: string, when the last tool call was recorded + status?(enum, lifecycle of the ledger): [open, closed] + type: tool_ledger + source?: string, harness source (claude-code or codex) + idempotency_key?: string, dedup key from the producer envelope +--- + +# Tool Ledger + +A **ToolLedger** records the sequence of tool calls and their outcomes during +an agent session. It complements a SessionNote by capturing the *mechanical* +work — which tools were invoked, what files they touched, what failed — rather +than the *narrative* summary of what happened. + +ToolLedger notes are found by structured recall: +`search_notes(metadata_filters={"type": "tool_ledger"}, after_date="7d")`. + +## What Goes In A ToolLedger + +- **summary** — one paragraph of what the tool sequence accomplished. +- **tool_call** — each significant tool invocation with abbreviated arguments. +- **tool_result** — the outcome of each call (pass/fail/partial). +- **file_changed** — paths touched, useful for resume and conflict detection. +- **decision** — any decisions that emerged from tool results. + +## When It's Written + +V0 defines the schema for forward compatibility. Actual ToolLedger notes will +be produced when PostToolUse hooks become available (v1). For now, tool-level +observations can be included in SessionNote checkpoints. + +Validation is `warn` so ledger creation never blocks the user's flow. diff --git a/plugins/claude-code/settings.example.json b/plugins/claude-code/settings.example.json index a6432b165..8f0cfab2c 100644 --- a/plugins/claude-code/settings.example.json +++ b/plugins/claude-code/settings.example.json @@ -9,6 +9,10 @@ "recallPrompt": "You have Basic Memory available for this project. Before answering recall questions (\"what did we decide\", \"where did we leave off\"), search the graph first — prefer structured filters (search_notes with type/status). When the user makes a material decision, capture it as a note with type: decision. Cite permalinks when referencing prior work.", "preCompactCapture": "extractive", "placementConventions": null, + "captureEvents": false, + "redactKeys": [], + "redactPaths": [], + "eventRetention": 1000, "teamProjects": { "my-team/notes": { "promoteFolder": "shared" } } diff --git a/plugins/codex/hooks/pre-compact.py b/plugins/codex/hooks/pre-compact.py index ba996e1be..93870c300 100755 --- a/plugins/codex/hooks/pre-compact.py +++ b/plugins/codex/hooks/pre-compact.py @@ -11,6 +11,25 @@ from datetime import datetime, timezone from pathlib import Path +# --- Load the shared envelope module (lives two directories up in plugins/shared/) --- +# Trigger: this hook wants to stamp provenance and idempotency on the checkpoint. +# Why: the envelope normalizes hook events so downstream consumers (recall, +# consolidation, memory routines) can trace where each note came from. +_shared_dir = str(Path(__file__).resolve().parent.parent.parent / "shared") +sys.path.insert(0, _shared_dir) +try: + from harness_envelope import ( + COMPACTION_IMMINENT, + append_to_event_log, + create_envelope, + to_frontmatter_fields, + to_provenance_observations, + ) + + _HAS_ENVELOPE = True +except ImportError: + _HAS_ENVELOPE = False + UUID_RE = re.compile( r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", @@ -131,6 +150,9 @@ def main() -> int: cfg = load_config(cwd) primary_project = str(cfg.get("primaryProject") or "").strip() capture_folder = str(cfg.get("captureFolder") or "codex-sessions").strip() + capture_events = bool(cfg.get("captureEvents", False)) + redact_keys = cfg.get("redactKeys") or [] + redact_paths_cfg = cfg.get("redactPaths") or [] if not primary_project: return 0 @@ -167,6 +189,38 @@ def main() -> int: frontmatter.append(f"trigger: {trigger}") if model: frontmatter.append(f"model: {model}") + + # --- Harness envelope: stamp provenance and idempotency onto the checkpoint --- + # Trigger: the shared envelope module is available (always, unless the shared/ + # directory is missing). Why: provenance makes each checkpoint traceable + # to its source hook, session, and exact event. Idempotency prevents + # duplicate notes when the hook fires more than once in the same minute. + envelope = None + if _HAS_ENVELOPE: + try: + envelope = create_envelope( + event_type=COMPACTION_IMMINENT, + source="codex", + session_id=session_id or "unknown", + cwd=str(cwd), + project_hint=primary_project, + hook_name="PreCompact", + turn_id=turn_id or None, + timestamp=iso, + payload_summary={ + "opening": clip(opening, 200), + "trigger": trigger, + } + if opening + else {"trigger": trigger}, + redact_keys=redact_keys, + redact_paths=redact_paths_cfg, + ) + for key, value in to_frontmatter_fields(envelope).items(): + frontmatter.append(f"{key}: {value}") + except Exception: + pass # envelope creation failure is non-fatal + frontmatter += ["capture: extractive", "---"] body = [ @@ -198,6 +252,18 @@ def main() -> int: "continue from the latest user request", ] + # --- Append envelope provenance observations --- + # These stamp the note with its producer source so downstream consumers can + # trace provenance without storing the full raw event. + if _HAS_ENVELOPE and envelope: + body += to_provenance_observations(envelope) + + # --- Log the event locally for coalescing --- + # Trigger: captureEvents is enabled. Why: the local event log feeds future + # memory routines (SPEC-61) without requiring the note to carry every detail. + if _HAS_ENVELOPE and envelope and capture_events: + append_to_event_log(envelope, str(cwd)) + content = "\n".join(frontmatter + body) project_flag = "--project-id" if UUID_RE.match(primary_project) else "--project" diff --git a/plugins/codex/hooks/session-start.py b/plugins/codex/hooks/session-start.py index 1a401d25e..28cfa24fc 100755 --- a/plugins/codex/hooks/session-start.py +++ b/plugins/codex/hooks/session-start.py @@ -11,6 +11,22 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path +# --- Load the shared envelope module (lives two directories up in plugins/shared/) --- +# SessionStart is read-only (no note writes), so the envelope is only used for +# local event logging when captureEvents is enabled. +_shared_dir = str(Path(__file__).resolve().parent.parent.parent / "shared") +sys.path.insert(0, _shared_dir) +try: + from harness_envelope import ( + SESSION_STARTED, + append_to_event_log, + create_envelope, + ) + + _HAS_ENVELOPE = True +except ImportError: + _HAS_ENVELOPE = False + UUID_RE = re.compile( r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", @@ -141,6 +157,7 @@ def main() -> int: capture_folder = str(cfg.get("captureFolder") or "codex-sessions").strip() placement = str(cfg.get("placementConventions") or "").strip() focus = str(cfg.get("focus") or "").strip() + capture_events = bool(cfg.get("captureEvents", False)) shared_refs, shared_capped = shared_project_refs(cfg, primary_project) active_tasks = ["--type", "task", "--status", "active"] @@ -230,6 +247,28 @@ def main() -> int: ] print("\n".join(lines)) + + # --- Log the session_started event locally (opt-in) --- + # Trigger: captureEvents is enabled and the envelope module is available. + # Why: the local event log records session starts for later coalescing by + # memory routines (SPEC-61). This is separate from the brief printed + # above — it's durable metadata, not context for the current session. + # Outcome: a JSONL line is appended to /.basic-memory/events.jsonl. + if _HAS_ENVELOPE and capture_events and primary_project: + try: + session_id = payload.get("session_id") or "" + envelope = create_envelope( + event_type=SESSION_STARTED, + source="codex", + session_id=session_id or "unknown", + cwd=str(cwd), + project_hint=primary_project, + hook_name="SessionStart", + ) + append_to_event_log(envelope, str(cwd)) + except Exception: + pass # event logging failure is non-fatal + return 0 diff --git a/plugins/codex/schemas/tool-ledger.md b/plugins/codex/schemas/tool-ledger.md new file mode 100644 index 000000000..c2711f74a --- /dev/null +++ b/plugins/codex/schemas/tool-ledger.md @@ -0,0 +1,49 @@ +--- +title: Tool Ledger +type: schema +entity: ToolLedger +version: 1 +schema: + summary?: string, what tools were used and their overall outcome + tool_call?(array): string, tool name with abbreviated args summary + tool_result?(array): string, tool outcome summary (success or failure reason) + file_changed?(array): string, paths created or modified by tool calls + decision?(array): string, decisions made based on tool results +settings: + validation: warn + frontmatter: + project: string, the Basic Memory project this ledger belongs to + session_id?: string, harness session identifier + started: string, when the first tool call was recorded + ended?: string, when the last tool call was recorded + status?(enum, lifecycle of the ledger): [open, closed] + type: tool_ledger + source?: string, harness source (claude-code or codex) + idempotency_key?: string, dedup key from the producer envelope +--- + +# Tool Ledger + +A **ToolLedger** records the sequence of tool calls and their outcomes during +an agent session. It complements a SessionNote by capturing the *mechanical* +work — which tools were invoked, what files they touched, what failed — rather +than the *narrative* summary of what happened. + +ToolLedger notes are found by structured recall: +`search_notes(metadata_filters={"type": "tool_ledger"}, after_date="7d")`. + +## What Goes In A ToolLedger + +- **summary** — one paragraph of what the tool sequence accomplished. +- **tool_call** — each significant tool invocation with abbreviated arguments. +- **tool_result** — the outcome of each call (pass/fail/partial). +- **file_changed** — paths touched, useful for resume and conflict detection. +- **decision** — any decisions that emerged from tool results. + +## When It's Written + +V0 defines the schema for forward compatibility. Actual ToolLedger notes will +be produced when PostToolUse hooks become available (v1). For now, tool-level +observations can be included in CodexSession checkpoints. + +Validation is `warn` so ledger creation never blocks the user's flow. diff --git a/plugins/shared/harness_envelope.py b/plugins/shared/harness_envelope.py new file mode 100644 index 000000000..9eb2a617a --- /dev/null +++ b/plugins/shared/harness_envelope.py @@ -0,0 +1,302 @@ +"""Normalized harness-event envelope for Claude Code and Codex plugins. + +This module is the producer side of the harness WAL (issue #997, SPEC-55). +It normalizes supported hook events into a shared envelope shape and provides +helpers for idempotency, redaction, and coalescing into Basic Memory artifacts. + +Design constraints: + - Stdlib only — no third-party imports. Both Claude Code (inline Python inside + bash) and Codex (standalone scripts) must import this without an install step. + - Never captures private model reasoning or hidden chain-of-thought. + - Prefers summaries and metadata over raw transcript dumps. + - Fails fast on missing project mapping rather than writing to the wrong project. + +Usage from a hook script: + + import sys, os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "shared")) + from harness_envelope import create_envelope, to_provenance_observations + + envelope = create_envelope( + event_type="compaction_imminent", + source="claude-code", + session_id=session_id, + cwd=cwd, + project_hint=primary_project, + hook_name="PreCompact", + payload_summary={"opening": clip(opening, 200)}, + ) + provenance_lines = to_provenance_observations(envelope) +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +# --- Event type constants --- +# V0 only captures events exposed through supported harness hooks. +# Future versions may add tool_called, file_changed, test_ran, etc. +# when PostToolUse hooks become available. + +SESSION_STARTED = "session_started" +COMPACTION_IMMINENT = "compaction_imminent" +SESSION_ENDED = "session_ended" + +V0_EVENT_TYPES = frozenset({SESSION_STARTED, COMPACTION_IMMINENT, SESSION_ENDED}) + +# --- Redaction defaults --- +# Keys whose values look like secrets. Matched case-insensitively against +# payload dict keys. Uses explicit patterns for common secret key naming +# conventions. The user can extend this via config (extra_redact_keys). +# +# Strategy: match keys that contain well-known secret indicators as full +# word segments (delimited by _ or . or at string boundaries). This catches +# API_KEY, AUTH_TOKEN, DB_PASSWORD but not "safe_key" or "monkey". +DEFAULT_REDACT_KEY_PATTERNS = ( + re.compile(r"(?i)(?:^|[_.])(?:SECRET|TOKEN|PASSWORD|CREDENTIAL|AUTH)(?:[_.]|$)"), + re.compile(r"(?i)(?:^|[_.])(?:API[_.]KEY|ACCESS[_.]KEY|PRIVATE[_.]KEY)(?:[_.]|$)"), +) + +# Paths that should never appear in payload summaries. +DEFAULT_REDACT_PATHS = ( + os.path.expanduser("~/.ssh/"), + os.path.expanduser("~/.aws/"), + os.path.expanduser("~/.gnupg/"), +) + +# Values that look like environment secrets: KEY= +SECRET_VALUE_RE = re.compile(r"^[A-Za-z0-9_]+=.{20,}$") + +# Maximum length for any single payload value before truncation. +MAX_PAYLOAD_VALUE_LEN = 500 + +# Maximum event log entries before rotation. +DEFAULT_EVENT_LOG_CAP = 1000 + + +@dataclass(frozen=True) +class HarnessEnvelope: + """Normalized event record from a Claude Code or Codex harness hook. + + This is the producer envelope shape from SPEC-55. Each field is chosen to + be useful for coalescing into SessionNote / ToolLedger artifacts without + requiring the downstream consumer to understand the raw hook payload format. + """ + + event_type: str + source: str # "claude-code" or "codex" + session_id: str + turn_id: str | None + timestamp: str # ISO 8601 + cwd: str + project_hint: str # basicMemory.primaryProject + hook_name: str # "SessionStart", "PreCompact" + idempotency_key: str + payload_summary: dict = field(default_factory=dict) + + +def idempotency_key( + source: str, + session_id: str, + hook_name: str, + timestamp: str, +) -> str: + """Generate a deterministic key from (source, session_id, hook, timestamp_minute). + + Minute granularity means that repeated hooks within the same minute for the + same session+hook combination produce the same key — preventing duplicate + notes without requiring persistent state. Two hooks one minute apart get + distinct keys, which is the right behavior (a second compaction a minute + later is a genuinely new event). + """ + # Truncate timestamp to minute precision for dedup window. + # Handles both ISO format (2026-06-13T16:48:00+00:00) and plain timestamps. + minute_key = timestamp[:16] # "2026-06-13T16:48" + raw = f"{source}:{session_id}:{hook_name}:{minute_key}" + return hashlib.sha256(raw.encode()).hexdigest()[:16] + + +def redact_payload( + payload: dict, + extra_redact_keys: list[str] | None = None, + extra_redact_paths: list[str] | None = None, +) -> dict: + """Strip secrets, large values, and denied paths from a payload summary. + + Returns a new dict with sensitive content replaced by "[REDACTED]" markers. + This is the safety layer: nothing downstream sees unredacted payload values. + """ + deny_key_patterns = list(DEFAULT_REDACT_KEY_PATTERNS) + if extra_redact_keys: + for pattern in extra_redact_keys: + try: + deny_key_patterns.append(re.compile(re.escape(pattern), re.IGNORECASE)) + except re.error: + continue + + deny_paths = list(DEFAULT_REDACT_PATHS) + if extra_redact_paths: + deny_paths.extend(extra_redact_paths) + + result = {} + for key, value in payload.items(): + # Check key against deny patterns + if any(pat.search(key) for pat in deny_key_patterns): + result[key] = "[REDACTED]" + continue + + if isinstance(value, str): + # Check for environment-secret-like values + if SECRET_VALUE_RE.match(value): + result[key] = "[REDACTED]" + continue + + # Check for denied paths + if any(value.startswith(p) for p in deny_paths): + result[key] = "[REDACTED_PATH]" + continue + + # Truncate overly long values + if len(value) > MAX_PAYLOAD_VALUE_LEN: + result[key] = value[:MAX_PAYLOAD_VALUE_LEN] + "…[truncated]" + continue + + result[key] = value + + return result + + +def create_envelope( + *, + event_type: str, + source: str, + session_id: str, + cwd: str, + project_hint: str, + hook_name: str, + turn_id: str | None = None, + timestamp: str | None = None, + payload_summary: dict | None = None, + redact_keys: list[str] | None = None, + redact_paths: list[str] | None = None, +) -> HarnessEnvelope: + """Factory: build a normalized envelope from hook inputs. + + All arguments are keyword-only to prevent positional-order mistakes in + hook scripts that construct envelopes from heterogeneous payload shapes. + """ + if event_type not in V0_EVENT_TYPES: + raise ValueError( + f"Unknown event type {event_type!r}; v0 supports: {sorted(V0_EVENT_TYPES)}" + ) + + ts = timestamp or datetime.now(timezone.utc).isoformat(timespec="seconds") + safe_payload = redact_payload( + payload_summary or {}, + extra_redact_keys=redact_keys, + extra_redact_paths=redact_paths, + ) + + idem_key = idempotency_key(source, session_id, hook_name, ts) + + return HarnessEnvelope( + event_type=event_type, + source=source, + session_id=session_id, + turn_id=turn_id, + timestamp=ts, + cwd=cwd, + project_hint=project_hint, + hook_name=hook_name, + idempotency_key=idem_key, + payload_summary=safe_payload, + ) + + +def to_provenance_observations(envelope: HarnessEnvelope) -> list[str]: + """Convert an envelope into observation lines for a SessionNote body. + + These are appended to the "## Observations" section of the note. They + stamp the note with its producer source so downstream consumers (recall, + consolidation, memory routines) can trace provenance without storing the + full raw event. + """ + lines = [ + f"- [source] {envelope.source}/{envelope.session_id}", + f"- [hook] {envelope.hook_name}", + f"- [event] {envelope.event_type} at {envelope.timestamp}", + f"- [idempotency] {envelope.idempotency_key}", + ] + if envelope.turn_id: + lines.append(f"- [turn] {envelope.turn_id}") + return lines + + +def to_frontmatter_fields(envelope: HarnessEnvelope) -> dict[str, str]: + """Extract envelope fields suitable for inclusion in note frontmatter. + + These are added alongside the existing session/codex_session frontmatter + to make the note queryable by source, hook, and idempotency key. + """ + fields = { + "envelope_source": envelope.source, + "envelope_event": envelope.event_type, + "envelope_hook": envelope.hook_name, + "idempotency_key": envelope.idempotency_key, + } + if envelope.turn_id: + fields["envelope_turn_id"] = envelope.turn_id + return fields + + +def envelope_to_json(envelope: HarnessEnvelope) -> str: + """Serialize an envelope to a compact JSON string for event log storage.""" + return json.dumps(asdict(envelope), separators=(",", ":")) + + +def append_to_event_log( + envelope: HarnessEnvelope, + cwd: str, + cap: int = DEFAULT_EVENT_LOG_CAP, +) -> bool: + """Append a serialized envelope to the local event log. + + The event log lives at /.basic-memory/events.jsonl and stores raw + envelopes for later coalescing by memory routines (SPEC-61). The log is + capped at `cap` lines; when exceeded, the oldest half is rotated out. + + Returns True if the write succeeded, False on any error (best-effort). + """ + log_dir = Path(cwd) / ".basic-memory" + log_path = log_dir / "events.jsonl" + + try: + log_dir.mkdir(parents=True, exist_ok=True) + + # Append the new event + line = envelope_to_json(envelope) + "\n" + with open(log_path, "a") as fh: + fh.write(line) + + # --- Rotation check --- + # Trigger: log exceeds the cap. Why: unbounded growth would fill disk + # on long-running projects. Outcome: keep the newest half. + try: + with open(log_path) as fh: + lines = fh.readlines() + if len(lines) > cap: + keep = lines[len(lines) // 2 :] + with open(log_path, "w") as fh: + fh.writelines(keep) + except Exception: + pass # rotation failure is non-fatal + + return True + except Exception: + return False From 24536d317fc4b79eb0811a9cf29f6d77b2498cce Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 15 Jul 2026 08:36:09 -0500 Subject: [PATCH 2/8] fix(plugins): redact nested harness payload values recursively redact_payload() only inspected top-level string values, so a secret inside a nested dict or list passed through unredacted. Redaction now recurses over dicts, lists, and tuples; a denied key redacts its whole subtree rather than risking partial redaction inside it. Addresses maintainer review on #998. Co-Authored-By: Claude Fable 5 Signed-off-by: phernandez --- plugins/claude-code/DESIGN.md | 4 +- plugins/shared/harness_envelope.py | 70 ++++++++++++++++++------------ 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/plugins/claude-code/DESIGN.md b/plugins/claude-code/DESIGN.md index 6e6f4473d..6ebbddd40 100644 --- a/plugins/claude-code/DESIGN.md +++ b/plugins/claude-code/DESIGN.md @@ -709,7 +709,9 @@ New keys in `basicMemory` (both `.claude/settings.json` and `.codex/basic-memory ### 14.7 Privacy and redaction -The envelope module applies layered redaction before any payload summary is stored: +The envelope module applies layered redaction before any payload summary is +stored, recursively over nested dicts and lists — a secret nested inside a list +or sub-object is redacted exactly like a top-level one: 1. **Key-pattern deny list** — keys matching `SECRET`, `TOKEN`, `KEY`, `PASSWORD`, `CREDENTIAL`, `AUTH` (case-insensitive) are replaced with `[REDACTED]` diff --git a/plugins/shared/harness_envelope.py b/plugins/shared/harness_envelope.py index 9eb2a617a..1729f686b 100644 --- a/plugins/shared/harness_envelope.py +++ b/plugins/shared/harness_envelope.py @@ -122,6 +122,44 @@ def idempotency_key( return hashlib.sha256(raw.encode()).hexdigest()[:16] +def _redact_str(value: str, deny_paths: list[str]) -> str: + """Apply the string-level redaction rules: secret values, paths, truncation.""" + if SECRET_VALUE_RE.match(value): + return "[REDACTED]" + if any(value.startswith(p) for p in deny_paths): + return "[REDACTED_PATH]" + if len(value) > MAX_PAYLOAD_VALUE_LEN: + return value[:MAX_PAYLOAD_VALUE_LEN] + "…[truncated]" + return value + + +def _redact_value(value, deny_key_patterns: list, deny_paths: list[str]): + """Recursively redact a payload value of any JSON-compatible shape. + + Payloads arrive from hook JSON, so nested dicts and lists are normal — + a secret one level down must be caught just like a top-level one. + """ + if isinstance(value, str): + return _redact_str(value, deny_paths) + if isinstance(value, dict): + return _redact_dict(value, deny_key_patterns, deny_paths) + if isinstance(value, (list, tuple)): + return [_redact_value(item, deny_key_patterns, deny_paths) for item in value] + return value + + +def _redact_dict(payload: dict, deny_key_patterns: list, deny_paths: list[str]) -> dict: + result = {} + for key, value in payload.items(): + # A denied key redacts the whole value, however deeply nested it is — + # partial redaction inside a secret-named subtree is not worth the risk. + if any(pat.search(str(key)) for pat in deny_key_patterns): + result[key] = "[REDACTED]" + continue + result[key] = _redact_value(value, deny_key_patterns, deny_paths) + return result + + def redact_payload( payload: dict, extra_redact_keys: list[str] | None = None, @@ -129,8 +167,9 @@ def redact_payload( ) -> dict: """Strip secrets, large values, and denied paths from a payload summary. - Returns a new dict with sensitive content replaced by "[REDACTED]" markers. - This is the safety layer: nothing downstream sees unredacted payload values. + Returns a new dict with sensitive content replaced by "[REDACTED]" markers, + applied recursively over nested dicts and lists. This is the safety layer: + nothing downstream sees unredacted payload values at any depth. """ deny_key_patterns = list(DEFAULT_REDACT_KEY_PATTERNS) if extra_redact_keys: @@ -144,32 +183,7 @@ def redact_payload( if extra_redact_paths: deny_paths.extend(extra_redact_paths) - result = {} - for key, value in payload.items(): - # Check key against deny patterns - if any(pat.search(key) for pat in deny_key_patterns): - result[key] = "[REDACTED]" - continue - - if isinstance(value, str): - # Check for environment-secret-like values - if SECRET_VALUE_RE.match(value): - result[key] = "[REDACTED]" - continue - - # Check for denied paths - if any(value.startswith(p) for p in deny_paths): - result[key] = "[REDACTED_PATH]" - continue - - # Truncate overly long values - if len(value) > MAX_PAYLOAD_VALUE_LEN: - result[key] = value[:MAX_PAYLOAD_VALUE_LEN] + "…[truncated]" - continue - - result[key] = value - - return result + return _redact_dict(payload, deny_key_patterns, deny_paths) def create_envelope( From 1f45bff365b1d7fb5bfca7393aafcfd5632052cb Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 15 Jul 2026 08:37:44 -0500 Subject: [PATCH 3/8] fix(plugins): move harness event log out of the user repo append_to_event_log() wrote /.basic-memory/events.jsonl into whatever repository the session ran in, dirtying user worktrees. The log now lives under the Basic Memory data dir at /events//events.jsonl, resolved the same way as core's resolve_data_dir() (BASIC_MEMORY_CONFIG_DIR, then XDG_CONFIG_HOME, then ~/.basic-memory) without importing core. Rotation previously read the whole file on every hook write, which contradicted the documented append-only behavior. The hot path is now one append plus one stat; rotation only runs when the file passes cap x 512 bytes and drops the oldest half, so retention is documented as approximate. All open() calls pin encoding=utf-8 so envelopes read back identically across platform default encodings. DESIGN.md notes that a future version may write events through Basic Memory MCP/API calls instead of a local file (#997 leaves that open). Addresses maintainer review on #998. Co-Authored-By: Claude Fable 5 Signed-off-by: phernandez --- plugins/claude-code/DESIGN.md | 17 +++- plugins/claude-code/hooks/pre-compact.sh | 2 +- plugins/claude-code/hooks/session-start.sh | 5 +- plugins/codex/hooks/pre-compact.py | 2 +- plugins/codex/hooks/session-start.py | 5 +- plugins/shared/harness_envelope.py | 90 ++++++++++++++++++---- 6 files changed, 94 insertions(+), 27 deletions(-) diff --git a/plugins/claude-code/DESIGN.md b/plugins/claude-code/DESIGN.md index 6ebbddd40..4295f2e8d 100644 --- a/plugins/claude-code/DESIGN.md +++ b/plugins/claude-code/DESIGN.md @@ -689,13 +689,22 @@ PreCompact checkpoints gain two additions: ### 14.5 Local event log When `captureEvents: true` is set, both SessionStart and PreCompact append a JSONL -record to `/.basic-memory/events.jsonl`. This log: - -- Is append-only (no reads during hook execution) -- Is capped at 1000 lines (configurable via `eventRetention`); oldest half rotates out +record to the project's event log under the Basic Memory data dir: +`/events//events.jsonl`, where `` follows +core's `resolve_data_dir()` (`BASIC_MEMORY_CONFIG_DIR`, then `XDG_CONFIG_HOME`, +then `~/.basic-memory`). The log deliberately does **not** live in the working +directory — capture must never dirty the user's repository. This log: + +- Keeps the hook hot path cheap: each write is one append plus one `stat()`. + Rotation (drop the oldest half, bounded at `eventRetention` lines) only runs + when the file passes `eventRetention × 512` bytes, so retention is approximate. - Feeds future SPEC-61 memory routines (nightly coalescing, session summaries) - Never blocks the hook — write failures are silently swallowed +A future version may write events through Basic Memory MCP/API calls instead of a +local file — issue #997 explicitly leaves that open. The envelope shape is the +contract; the transport can change without touching the hooks' capture logic. + ### 14.6 Configuration New keys in `basicMemory` (both `.claude/settings.json` and `.codex/basic-memory.json`): diff --git a/plugins/claude-code/hooks/pre-compact.sh b/plugins/claude-code/hooks/pre-compact.sh index 95117d437..7ed316af4 100755 --- a/plugins/claude-code/hooks/pre-compact.sh +++ b/plugins/claude-code/hooks/pre-compact.sh @@ -313,7 +313,7 @@ if _HAS_ENVELOPE and envelope: # Trigger: captureEvents is enabled. Why: the local event log feeds future # memory routines (SPEC-61) without requiring the note to carry every detail. if _HAS_ENVELOPE and envelope and capture_events: - append_to_event_log(envelope, cwd) + append_to_event_log(envelope) content = "\n".join(frontmatter + body) diff --git a/plugins/claude-code/hooks/session-start.sh b/plugins/claude-code/hooks/session-start.sh index 1ee504c26..384bbb4c1 100755 --- a/plugins/claude-code/hooks/session-start.sh +++ b/plugins/claude-code/hooks/session-start.sh @@ -362,7 +362,8 @@ print("\n".join(lines)) # Why: the local event log records session starts for later coalescing by memory # routines (SPEC-61). This is separate from the brief printed above — it's # durable metadata, not context for the current session. -# Outcome: a JSONL line is appended to /.basic-memory/events.jsonl. +# Outcome: a JSONL line is appended to the project's event log under the +# Basic Memory data dir (never inside the user's repository). if _HAS_ENVELOPE and capture_events and primary_project: try: envelope = create_envelope( @@ -373,7 +374,7 @@ if _HAS_ENVELOPE and capture_events and primary_project: project_hint=primary_project, hook_name="SessionStart", ) - append_to_event_log(envelope, cwd) + append_to_event_log(envelope) except Exception: pass # event logging failure is non-fatal PY diff --git a/plugins/codex/hooks/pre-compact.py b/plugins/codex/hooks/pre-compact.py index 93870c300..30084e849 100755 --- a/plugins/codex/hooks/pre-compact.py +++ b/plugins/codex/hooks/pre-compact.py @@ -262,7 +262,7 @@ def main() -> int: # Trigger: captureEvents is enabled. Why: the local event log feeds future # memory routines (SPEC-61) without requiring the note to carry every detail. if _HAS_ENVELOPE and envelope and capture_events: - append_to_event_log(envelope, str(cwd)) + append_to_event_log(envelope) content = "\n".join(frontmatter + body) project_flag = "--project-id" if UUID_RE.match(primary_project) else "--project" diff --git a/plugins/codex/hooks/session-start.py b/plugins/codex/hooks/session-start.py index 28cfa24fc..4f4949620 100755 --- a/plugins/codex/hooks/session-start.py +++ b/plugins/codex/hooks/session-start.py @@ -253,7 +253,8 @@ def main() -> int: # Why: the local event log records session starts for later coalescing by # memory routines (SPEC-61). This is separate from the brief printed # above — it's durable metadata, not context for the current session. - # Outcome: a JSONL line is appended to /.basic-memory/events.jsonl. + # Outcome: a JSONL line is appended to the project's event log under the + # Basic Memory data dir (never inside the user's repository). if _HAS_ENVELOPE and capture_events and primary_project: try: session_id = payload.get("session_id") or "" @@ -265,7 +266,7 @@ def main() -> int: project_hint=primary_project, hook_name="SessionStart", ) - append_to_event_log(envelope, str(cwd)) + append_to_event_log(envelope) except Exception: pass # event logging failure is non-fatal diff --git a/plugins/shared/harness_envelope.py b/plugins/shared/harness_envelope.py index 1729f686b..3656e1f25 100644 --- a/plugins/shared/harness_envelope.py +++ b/plugins/shared/harness_envelope.py @@ -79,6 +79,11 @@ # Maximum event log entries before rotation. DEFAULT_EVENT_LOG_CAP = 1000 +# Rotation is size-triggered (a single stat per append), so the cap in lines is +# converted to a byte threshold using this conservative per-entry estimate. +# Envelope JSON lines run ~300-600 bytes (payload values truncate at 500 chars). +APPROX_BYTES_PER_EVENT = 512 + @dataclass(frozen=True) class HarnessEnvelope: @@ -274,39 +279,90 @@ def envelope_to_json(envelope: HarnessEnvelope) -> str: return json.dumps(asdict(envelope), separators=(",", ":")) +def _events_root() -> Path: + """Resolve the root directory for local event logs. + + Mirrors core's ``basic_memory.config.resolve_data_dir()`` (stdlib-only, so + no import): ``BASIC_MEMORY_CONFIG_DIR`` first, then ``XDG_CONFIG_HOME``, + then ``~/.basic-memory``. Event logs live under the per-user Basic Memory + data dir — never inside the working directory, where they would dirty the + user's repository. + """ + if config_dir := os.environ.get("BASIC_MEMORY_CONFIG_DIR"): + return Path(config_dir) / "events" + if xdg_config := os.environ.get("XDG_CONFIG_HOME"): + return Path(xdg_config) / "basic-memory" / "events" + return Path.home() / ".basic-memory" / "events" + + +def _event_log_slug(project_hint: str, cwd: str) -> str: + """Directory name isolating one project's event log from another's. + + Prefers the configured project (stable across checkouts of the same + project); falls back to the working directory. A short digest of the raw + seed disambiguates values that slug identically ("my/proj" vs "my-proj"). + """ + seed = (project_hint or "").strip() or cwd + slug = re.sub(r"[^a-z0-9]+", "-", seed.lower()).strip("-") or "default" + digest = hashlib.sha256(seed.encode("utf-8")).hexdigest()[:8] + return f"{slug[:48]}-{digest}" + + +def event_log_path(envelope: HarnessEnvelope) -> Path: + """Compute where this envelope's event log lives (see _events_root).""" + return _events_root() / _event_log_slug(envelope.project_hint, envelope.cwd) / "events.jsonl" + + +def _normalize_cap(cap) -> int: + # Trigger: eventRetention comes straight from user JSON config. + # Why: a junk value must neither break best-effort logging nor unbound the + # log. Outcome: positive ints win; anything else uses the default cap. + if isinstance(cap, bool) or not isinstance(cap, int) or cap <= 0: + return DEFAULT_EVENT_LOG_CAP + return cap + + def append_to_event_log( envelope: HarnessEnvelope, - cwd: str, - cap: int = DEFAULT_EVENT_LOG_CAP, + cap: int | None = None, ) -> bool: """Append a serialized envelope to the local event log. - The event log lives at /.basic-memory/events.jsonl and stores raw - envelopes for later coalescing by memory routines (SPEC-61). The log is - capped at `cap` lines; when exceeded, the oldest half is rotated out. + The event log lives under the Basic Memory data dir at + ``/events//events.jsonl`` and stores raw + envelopes for later coalescing by memory routines (SPEC-61). ``cap`` is the + approximate retention limit in entries (the ``eventRetention`` setting); + ``None`` uses DEFAULT_EVENT_LOG_CAP. + + The hot path stays cheap: one append plus one stat. Rotation only runs when + the file size passes ``cap * APPROX_BYTES_PER_EVENT`` bytes; it then drops + the oldest half (bounded at ``cap`` lines), so the file halves and the next + many appends are stat-only again. Returns True if the write succeeded, False on any error (best-effort). """ - log_dir = Path(cwd) / ".basic-memory" - log_path = log_dir / "events.jsonl" + cap = _normalize_cap(cap) + log_path = event_log_path(envelope) try: - log_dir.mkdir(parents=True, exist_ok=True) + log_path.parent.mkdir(parents=True, exist_ok=True) # Append the new event line = envelope_to_json(envelope) + "\n" - with open(log_path, "a") as fh: + with open(log_path, "a", encoding="utf-8") as fh: fh.write(line) - # --- Rotation check --- - # Trigger: log exceeds the cap. Why: unbounded growth would fill disk - # on long-running projects. Outcome: keep the newest half. + # --- Rotation check (stat-only on the hot path) --- + # Trigger: log size passes the byte threshold derived from the cap. + # Why: unbounded growth would fill disk on long-running projects, but + # counting lines on every hook write would make the hot path O(n). + # Outcome: the oldest half rotates out; retention is approximate. try: - with open(log_path) as fh: - lines = fh.readlines() - if len(lines) > cap: - keep = lines[len(lines) // 2 :] - with open(log_path, "w") as fh: + if log_path.stat().st_size > cap * APPROX_BYTES_PER_EVENT: + with open(log_path, encoding="utf-8") as fh: + lines = fh.readlines() + keep = lines[len(lines) // 2 :][-cap:] + with open(log_path, "w", encoding="utf-8") as fh: fh.writelines(keep) except Exception: pass # rotation failure is non-fatal From 563f84a5e78ea72266c92dcf3dd4c96916e25a14 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 15 Jul 2026 08:38:20 -0500 Subject: [PATCH 4/8] fix(plugins): wire eventRetention through to event log rotation The eventRetention setting was documented in settings.example.json and DESIGN.md but never read: every call site logged with the hard-coded default cap. All four hooks (Claude Code and Codex SessionStart and PreCompact) now pass the configured value through to append_to_event_log(), which validates it and falls back to the default for junk or non-positive values. Addresses maintainer review on #998. Co-Authored-By: Claude Fable 5 Signed-off-by: phernandez --- plugins/claude-code/hooks/pre-compact.sh | 5 ++++- plugins/claude-code/hooks/session-start.sh | 5 ++++- plugins/codex/hooks/pre-compact.py | 5 ++++- plugins/codex/hooks/session-start.py | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/plugins/claude-code/hooks/pre-compact.sh b/plugins/claude-code/hooks/pre-compact.sh index 7ed316af4..c4729f214 100755 --- a/plugins/claude-code/hooks/pre-compact.sh +++ b/plugins/claude-code/hooks/pre-compact.sh @@ -152,6 +152,9 @@ capture_folder = (cfg.get("captureFolder") or "sessions").strip() capture_events = bool(cfg.get("captureEvents", False)) redact_keys = cfg.get("redactKeys") or [] redact_paths = cfg.get("redactPaths") or [] +# Approximate retention cap for the local event log; the envelope module +# validates it (non-positive/junk values fall back to its default). +event_retention = cfg.get("eventRetention") # Trigger: no project pinned for this Claude Code project. # Why: a checkpoint must land somewhere intentional. Writing to the default graph @@ -313,7 +316,7 @@ if _HAS_ENVELOPE and envelope: # Trigger: captureEvents is enabled. Why: the local event log feeds future # memory routines (SPEC-61) without requiring the note to carry every detail. if _HAS_ENVELOPE and envelope and capture_events: - append_to_event_log(envelope) + append_to_event_log(envelope, cap=event_retention) content = "\n".join(frontmatter + body) diff --git a/plugins/claude-code/hooks/session-start.sh b/plugins/claude-code/hooks/session-start.sh index 384bbb4c1..5e6ea870e 100755 --- a/plugins/claude-code/hooks/session-start.sh +++ b/plugins/claude-code/hooks/session-start.sh @@ -179,6 +179,9 @@ recall_prompt = cfg.get("recallPrompt") or default_prompt placement_conventions = (cfg.get("placementConventions") or "").strip() capture_folder = (cfg.get("captureFolder") or "sessions").strip() capture_events = bool(cfg.get("captureEvents", False)) +# Approximate retention cap for the local event log; the envelope module +# validates it (non-positive/junk values fall back to its default). +event_retention = cfg.get("eventRetention") session_id = payload.get("session_id") or "" # --- Resolve the shared/team read set --- @@ -374,7 +377,7 @@ if _HAS_ENVELOPE and capture_events and primary_project: project_hint=primary_project, hook_name="SessionStart", ) - append_to_event_log(envelope) + append_to_event_log(envelope, cap=event_retention) except Exception: pass # event logging failure is non-fatal PY diff --git a/plugins/codex/hooks/pre-compact.py b/plugins/codex/hooks/pre-compact.py index 30084e849..c129b050f 100755 --- a/plugins/codex/hooks/pre-compact.py +++ b/plugins/codex/hooks/pre-compact.py @@ -153,6 +153,9 @@ def main() -> int: capture_events = bool(cfg.get("captureEvents", False)) redact_keys = cfg.get("redactKeys") or [] redact_paths_cfg = cfg.get("redactPaths") or [] + # Approximate retention cap for the local event log; the envelope module + # validates it (non-positive/junk values fall back to its default). + event_retention = cfg.get("eventRetention") if not primary_project: return 0 @@ -262,7 +265,7 @@ def main() -> int: # Trigger: captureEvents is enabled. Why: the local event log feeds future # memory routines (SPEC-61) without requiring the note to carry every detail. if _HAS_ENVELOPE and envelope and capture_events: - append_to_event_log(envelope) + append_to_event_log(envelope, cap=event_retention) content = "\n".join(frontmatter + body) project_flag = "--project-id" if UUID_RE.match(primary_project) else "--project" diff --git a/plugins/codex/hooks/session-start.py b/plugins/codex/hooks/session-start.py index 4f4949620..aaf6596c0 100755 --- a/plugins/codex/hooks/session-start.py +++ b/plugins/codex/hooks/session-start.py @@ -158,6 +158,9 @@ def main() -> int: placement = str(cfg.get("placementConventions") or "").strip() focus = str(cfg.get("focus") or "").strip() capture_events = bool(cfg.get("captureEvents", False)) + # Approximate retention cap for the local event log; the envelope module + # validates it (non-positive/junk values fall back to its default). + event_retention = cfg.get("eventRetention") shared_refs, shared_capped = shared_project_refs(cfg, primary_project) active_tasks = ["--type", "task", "--status", "active"] @@ -266,7 +269,7 @@ def main() -> int: project_hint=primary_project, hook_name="SessionStart", ) - append_to_event_log(envelope) + append_to_event_log(envelope, cap=event_retention) except Exception: pass # event logging failure is non-fatal From f399263f22272b992d1da2385021a5c0f38bc756 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 15 Jul 2026 08:40:22 -0500 Subject: [PATCH 5/8] fix(plugins): vendor harness envelope module into installed plugins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plugin marketplaces install a single plugin directory (plugins/ claude-code/ or plugins/codex/); nothing outside that tree ships. The hooks imported harness_envelope from plugins/shared/ two directories up, which only resolves in a repo checkout — on every installed plugin the import silently failed (_HAS_ENVELOPE=False) and users got no envelope capture at all. plugins/shared/ stays the canonical source. A new scripts/sync_plugin_shared.py vendors each shared module into every plugin's hooks/ directory as a committed copy, and the hooks now import from their own directory, which resolves identically in the repo checkout and the installed layout. Both plugin justfiles gain a vendor-check recipe (wired into check/ci-check, and therefore just package-check) that byte-compares the vendored copies against the canonical source so drift fails CI. Addresses maintainer review on #998. Co-Authored-By: Claude Fable 5 Signed-off-by: phernandez --- plugins/claude-code/DESIGN.md | 7 +- plugins/claude-code/hooks/harness_envelope.py | 375 ++++++++++++++++++ plugins/claude-code/hooks/pre-compact.sh | 8 +- plugins/claude-code/hooks/session-start.sh | 8 +- plugins/claude-code/justfile | 8 +- plugins/codex/hooks/harness_envelope.py | 375 ++++++++++++++++++ plugins/codex/hooks/pre-compact.py | 8 +- plugins/codex/hooks/session-start.py | 8 +- plugins/codex/justfile | 6 +- plugins/shared/harness_envelope.py | 7 +- scripts/sync_plugin_shared.py | 76 ++++ 11 files changed, 867 insertions(+), 19 deletions(-) create mode 100644 plugins/claude-code/hooks/harness_envelope.py create mode 100644 plugins/codex/hooks/harness_envelope.py create mode 100644 scripts/sync_plugin_shared.py diff --git a/plugins/claude-code/DESIGN.md b/plugins/claude-code/DESIGN.md index 4295f2e8d..b19c7d262 100644 --- a/plugins/claude-code/DESIGN.md +++ b/plugins/claude-code/DESIGN.md @@ -654,8 +654,11 @@ class HarnessEnvelope: payload_summary: dict # safe, redacted payload excerpt ``` -The module lives at `plugins/shared/harness_envelope.py` — stdlib-only, no install -step. Both plugins import it via `sys.path.insert`. +The canonical module lives at `plugins/shared/harness_envelope.py` — stdlib-only, +no install step. Because a marketplace install copies only the plugin's own +directory, `scripts/sync_plugin_shared.py` vendors the module into each plugin's +`hooks/` directory (the copies are committed and byte-compared in +`just package-check`), and each hook imports it from its own directory. ### 14.3 V0 event types diff --git a/plugins/claude-code/hooks/harness_envelope.py b/plugins/claude-code/hooks/harness_envelope.py new file mode 100644 index 000000000..b8bd5b473 --- /dev/null +++ b/plugins/claude-code/hooks/harness_envelope.py @@ -0,0 +1,375 @@ +"""Normalized harness-event envelope for Claude Code and Codex plugins. + +This module is the producer side of the harness WAL (issue #997, SPEC-55). +It normalizes supported hook events into a shared envelope shape and provides +helpers for idempotency, redaction, and coalescing into Basic Memory artifacts. + +Design constraints: + - Stdlib only — no third-party imports. Both Claude Code (inline Python inside + bash) and Codex (standalone scripts) must import this without an install step. + - Never captures private model reasoning or hidden chain-of-thought. + - Prefers summaries and metadata over raw transcript dumps. + - Fails fast on missing project mapping rather than writing to the wrong project. + +Packaging: plugin marketplaces install a single plugin directory, so this +canonical copy in plugins/shared/ never ships. scripts/sync_plugin_shared.py +vendors it into each plugin's hooks/ directory (verified in package-check), and +hooks import it from their own directory: + + import sys, os + sys.path.insert(0, hook_dir) # the directory containing the hook script + from harness_envelope import create_envelope, to_provenance_observations + + envelope = create_envelope( + event_type="compaction_imminent", + source="claude-code", + session_id=session_id, + cwd=cwd, + project_hint=primary_project, + hook_name="PreCompact", + payload_summary={"opening": clip(opening, 200)}, + ) + provenance_lines = to_provenance_observations(envelope) +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +# --- Event type constants --- +# V0 only captures events exposed through supported harness hooks. +# Future versions may add tool_called, file_changed, test_ran, etc. +# when PostToolUse hooks become available. + +SESSION_STARTED = "session_started" +COMPACTION_IMMINENT = "compaction_imminent" +SESSION_ENDED = "session_ended" + +V0_EVENT_TYPES = frozenset({SESSION_STARTED, COMPACTION_IMMINENT, SESSION_ENDED}) + +# --- Redaction defaults --- +# Keys whose values look like secrets. Matched case-insensitively against +# payload dict keys. Uses explicit patterns for common secret key naming +# conventions. The user can extend this via config (extra_redact_keys). +# +# Strategy: match keys that contain well-known secret indicators as full +# word segments (delimited by _ or . or at string boundaries). This catches +# API_KEY, AUTH_TOKEN, DB_PASSWORD but not "safe_key" or "monkey". +DEFAULT_REDACT_KEY_PATTERNS = ( + re.compile(r"(?i)(?:^|[_.])(?:SECRET|TOKEN|PASSWORD|CREDENTIAL|AUTH)(?:[_.]|$)"), + re.compile(r"(?i)(?:^|[_.])(?:API[_.]KEY|ACCESS[_.]KEY|PRIVATE[_.]KEY)(?:[_.]|$)"), +) + +# Paths that should never appear in payload summaries. +DEFAULT_REDACT_PATHS = ( + os.path.expanduser("~/.ssh/"), + os.path.expanduser("~/.aws/"), + os.path.expanduser("~/.gnupg/"), +) + +# Values that look like environment secrets: KEY= +SECRET_VALUE_RE = re.compile(r"^[A-Za-z0-9_]+=.{20,}$") + +# Maximum length for any single payload value before truncation. +MAX_PAYLOAD_VALUE_LEN = 500 + +# Maximum event log entries before rotation. +DEFAULT_EVENT_LOG_CAP = 1000 + +# Rotation is size-triggered (a single stat per append), so the cap in lines is +# converted to a byte threshold using this conservative per-entry estimate. +# Envelope JSON lines run ~300-600 bytes (payload values truncate at 500 chars). +APPROX_BYTES_PER_EVENT = 512 + + +@dataclass(frozen=True) +class HarnessEnvelope: + """Normalized event record from a Claude Code or Codex harness hook. + + This is the producer envelope shape from SPEC-55. Each field is chosen to + be useful for coalescing into SessionNote / ToolLedger artifacts without + requiring the downstream consumer to understand the raw hook payload format. + """ + + event_type: str + source: str # "claude-code" or "codex" + session_id: str + turn_id: str | None + timestamp: str # ISO 8601 + cwd: str + project_hint: str # basicMemory.primaryProject + hook_name: str # "SessionStart", "PreCompact" + idempotency_key: str + payload_summary: dict = field(default_factory=dict) + + +def idempotency_key( + source: str, + session_id: str, + hook_name: str, + timestamp: str, +) -> str: + """Generate a deterministic key from (source, session_id, hook, timestamp_minute). + + Minute granularity means that repeated hooks within the same minute for the + same session+hook combination produce the same key — preventing duplicate + notes without requiring persistent state. Two hooks one minute apart get + distinct keys, which is the right behavior (a second compaction a minute + later is a genuinely new event). + """ + # Truncate timestamp to minute precision for dedup window. + # Handles both ISO format (2026-06-13T16:48:00+00:00) and plain timestamps. + minute_key = timestamp[:16] # "2026-06-13T16:48" + raw = f"{source}:{session_id}:{hook_name}:{minute_key}" + return hashlib.sha256(raw.encode()).hexdigest()[:16] + + +def _redact_str(value: str, deny_paths: list[str]) -> str: + """Apply the string-level redaction rules: secret values, paths, truncation.""" + if SECRET_VALUE_RE.match(value): + return "[REDACTED]" + if any(value.startswith(p) for p in deny_paths): + return "[REDACTED_PATH]" + if len(value) > MAX_PAYLOAD_VALUE_LEN: + return value[:MAX_PAYLOAD_VALUE_LEN] + "…[truncated]" + return value + + +def _redact_value(value, deny_key_patterns: list, deny_paths: list[str]): + """Recursively redact a payload value of any JSON-compatible shape. + + Payloads arrive from hook JSON, so nested dicts and lists are normal — + a secret one level down must be caught just like a top-level one. + """ + if isinstance(value, str): + return _redact_str(value, deny_paths) + if isinstance(value, dict): + return _redact_dict(value, deny_key_patterns, deny_paths) + if isinstance(value, (list, tuple)): + return [_redact_value(item, deny_key_patterns, deny_paths) for item in value] + return value + + +def _redact_dict(payload: dict, deny_key_patterns: list, deny_paths: list[str]) -> dict: + result = {} + for key, value in payload.items(): + # A denied key redacts the whole value, however deeply nested it is — + # partial redaction inside a secret-named subtree is not worth the risk. + if any(pat.search(str(key)) for pat in deny_key_patterns): + result[key] = "[REDACTED]" + continue + result[key] = _redact_value(value, deny_key_patterns, deny_paths) + return result + + +def redact_payload( + payload: dict, + extra_redact_keys: list[str] | None = None, + extra_redact_paths: list[str] | None = None, +) -> dict: + """Strip secrets, large values, and denied paths from a payload summary. + + Returns a new dict with sensitive content replaced by "[REDACTED]" markers, + applied recursively over nested dicts and lists. This is the safety layer: + nothing downstream sees unredacted payload values at any depth. + """ + deny_key_patterns = list(DEFAULT_REDACT_KEY_PATTERNS) + if extra_redact_keys: + for pattern in extra_redact_keys: + try: + deny_key_patterns.append(re.compile(re.escape(pattern), re.IGNORECASE)) + except re.error: + continue + + deny_paths = list(DEFAULT_REDACT_PATHS) + if extra_redact_paths: + deny_paths.extend(extra_redact_paths) + + return _redact_dict(payload, deny_key_patterns, deny_paths) + + +def create_envelope( + *, + event_type: str, + source: str, + session_id: str, + cwd: str, + project_hint: str, + hook_name: str, + turn_id: str | None = None, + timestamp: str | None = None, + payload_summary: dict | None = None, + redact_keys: list[str] | None = None, + redact_paths: list[str] | None = None, +) -> HarnessEnvelope: + """Factory: build a normalized envelope from hook inputs. + + All arguments are keyword-only to prevent positional-order mistakes in + hook scripts that construct envelopes from heterogeneous payload shapes. + """ + if event_type not in V0_EVENT_TYPES: + raise ValueError( + f"Unknown event type {event_type!r}; v0 supports: {sorted(V0_EVENT_TYPES)}" + ) + + ts = timestamp or datetime.now(timezone.utc).isoformat(timespec="seconds") + safe_payload = redact_payload( + payload_summary or {}, + extra_redact_keys=redact_keys, + extra_redact_paths=redact_paths, + ) + + idem_key = idempotency_key(source, session_id, hook_name, ts) + + return HarnessEnvelope( + event_type=event_type, + source=source, + session_id=session_id, + turn_id=turn_id, + timestamp=ts, + cwd=cwd, + project_hint=project_hint, + hook_name=hook_name, + idempotency_key=idem_key, + payload_summary=safe_payload, + ) + + +def to_provenance_observations(envelope: HarnessEnvelope) -> list[str]: + """Convert an envelope into observation lines for a SessionNote body. + + These are appended to the "## Observations" section of the note. They + stamp the note with its producer source so downstream consumers (recall, + consolidation, memory routines) can trace provenance without storing the + full raw event. + """ + lines = [ + f"- [source] {envelope.source}/{envelope.session_id}", + f"- [hook] {envelope.hook_name}", + f"- [event] {envelope.event_type} at {envelope.timestamp}", + f"- [idempotency] {envelope.idempotency_key}", + ] + if envelope.turn_id: + lines.append(f"- [turn] {envelope.turn_id}") + return lines + + +def to_frontmatter_fields(envelope: HarnessEnvelope) -> dict[str, str]: + """Extract envelope fields suitable for inclusion in note frontmatter. + + These are added alongside the existing session/codex_session frontmatter + to make the note queryable by source, hook, and idempotency key. + """ + fields = { + "envelope_source": envelope.source, + "envelope_event": envelope.event_type, + "envelope_hook": envelope.hook_name, + "idempotency_key": envelope.idempotency_key, + } + if envelope.turn_id: + fields["envelope_turn_id"] = envelope.turn_id + return fields + + +def envelope_to_json(envelope: HarnessEnvelope) -> str: + """Serialize an envelope to a compact JSON string for event log storage.""" + return json.dumps(asdict(envelope), separators=(",", ":")) + + +def _events_root() -> Path: + """Resolve the root directory for local event logs. + + Mirrors core's ``basic_memory.config.resolve_data_dir()`` (stdlib-only, so + no import): ``BASIC_MEMORY_CONFIG_DIR`` first, then ``XDG_CONFIG_HOME``, + then ``~/.basic-memory``. Event logs live under the per-user Basic Memory + data dir — never inside the working directory, where they would dirty the + user's repository. + """ + if config_dir := os.environ.get("BASIC_MEMORY_CONFIG_DIR"): + return Path(config_dir) / "events" + if xdg_config := os.environ.get("XDG_CONFIG_HOME"): + return Path(xdg_config) / "basic-memory" / "events" + return Path.home() / ".basic-memory" / "events" + + +def _event_log_slug(project_hint: str, cwd: str) -> str: + """Directory name isolating one project's event log from another's. + + Prefers the configured project (stable across checkouts of the same + project); falls back to the working directory. A short digest of the raw + seed disambiguates values that slug identically ("my/proj" vs "my-proj"). + """ + seed = (project_hint or "").strip() or cwd + slug = re.sub(r"[^a-z0-9]+", "-", seed.lower()).strip("-") or "default" + digest = hashlib.sha256(seed.encode("utf-8")).hexdigest()[:8] + return f"{slug[:48]}-{digest}" + + +def event_log_path(envelope: HarnessEnvelope) -> Path: + """Compute where this envelope's event log lives (see _events_root).""" + return _events_root() / _event_log_slug(envelope.project_hint, envelope.cwd) / "events.jsonl" + + +def _normalize_cap(cap) -> int: + # Trigger: eventRetention comes straight from user JSON config. + # Why: a junk value must neither break best-effort logging nor unbound the + # log. Outcome: positive ints win; anything else uses the default cap. + if isinstance(cap, bool) or not isinstance(cap, int) or cap <= 0: + return DEFAULT_EVENT_LOG_CAP + return cap + + +def append_to_event_log( + envelope: HarnessEnvelope, + cap: int | None = None, +) -> bool: + """Append a serialized envelope to the local event log. + + The event log lives under the Basic Memory data dir at + ``/events//events.jsonl`` and stores raw + envelopes for later coalescing by memory routines (SPEC-61). ``cap`` is the + approximate retention limit in entries (the ``eventRetention`` setting); + ``None`` uses DEFAULT_EVENT_LOG_CAP. + + The hot path stays cheap: one append plus one stat. Rotation only runs when + the file size passes ``cap * APPROX_BYTES_PER_EVENT`` bytes; it then drops + the oldest half (bounded at ``cap`` lines), so the file halves and the next + many appends are stat-only again. + + Returns True if the write succeeded, False on any error (best-effort). + """ + cap = _normalize_cap(cap) + log_path = event_log_path(envelope) + + try: + log_path.parent.mkdir(parents=True, exist_ok=True) + + # Append the new event + line = envelope_to_json(envelope) + "\n" + with open(log_path, "a", encoding="utf-8") as fh: + fh.write(line) + + # --- Rotation check (stat-only on the hot path) --- + # Trigger: log size passes the byte threshold derived from the cap. + # Why: unbounded growth would fill disk on long-running projects, but + # counting lines on every hook write would make the hot path O(n). + # Outcome: the oldest half rotates out; retention is approximate. + try: + if log_path.stat().st_size > cap * APPROX_BYTES_PER_EVENT: + with open(log_path, encoding="utf-8") as fh: + lines = fh.readlines() + keep = lines[len(lines) // 2 :][-cap:] + with open(log_path, "w", encoding="utf-8") as fh: + fh.writelines(keep) + except Exception: + pass # rotation failure is non-fatal + + return True + except Exception: + return False diff --git a/plugins/claude-code/hooks/pre-compact.sh b/plugins/claude-code/hooks/pre-compact.sh index c4729f214..c2599ddbc 100755 --- a/plugins/claude-code/hooks/pre-compact.sh +++ b/plugins/claude-code/hooks/pre-compact.sh @@ -51,16 +51,18 @@ import subprocess import sys from datetime import datetime -# --- Load the shared envelope module (lives two directories up in plugins/shared/) --- +# --- Load the harness envelope module (vendored next to this hook) --- # Trigger: this hook wants to stamp provenance and idempotency on the checkpoint. # Why: the envelope normalizes hook events so downstream consumers (recall, # consolidation, memory routines) can trace where each note came from. +# An installed plugin package is just this plugin directory — plugins/shared/ +# does not ship — so scripts/sync_plugin_shared.py vendors the module into +# hooks/ (canonical source: plugins/shared/harness_envelope.py). # Constraint: __file__ is '' inside a bash heredoc, so the hook script's # real directory is passed in via the BM_HOOK_DIR environment variable. _hook_dir = os.environ.get("BM_HOOK_DIR", "") if _hook_dir: - _shared_dir = os.path.join(_hook_dir, "..", "..", "shared") - sys.path.insert(0, os.path.normpath(_shared_dir)) + sys.path.insert(0, _hook_dir) try: from harness_envelope import ( COMPACTION_IMMINENT, diff --git a/plugins/claude-code/hooks/session-start.sh b/plugins/claude-code/hooks/session-start.sh index 5e6ea870e..e729f7e00 100755 --- a/plugins/claude-code/hooks/session-start.sh +++ b/plugins/claude-code/hooks/session-start.sh @@ -62,15 +62,17 @@ import subprocess import sys from concurrent.futures import ThreadPoolExecutor -# --- Load the shared envelope module (lives two directories up in plugins/shared/) --- +# --- Load the harness envelope module (vendored next to this hook) --- # SessionStart is read-only (no note writes), so the envelope is only used for # local event logging when captureEvents is enabled. +# An installed plugin package is just this plugin directory — plugins/shared/ +# does not ship — so scripts/sync_plugin_shared.py vendors the module into +# hooks/ (canonical source: plugins/shared/harness_envelope.py). # Constraint: __file__ is '' inside a bash heredoc, so the hook script's # real directory is passed in via the BM_HOOK_DIR environment variable. _hook_dir = os.environ.get("BM_HOOK_DIR", "") if _hook_dir: - _shared_dir = os.path.join(_hook_dir, "..", "..", "shared") - sys.path.insert(0, os.path.normpath(_shared_dir)) + sys.path.insert(0, _hook_dir) try: from harness_envelope import ( SESSION_STARTED, diff --git a/plugins/claude-code/justfile b/plugins/claude-code/justfile index 683008cb9..7714b8c3d 100644 --- a/plugins/claude-code/justfile +++ b/plugins/claude-code/justfile @@ -6,15 +6,19 @@ repo_root := "../.." manifest-check: python3 {{repo_root}}/scripts/validate_claude_plugin.py . +# Vendored plugins/shared modules must match their canonical source. +vendor-check: + python3 {{repo_root}}/scripts/sync_plugin_shared.py --check + # Strict Claude Code plugin validation. Requires the `claude` CLI. validate: claude plugin validate . --strict # CI-safe check that does not require the host Claude Code CLI. -ci-check: manifest-check +ci-check: manifest-check vendor-check # Full local check for maintainers. -check: manifest-check validate +check: manifest-check vendor-check validate # Show available recipes default: diff --git a/plugins/codex/hooks/harness_envelope.py b/plugins/codex/hooks/harness_envelope.py new file mode 100644 index 000000000..b8bd5b473 --- /dev/null +++ b/plugins/codex/hooks/harness_envelope.py @@ -0,0 +1,375 @@ +"""Normalized harness-event envelope for Claude Code and Codex plugins. + +This module is the producer side of the harness WAL (issue #997, SPEC-55). +It normalizes supported hook events into a shared envelope shape and provides +helpers for idempotency, redaction, and coalescing into Basic Memory artifacts. + +Design constraints: + - Stdlib only — no third-party imports. Both Claude Code (inline Python inside + bash) and Codex (standalone scripts) must import this without an install step. + - Never captures private model reasoning or hidden chain-of-thought. + - Prefers summaries and metadata over raw transcript dumps. + - Fails fast on missing project mapping rather than writing to the wrong project. + +Packaging: plugin marketplaces install a single plugin directory, so this +canonical copy in plugins/shared/ never ships. scripts/sync_plugin_shared.py +vendors it into each plugin's hooks/ directory (verified in package-check), and +hooks import it from their own directory: + + import sys, os + sys.path.insert(0, hook_dir) # the directory containing the hook script + from harness_envelope import create_envelope, to_provenance_observations + + envelope = create_envelope( + event_type="compaction_imminent", + source="claude-code", + session_id=session_id, + cwd=cwd, + project_hint=primary_project, + hook_name="PreCompact", + payload_summary={"opening": clip(opening, 200)}, + ) + provenance_lines = to_provenance_observations(envelope) +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +# --- Event type constants --- +# V0 only captures events exposed through supported harness hooks. +# Future versions may add tool_called, file_changed, test_ran, etc. +# when PostToolUse hooks become available. + +SESSION_STARTED = "session_started" +COMPACTION_IMMINENT = "compaction_imminent" +SESSION_ENDED = "session_ended" + +V0_EVENT_TYPES = frozenset({SESSION_STARTED, COMPACTION_IMMINENT, SESSION_ENDED}) + +# --- Redaction defaults --- +# Keys whose values look like secrets. Matched case-insensitively against +# payload dict keys. Uses explicit patterns for common secret key naming +# conventions. The user can extend this via config (extra_redact_keys). +# +# Strategy: match keys that contain well-known secret indicators as full +# word segments (delimited by _ or . or at string boundaries). This catches +# API_KEY, AUTH_TOKEN, DB_PASSWORD but not "safe_key" or "monkey". +DEFAULT_REDACT_KEY_PATTERNS = ( + re.compile(r"(?i)(?:^|[_.])(?:SECRET|TOKEN|PASSWORD|CREDENTIAL|AUTH)(?:[_.]|$)"), + re.compile(r"(?i)(?:^|[_.])(?:API[_.]KEY|ACCESS[_.]KEY|PRIVATE[_.]KEY)(?:[_.]|$)"), +) + +# Paths that should never appear in payload summaries. +DEFAULT_REDACT_PATHS = ( + os.path.expanduser("~/.ssh/"), + os.path.expanduser("~/.aws/"), + os.path.expanduser("~/.gnupg/"), +) + +# Values that look like environment secrets: KEY= +SECRET_VALUE_RE = re.compile(r"^[A-Za-z0-9_]+=.{20,}$") + +# Maximum length for any single payload value before truncation. +MAX_PAYLOAD_VALUE_LEN = 500 + +# Maximum event log entries before rotation. +DEFAULT_EVENT_LOG_CAP = 1000 + +# Rotation is size-triggered (a single stat per append), so the cap in lines is +# converted to a byte threshold using this conservative per-entry estimate. +# Envelope JSON lines run ~300-600 bytes (payload values truncate at 500 chars). +APPROX_BYTES_PER_EVENT = 512 + + +@dataclass(frozen=True) +class HarnessEnvelope: + """Normalized event record from a Claude Code or Codex harness hook. + + This is the producer envelope shape from SPEC-55. Each field is chosen to + be useful for coalescing into SessionNote / ToolLedger artifacts without + requiring the downstream consumer to understand the raw hook payload format. + """ + + event_type: str + source: str # "claude-code" or "codex" + session_id: str + turn_id: str | None + timestamp: str # ISO 8601 + cwd: str + project_hint: str # basicMemory.primaryProject + hook_name: str # "SessionStart", "PreCompact" + idempotency_key: str + payload_summary: dict = field(default_factory=dict) + + +def idempotency_key( + source: str, + session_id: str, + hook_name: str, + timestamp: str, +) -> str: + """Generate a deterministic key from (source, session_id, hook, timestamp_minute). + + Minute granularity means that repeated hooks within the same minute for the + same session+hook combination produce the same key — preventing duplicate + notes without requiring persistent state. Two hooks one minute apart get + distinct keys, which is the right behavior (a second compaction a minute + later is a genuinely new event). + """ + # Truncate timestamp to minute precision for dedup window. + # Handles both ISO format (2026-06-13T16:48:00+00:00) and plain timestamps. + minute_key = timestamp[:16] # "2026-06-13T16:48" + raw = f"{source}:{session_id}:{hook_name}:{minute_key}" + return hashlib.sha256(raw.encode()).hexdigest()[:16] + + +def _redact_str(value: str, deny_paths: list[str]) -> str: + """Apply the string-level redaction rules: secret values, paths, truncation.""" + if SECRET_VALUE_RE.match(value): + return "[REDACTED]" + if any(value.startswith(p) for p in deny_paths): + return "[REDACTED_PATH]" + if len(value) > MAX_PAYLOAD_VALUE_LEN: + return value[:MAX_PAYLOAD_VALUE_LEN] + "…[truncated]" + return value + + +def _redact_value(value, deny_key_patterns: list, deny_paths: list[str]): + """Recursively redact a payload value of any JSON-compatible shape. + + Payloads arrive from hook JSON, so nested dicts and lists are normal — + a secret one level down must be caught just like a top-level one. + """ + if isinstance(value, str): + return _redact_str(value, deny_paths) + if isinstance(value, dict): + return _redact_dict(value, deny_key_patterns, deny_paths) + if isinstance(value, (list, tuple)): + return [_redact_value(item, deny_key_patterns, deny_paths) for item in value] + return value + + +def _redact_dict(payload: dict, deny_key_patterns: list, deny_paths: list[str]) -> dict: + result = {} + for key, value in payload.items(): + # A denied key redacts the whole value, however deeply nested it is — + # partial redaction inside a secret-named subtree is not worth the risk. + if any(pat.search(str(key)) for pat in deny_key_patterns): + result[key] = "[REDACTED]" + continue + result[key] = _redact_value(value, deny_key_patterns, deny_paths) + return result + + +def redact_payload( + payload: dict, + extra_redact_keys: list[str] | None = None, + extra_redact_paths: list[str] | None = None, +) -> dict: + """Strip secrets, large values, and denied paths from a payload summary. + + Returns a new dict with sensitive content replaced by "[REDACTED]" markers, + applied recursively over nested dicts and lists. This is the safety layer: + nothing downstream sees unredacted payload values at any depth. + """ + deny_key_patterns = list(DEFAULT_REDACT_KEY_PATTERNS) + if extra_redact_keys: + for pattern in extra_redact_keys: + try: + deny_key_patterns.append(re.compile(re.escape(pattern), re.IGNORECASE)) + except re.error: + continue + + deny_paths = list(DEFAULT_REDACT_PATHS) + if extra_redact_paths: + deny_paths.extend(extra_redact_paths) + + return _redact_dict(payload, deny_key_patterns, deny_paths) + + +def create_envelope( + *, + event_type: str, + source: str, + session_id: str, + cwd: str, + project_hint: str, + hook_name: str, + turn_id: str | None = None, + timestamp: str | None = None, + payload_summary: dict | None = None, + redact_keys: list[str] | None = None, + redact_paths: list[str] | None = None, +) -> HarnessEnvelope: + """Factory: build a normalized envelope from hook inputs. + + All arguments are keyword-only to prevent positional-order mistakes in + hook scripts that construct envelopes from heterogeneous payload shapes. + """ + if event_type not in V0_EVENT_TYPES: + raise ValueError( + f"Unknown event type {event_type!r}; v0 supports: {sorted(V0_EVENT_TYPES)}" + ) + + ts = timestamp or datetime.now(timezone.utc).isoformat(timespec="seconds") + safe_payload = redact_payload( + payload_summary or {}, + extra_redact_keys=redact_keys, + extra_redact_paths=redact_paths, + ) + + idem_key = idempotency_key(source, session_id, hook_name, ts) + + return HarnessEnvelope( + event_type=event_type, + source=source, + session_id=session_id, + turn_id=turn_id, + timestamp=ts, + cwd=cwd, + project_hint=project_hint, + hook_name=hook_name, + idempotency_key=idem_key, + payload_summary=safe_payload, + ) + + +def to_provenance_observations(envelope: HarnessEnvelope) -> list[str]: + """Convert an envelope into observation lines for a SessionNote body. + + These are appended to the "## Observations" section of the note. They + stamp the note with its producer source so downstream consumers (recall, + consolidation, memory routines) can trace provenance without storing the + full raw event. + """ + lines = [ + f"- [source] {envelope.source}/{envelope.session_id}", + f"- [hook] {envelope.hook_name}", + f"- [event] {envelope.event_type} at {envelope.timestamp}", + f"- [idempotency] {envelope.idempotency_key}", + ] + if envelope.turn_id: + lines.append(f"- [turn] {envelope.turn_id}") + return lines + + +def to_frontmatter_fields(envelope: HarnessEnvelope) -> dict[str, str]: + """Extract envelope fields suitable for inclusion in note frontmatter. + + These are added alongside the existing session/codex_session frontmatter + to make the note queryable by source, hook, and idempotency key. + """ + fields = { + "envelope_source": envelope.source, + "envelope_event": envelope.event_type, + "envelope_hook": envelope.hook_name, + "idempotency_key": envelope.idempotency_key, + } + if envelope.turn_id: + fields["envelope_turn_id"] = envelope.turn_id + return fields + + +def envelope_to_json(envelope: HarnessEnvelope) -> str: + """Serialize an envelope to a compact JSON string for event log storage.""" + return json.dumps(asdict(envelope), separators=(",", ":")) + + +def _events_root() -> Path: + """Resolve the root directory for local event logs. + + Mirrors core's ``basic_memory.config.resolve_data_dir()`` (stdlib-only, so + no import): ``BASIC_MEMORY_CONFIG_DIR`` first, then ``XDG_CONFIG_HOME``, + then ``~/.basic-memory``. Event logs live under the per-user Basic Memory + data dir — never inside the working directory, where they would dirty the + user's repository. + """ + if config_dir := os.environ.get("BASIC_MEMORY_CONFIG_DIR"): + return Path(config_dir) / "events" + if xdg_config := os.environ.get("XDG_CONFIG_HOME"): + return Path(xdg_config) / "basic-memory" / "events" + return Path.home() / ".basic-memory" / "events" + + +def _event_log_slug(project_hint: str, cwd: str) -> str: + """Directory name isolating one project's event log from another's. + + Prefers the configured project (stable across checkouts of the same + project); falls back to the working directory. A short digest of the raw + seed disambiguates values that slug identically ("my/proj" vs "my-proj"). + """ + seed = (project_hint or "").strip() or cwd + slug = re.sub(r"[^a-z0-9]+", "-", seed.lower()).strip("-") or "default" + digest = hashlib.sha256(seed.encode("utf-8")).hexdigest()[:8] + return f"{slug[:48]}-{digest}" + + +def event_log_path(envelope: HarnessEnvelope) -> Path: + """Compute where this envelope's event log lives (see _events_root).""" + return _events_root() / _event_log_slug(envelope.project_hint, envelope.cwd) / "events.jsonl" + + +def _normalize_cap(cap) -> int: + # Trigger: eventRetention comes straight from user JSON config. + # Why: a junk value must neither break best-effort logging nor unbound the + # log. Outcome: positive ints win; anything else uses the default cap. + if isinstance(cap, bool) or not isinstance(cap, int) or cap <= 0: + return DEFAULT_EVENT_LOG_CAP + return cap + + +def append_to_event_log( + envelope: HarnessEnvelope, + cap: int | None = None, +) -> bool: + """Append a serialized envelope to the local event log. + + The event log lives under the Basic Memory data dir at + ``/events//events.jsonl`` and stores raw + envelopes for later coalescing by memory routines (SPEC-61). ``cap`` is the + approximate retention limit in entries (the ``eventRetention`` setting); + ``None`` uses DEFAULT_EVENT_LOG_CAP. + + The hot path stays cheap: one append plus one stat. Rotation only runs when + the file size passes ``cap * APPROX_BYTES_PER_EVENT`` bytes; it then drops + the oldest half (bounded at ``cap`` lines), so the file halves and the next + many appends are stat-only again. + + Returns True if the write succeeded, False on any error (best-effort). + """ + cap = _normalize_cap(cap) + log_path = event_log_path(envelope) + + try: + log_path.parent.mkdir(parents=True, exist_ok=True) + + # Append the new event + line = envelope_to_json(envelope) + "\n" + with open(log_path, "a", encoding="utf-8") as fh: + fh.write(line) + + # --- Rotation check (stat-only on the hot path) --- + # Trigger: log size passes the byte threshold derived from the cap. + # Why: unbounded growth would fill disk on long-running projects, but + # counting lines on every hook write would make the hot path O(n). + # Outcome: the oldest half rotates out; retention is approximate. + try: + if log_path.stat().st_size > cap * APPROX_BYTES_PER_EVENT: + with open(log_path, encoding="utf-8") as fh: + lines = fh.readlines() + keep = lines[len(lines) // 2 :][-cap:] + with open(log_path, "w", encoding="utf-8") as fh: + fh.writelines(keep) + except Exception: + pass # rotation failure is non-fatal + + return True + except Exception: + return False diff --git a/plugins/codex/hooks/pre-compact.py b/plugins/codex/hooks/pre-compact.py index c129b050f..0d9ef1417 100755 --- a/plugins/codex/hooks/pre-compact.py +++ b/plugins/codex/hooks/pre-compact.py @@ -11,12 +11,14 @@ from datetime import datetime, timezone from pathlib import Path -# --- Load the shared envelope module (lives two directories up in plugins/shared/) --- +# --- Load the harness envelope module (vendored next to this hook) --- # Trigger: this hook wants to stamp provenance and idempotency on the checkpoint. # Why: the envelope normalizes hook events so downstream consumers (recall, # consolidation, memory routines) can trace where each note came from. -_shared_dir = str(Path(__file__).resolve().parent.parent.parent / "shared") -sys.path.insert(0, _shared_dir) +# An installed plugin package is just plugins/codex/ — plugins/shared/ does not +# ship — so scripts/sync_plugin_shared.py vendors the module into hooks/ +# (canonical source: plugins/shared/harness_envelope.py). +sys.path.insert(0, str(Path(__file__).resolve().parent)) try: from harness_envelope import ( COMPACTION_IMMINENT, diff --git a/plugins/codex/hooks/session-start.py b/plugins/codex/hooks/session-start.py index aaf6596c0..61fc50a9a 100755 --- a/plugins/codex/hooks/session-start.py +++ b/plugins/codex/hooks/session-start.py @@ -11,11 +11,13 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path -# --- Load the shared envelope module (lives two directories up in plugins/shared/) --- +# --- Load the harness envelope module (vendored next to this hook) --- # SessionStart is read-only (no note writes), so the envelope is only used for # local event logging when captureEvents is enabled. -_shared_dir = str(Path(__file__).resolve().parent.parent.parent / "shared") -sys.path.insert(0, _shared_dir) +# An installed plugin package is just plugins/codex/ — plugins/shared/ does not +# ship — so scripts/sync_plugin_shared.py vendors the module into hooks/ +# (canonical source: plugins/shared/harness_envelope.py). +sys.path.insert(0, str(Path(__file__).resolve().parent)) try: from harness_envelope import ( SESSION_STARTED, diff --git a/plugins/codex/justfile b/plugins/codex/justfile index 796266eb1..051e0f51d 100644 --- a/plugins/codex/justfile +++ b/plugins/codex/justfile @@ -6,6 +6,10 @@ repo_root := "../.." manifest-check: python3 {{repo_root}}/scripts/validate_codex_plugin.py . +# Vendored plugins/shared modules must match their canonical source. +vendor-check: + python3 {{repo_root}}/scripts/sync_plugin_shared.py --check + # Validate against the local Codex plugin scaffold contract. scaffold-check: @validator="${CODEX_PLUGIN_VALIDATOR:-}"; \ @@ -16,4 +20,4 @@ scaffold-check: fi # Run every local package check for this plugin. -check: manifest-check scaffold-check +check: manifest-check vendor-check scaffold-check diff --git a/plugins/shared/harness_envelope.py b/plugins/shared/harness_envelope.py index 3656e1f25..b8bd5b473 100644 --- a/plugins/shared/harness_envelope.py +++ b/plugins/shared/harness_envelope.py @@ -11,10 +11,13 @@ - Prefers summaries and metadata over raw transcript dumps. - Fails fast on missing project mapping rather than writing to the wrong project. -Usage from a hook script: +Packaging: plugin marketplaces install a single plugin directory, so this +canonical copy in plugins/shared/ never ships. scripts/sync_plugin_shared.py +vendors it into each plugin's hooks/ directory (verified in package-check), and +hooks import it from their own directory: import sys, os - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "shared")) + sys.path.insert(0, hook_dir) # the directory containing the hook script from harness_envelope import create_envelope, to_provenance_observations envelope = create_envelope( diff --git a/scripts/sync_plugin_shared.py b/scripts/sync_plugin_shared.py new file mode 100644 index 000000000..76fff80b1 --- /dev/null +++ b/scripts/sync_plugin_shared.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Vendor plugins/shared modules into each installable plugin package. + +Plugin marketplaces install a single plugin directory (plugins/claude-code/ or +plugins/codex/); nothing outside that tree ships to user machines. Shared hook +helpers therefore cannot be imported from plugins/shared/ at runtime on an +installed plugin — each plugin carries a committed, vendored copy next to its +hooks instead, and the hooks import from their own directory. + +plugins/shared/ stays the canonical source. Edit modules there, then run this +script to refresh the vendored copies. `--check` verifies the copies are +byte-identical without writing (wired into `just package-check` so a drifted +copy fails CI). +""" + +from __future__ import annotations + +import argparse +import shutil +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +SHARED_DIR = REPO_ROOT / "plugins" / "shared" + +# Every installable plugin package that vendors the shared hook helpers. +# The copy lands next to the hooks so each hook can import from its own +# directory in both the repo checkout and the installed layout. +VENDOR_DIRS = ( + REPO_ROOT / "plugins" / "claude-code" / "hooks", + REPO_ROOT / "plugins" / "codex" / "hooks", +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="verify vendored copies match plugins/shared/; write nothing", + ) + args = parser.parse_args() + + sources = sorted(SHARED_DIR.glob("*.py")) + if not sources: + print(f"no shared modules found under {SHARED_DIR}", file=sys.stderr) + return 1 + + stale: list[Path] = [] + for source in sources: + for vendor_dir in VENDOR_DIRS: + target = vendor_dir / source.name + if args.check: + if not target.is_file() or target.read_bytes() != source.read_bytes(): + stale.append(target) + else: + shutil.copyfile(source, target) + print( + f"vendored {source.relative_to(REPO_ROOT)}" + f" -> {target.relative_to(REPO_ROOT)}" + ) + + if stale: + listing = "\n".join(f" {path.relative_to(REPO_ROOT)}" for path in stale) + print( + "vendored plugin modules are out of sync with plugins/shared/:\n" + f"{listing}\n" + "run: python3 scripts/sync_plugin_shared.py", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 6caeb79fc8d22c93496dbf165b6734c4b86e7f4c Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 15 Jul 2026 08:46:34 -0500 Subject: [PATCH 6/8] test(plugins): cover harness envelope safety paths Adds tests/test_harness_envelope.py for the shared module: recursive redaction over nested dicts and lists (including denied-key subtree redaction, nested path denial, truncation, and extra keys at depth), idempotency key stability within the minute window, event log location under BASIC_MEMORY_CONFIG_DIR / XDG_CONFIG_HOME with project-hint slugging, retention cap normalization, size-triggered rotation keeping the newest entries, a vendored-copy sync guard, and an installed-layout Codex run (bare plugins/codex copy, no plugins/shared sibling) proving the vendored import stamps envelopes. Extends the Claude hook harness with note-content capture, data-dir isolation, and a hooks_dir override, then covers: envelope provenance stamped on PreCompact checkpoints, opt-in event logging landing in the data dir (never the repo cwd), eventRetention reaching the rotation cap through the hook, and the installed plugin layout importing the vendored module. Co-Authored-By: Claude Fable 5 Signed-off-by: phernandez --- scripts/sync_plugin_shared.py | 3 +- tests/test_claude_plugin_hooks.py | 182 +++++++++++++++++- tests/test_harness_envelope.py | 305 ++++++++++++++++++++++++++++++ 3 files changed, 487 insertions(+), 3 deletions(-) create mode 100644 tests/test_harness_envelope.py diff --git a/scripts/sync_plugin_shared.py b/scripts/sync_plugin_shared.py index 76fff80b1..ee78e9b71 100644 --- a/scripts/sync_plugin_shared.py +++ b/scripts/sync_plugin_shared.py @@ -56,8 +56,7 @@ def main() -> int: else: shutil.copyfile(source, target) print( - f"vendored {source.relative_to(REPO_ROOT)}" - f" -> {target.relative_to(REPO_ROOT)}" + f"vendored {source.relative_to(REPO_ROOT)} -> {target.relative_to(REPO_ROOT)}" ) if stale: diff --git a/tests/test_claude_plugin_hooks.py b/tests/test_claude_plugin_hooks.py index eecf834ec..7a0e7f659 100644 --- a/tests/test_claude_plugin_hooks.py +++ b/tests/test_claude_plugin_hooks.py @@ -35,6 +35,8 @@ class HookHarness: home: Path bin_dir: Path command_log: Path + note_log: Path + config_dir: Path def write_settings(self, directory: Path, name: str, basic_memory: dict[str, object]) -> None: settings_dir = directory / ".claude" @@ -51,12 +53,18 @@ def run_hook( *, basic_memory_command: str | None = None, use_default_cli_discovery: bool = False, + hooks_dir: Path | None = None, ) -> subprocess.CompletedProcess[str]: assert BASH_EXECUTABLE is not None env = os.environ.copy() env.update( { + # Isolate the harness event log (and any future data-dir use) + # from the developer's real ~/.basic-memory and any + # XDG_CONFIG_HOME leaking in from the host environment. + "BASIC_MEMORY_CONFIG_DIR": str(self.config_dir), "BM_TEST_COMMAND_LOG": str(self.command_log), + "BM_TEST_NOTE_LOG": str(self.note_log), "HOME": str(self.home), "PATH": f"{self.bin_dir}{os.pathsep}{env['PATH']}", "USERPROFILE": str(self.home), @@ -68,8 +76,9 @@ def run_hook( env["BM_BIN"] = basic_memory_command or shlex.join( [sys.executable, str(self.bin_dir / "basic memory")] ) + hooks_root = hooks_dir or (self.repo_root / "plugins/claude-code/hooks") return subprocess.run( - [BASH_EXECUTABLE, str(self.repo_root / "plugins/claude-code/hooks" / hook_name)], + [BASH_EXECUTABLE, str(hooks_root / hook_name)], input=json.dumps(payload), capture_output=True, check=False, @@ -82,6 +91,18 @@ def logged_commands(self) -> list[list[str]]: return [] return [json.loads(line) for line in self.command_log.read_text().splitlines()] + def written_notes(self) -> list[str]: + if not self.note_log.exists(): + return [] + content = self.note_log.read_text(encoding="utf-8") + return [note for note in content.split("\n===NOTE-END===\n") if note.strip()] + + def event_log_files(self) -> list[Path]: + events_root = self.config_dir / "events" + if not events_root.exists(): + return [] + return sorted(events_root.rglob("events.jsonl")) + @pytest.fixture def hook_harness(tmp_path: Path) -> HookHarness: @@ -91,6 +112,8 @@ def hook_harness(tmp_path: Path) -> HookHarness: home.mkdir() bin_dir.mkdir() command_log = tmp_path / "basic-memory-commands.jsonl" + note_log = tmp_path / "basic-memory-notes.log" + config_dir = tmp_path / "bm-config" fake_script = """#!/usr/bin/env python3 import json @@ -102,6 +125,13 @@ def hook_harness(tmp_path: Path) -> HookHarness: if sys.argv[1:3] == ["tool", "search-notes"]: print(json.dumps({"results": []})) + +if sys.argv[1:3] == ["tool", "write-note"]: + note_log_path = os.environ.get("BM_TEST_NOTE_LOG") + if note_log_path: + with open(note_log_path, "a", encoding="utf-8") as note_log: + note_log.write(sys.stdin.read()) + note_log.write("\\n===NOTE-END===\\n") """ for command_name in ("basic memory", "basic-memory"): fake_basic_memory = bin_dir / command_name @@ -113,6 +143,8 @@ def hook_harness(tmp_path: Path) -> HookHarness: home=home, bin_dir=bin_dir, command_log=command_log, + note_log=note_log, + config_dir=config_dir, ) @@ -298,3 +330,151 @@ def test_pre_compact_uses_merged_project_and_capture_folder( write_command = write_commands[0] assert write_command[write_command.index("--project") + 1] == "project-override" assert write_command[write_command.index("--folder") + 1] == "local-sessions" + + +def _write_transcript(path: Path, opening: str) -> None: + path.write_text( + json.dumps({"message": {"role": "user", "content": opening}}) + "\n", + encoding="utf-8", + ) + + +def _pre_compact_payload(harness: HookHarness, tmp_path: Path, opening: str) -> dict[str, str]: + cwd = harness.home / "work/repo" + cwd.mkdir(parents=True, exist_ok=True) + transcript = tmp_path / "transcript.jsonl" + _write_transcript(transcript, opening) + return { + "cwd": str(cwd), + "session_id": "session-envelope-1", + "transcript_path": str(transcript), + } + + +def test_pre_compact_stamps_envelope_provenance_on_checkpoint( + hook_harness: HookHarness, + tmp_path: Path, +) -> None: + hook_harness.write_settings( + hook_harness.home, + "settings.json", + {"primaryProject": "envelope-project"}, + ) + payload = _pre_compact_payload(hook_harness, tmp_path, "Trace envelope provenance") + + result = hook_harness.run_hook("pre-compact.sh", payload) + + assert result.returncode == 0, result.stderr + notes = hook_harness.written_notes() + assert len(notes) == 1 + note = notes[0] + # Frontmatter fields make the checkpoint queryable by source and dedup-safe. + assert "envelope_source: claude-code" in note + assert "envelope_event: compaction_imminent" in note + assert "envelope_hook: PreCompact" in note + assert "idempotency_key: " in note + # Provenance observations trace the note back to its producing session. + assert "- [source] claude-code/session-envelope-1" in note + assert "- [hook] PreCompact" in note + + +def test_pre_compact_event_log_is_opt_in_and_lives_in_data_dir( + hook_harness: HookHarness, + tmp_path: Path, +) -> None: + # Without captureEvents the hook must not write any local event log. + hook_harness.write_settings( + hook_harness.home, + "settings.json", + {"primaryProject": "envelope-project"}, + ) + payload = _pre_compact_payload(hook_harness, tmp_path, "Check the event log location") + + result = hook_harness.run_hook("pre-compact.sh", payload) + + assert result.returncode == 0, result.stderr + assert hook_harness.event_log_files() == [] + + # Opting in writes the log under the Basic Memory data dir, never the repo cwd. + hook_harness.write_settings( + hook_harness.home, + "settings.json", + {"primaryProject": "envelope-project", "captureEvents": True}, + ) + result = hook_harness.run_hook("pre-compact.sh", payload) + + assert result.returncode == 0, result.stderr + event_logs = hook_harness.event_log_files() + assert len(event_logs) == 1 + assert not (Path(payload["cwd"]) / ".basic-memory").exists() + events = [json.loads(line) for line in event_logs[0].read_text(encoding="utf-8").splitlines()] + assert len(events) == 1 + event = events[0] + assert event["event_type"] == "compaction_imminent" + assert event["source"] == "claude-code" + assert event["session_id"] == "session-envelope-1" + assert event["idempotency_key"] + + +def test_pre_compact_event_retention_caps_local_event_log( + hook_harness: HookHarness, + tmp_path: Path, +) -> None: + # A long opening keeps every event line above the rotation byte estimate, + # so eventRetention=1 forces rotation as soon as a second line lands. + hook_harness.write_settings( + hook_harness.home, + "settings.json", + { + "primaryProject": "envelope-project", + "captureEvents": True, + "eventRetention": 1, + }, + ) + payload = _pre_compact_payload(hook_harness, tmp_path, "Retention check " + "x" * 400) + + runs = 4 + for _ in range(runs): + result = hook_harness.run_hook("pre-compact.sh", payload) + assert result.returncode == 0, result.stderr + + event_logs = hook_harness.event_log_files() + assert len(event_logs) == 1 + lines = event_logs[0].read_text(encoding="utf-8").splitlines() + # The default cap (1000) would keep all lines; rotation proves the + # configured eventRetention reached the writer. + assert 1 <= len(lines) < runs + assert json.loads(lines[-1])["event_type"] == "compaction_imminent" + + +def test_installed_plugin_layout_imports_vendored_envelope_module( + hook_harness: HookHarness, + tmp_path: Path, +) -> None: + # A marketplace install copies only the plugin directory — plugins/shared/ + # never ships. Running the hook from a bare copy of plugins/claude-code + # proves the vendored harness_envelope module resolves without the repo + # layout around it (regression: the import used to silently fail and + # installed users got no envelope capture at all). + installed_plugin = tmp_path / "installed/claude-code" + shutil.copytree(hook_harness.repo_root / "plugins/claude-code", installed_plugin) + assert not (tmp_path / "installed/shared").exists() + + hook_harness.write_settings( + hook_harness.home, + "settings.json", + {"primaryProject": "envelope-project", "captureEvents": True}, + ) + payload = _pre_compact_payload(hook_harness, tmp_path, "Installed layout import") + + result = hook_harness.run_hook( + "pre-compact.sh", + payload, + hooks_dir=installed_plugin / "hooks", + ) + + assert result.returncode == 0, result.stderr + notes = hook_harness.written_notes() + assert len(notes) == 1 + assert "envelope_source: claude-code" in notes[0] + assert len(hook_harness.event_log_files()) == 1 diff --git a/tests/test_harness_envelope.py b/tests/test_harness_envelope.py new file mode 100644 index 000000000..86cd00d8c --- /dev/null +++ b/tests/test_harness_envelope.py @@ -0,0 +1,305 @@ +"""Unit tests for the shared harness envelope module (plugins/shared/). + +The module is stdlib-only and lives outside the basic_memory package (it ships +vendored inside each plugin), so it is loaded straight from its file path. +""" + +import importlib.util +import json +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +CANONICAL_MODULE = REPO_ROOT / "plugins/shared/harness_envelope.py" +VENDORED_MODULES = ( + REPO_ROOT / "plugins/claude-code/hooks/harness_envelope.py", + REPO_ROOT / "plugins/codex/hooks/harness_envelope.py", +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("harness_envelope", CANONICAL_MODULE) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + # dataclass field resolution looks the module up in sys.modules, so the + # standard importlib recipe registers it before executing. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +he = _load_module() + + +def _envelope(**overrides): + kwargs = { + "event_type": he.COMPACTION_IMMINENT, + "source": "claude-code", + "session_id": "session-1", + "cwd": "/tmp/workdir", + "project_hint": "test-project", + "hook_name": "PreCompact", + "timestamp": "2026-07-15T10:00:00+00:00", + } + kwargs.update(overrides) + return he.create_envelope(**kwargs) + + +# --- Vendored copies stay in sync (mirrors `just package-check` vendor-check) --- + + +def test_vendored_copies_match_canonical_module() -> None: + canonical = CANONICAL_MODULE.read_bytes() + for vendored in VENDORED_MODULES: + assert vendored.is_file(), f"missing vendored copy: {vendored}" + assert vendored.read_bytes() == canonical, ( + f"{vendored} is out of sync; run: python3 scripts/sync_plugin_shared.py" + ) + + +# --- Redaction (recursive) --- + + +def test_redact_payload_redacts_nested_dict_secrets() -> None: + payload = {"config": {"api_key": "sk-" + "a" * 30, "region": "us-east-1"}} + + redacted = he.redact_payload(payload) + + assert redacted["config"]["api_key"] == "[REDACTED]" + assert redacted["config"]["region"] == "us-east-1" + + +def test_redact_payload_redacts_secrets_inside_lists() -> None: + payload = { + "env_dump": ["PATH=/usr/bin", "AWS_SECRET_ACCESS_KEY=" + "s" * 30], + "steps": [{"auth_token": "t" * 30}, {"note": "safe"}], + } + + redacted = he.redact_payload(payload) + + assert redacted["env_dump"][0] == "PATH=/usr/bin" + assert redacted["env_dump"][1] == "[REDACTED]" + assert redacted["steps"][0]["auth_token"] == "[REDACTED]" + assert redacted["steps"][1]["note"] == "safe" + + +def test_redact_payload_denied_key_redacts_whole_subtree() -> None: + payload = {"auth": {"user": "alice", "nested": {"deep": "value"}}} + + redacted = he.redact_payload(payload) + + assert redacted["auth"] == "[REDACTED]" + + +def test_redact_payload_applies_paths_and_truncation_at_depth() -> None: + home_ssh = str(Path("~/.ssh/id_rsa").expanduser()) + payload = {"files": [{"path": home_ssh, "preview": "y" * 600}]} + + redacted = he.redact_payload(payload) + + entry = redacted["files"][0] + assert entry["path"] == "[REDACTED_PATH]" + assert entry["preview"].endswith("…[truncated]") + assert len(entry["preview"]) < 600 + + +def test_redact_payload_extra_keys_apply_at_depth() -> None: + payload = {"outer": {"internal_id": "abc"}} + + redacted = he.redact_payload(payload, extra_redact_keys=["internal_id"]) + + assert redacted["outer"]["internal_id"] == "[REDACTED]" + + +def test_create_envelope_redacts_payload_recursively() -> None: + envelope = _envelope(payload_summary={"nested": {"password": "p" * 30}}) + + assert envelope.payload_summary["nested"]["password"] == "[REDACTED]" + + +# --- Idempotency --- + + +def test_idempotency_key_is_stable_within_the_same_minute() -> None: + key_a = he.idempotency_key("codex", "s-1", "PreCompact", "2026-07-15T10:00:01+00:00") + key_b = he.idempotency_key("codex", "s-1", "PreCompact", "2026-07-15T10:00:59+00:00") + + assert key_a == key_b + assert len(key_a) == 16 + assert int(key_a, 16) is not None # 16 hex chars + + +def test_idempotency_key_differs_across_minutes_and_inputs() -> None: + base = he.idempotency_key("codex", "s-1", "PreCompact", "2026-07-15T10:00:00+00:00") + + assert base != he.idempotency_key("codex", "s-1", "PreCompact", "2026-07-15T10:01:00+00:00") + assert base != he.idempotency_key( + "claude-code", "s-1", "PreCompact", "2026-07-15T10:00:00+00:00" + ) + assert base != he.idempotency_key("codex", "s-2", "PreCompact", "2026-07-15T10:00:00+00:00") + + +def test_create_envelope_rejects_unknown_event_type() -> None: + with pytest.raises(ValueError): + _envelope(event_type="tool_called") + + +# --- Event log location, retention, and rotation --- + + +def test_event_log_honors_basic_memory_config_dir( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + config_dir = tmp_path / "bm-config" + monkeypatch.setenv("BASIC_MEMORY_CONFIG_DIR", str(config_dir)) + envelope = _envelope() + + assert he.append_to_event_log(envelope) is True + + log_path = he.event_log_path(envelope) + assert log_path.is_file() + assert config_dir / "events" in log_path.parents + # The slug derives from the project hint, so the same project shares a log + # across checkouts. + assert log_path.parent.name.startswith("test-project-") + event = json.loads(log_path.read_text(encoding="utf-8").splitlines()[0]) + assert event["idempotency_key"] == envelope.idempotency_key + + +def test_event_log_falls_back_to_xdg_config_home( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.delenv("BASIC_MEMORY_CONFIG_DIR", raising=False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + envelope = _envelope() + + log_path = he.event_log_path(envelope) + + assert tmp_path / "xdg" / "basic-memory" / "events" in log_path.parents + + +def test_event_log_slug_falls_back_to_cwd_without_project_hint( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("BASIC_MEMORY_CONFIG_DIR", str(tmp_path / "bm-config")) + envelope = _envelope(project_hint="", cwd="/tmp/some/workdir") + + log_path = he.event_log_path(envelope) + + assert log_path.parent.name.startswith("tmp-some-workdir-") + + +def test_normalize_cap_falls_back_on_junk_values() -> None: + assert he._normalize_cap(None) == he.DEFAULT_EVENT_LOG_CAP + assert he._normalize_cap("500") == he.DEFAULT_EVENT_LOG_CAP + assert he._normalize_cap(True) == he.DEFAULT_EVENT_LOG_CAP + assert he._normalize_cap(0) == he.DEFAULT_EVENT_LOG_CAP + assert he._normalize_cap(-5) == he.DEFAULT_EVENT_LOG_CAP + assert he._normalize_cap(250) == 250 + + +def test_event_log_rotation_keeps_newest_entries( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("BASIC_MEMORY_CONFIG_DIR", str(tmp_path / "bm-config")) + # Each payload keeps the serialized line above APPROX_BYTES_PER_EVENT, so a + # cap of 1 trips the size threshold on every append after the first. + filler = "z" * he.APPROX_BYTES_PER_EVENT + + last_envelope = None + for index in range(5): + last_envelope = _envelope( + session_id=f"session-{index}", + payload_summary={"opening": filler}, + ) + assert he.append_to_event_log(last_envelope, cap=1) is True + + assert last_envelope is not None + lines = he.event_log_path(last_envelope).read_text(encoding="utf-8").splitlines() + assert len(lines) == 1 + assert json.loads(lines[0])["session_id"] == "session-4" + + +# --- Installed Codex plugin layout --- + + +def test_installed_codex_plugin_stamps_envelope_from_vendored_module( + tmp_path: Path, +) -> None: + # Codex installs copy only plugins/codex/ — run the pre-compact hook from a + # bare copy (no plugins/shared sibling) and prove the vendored module loads + # and stamps the checkpoint. + installed_plugin = tmp_path / "installed/codex" + shutil.copytree(REPO_ROOT / "plugins/codex", installed_plugin) + assert not (tmp_path / "installed/shared").exists() + + workdir = tmp_path / "workdir" + (workdir / ".codex").mkdir(parents=True) + (workdir / ".codex/basic-memory.json").write_text( + json.dumps( + { + "basicMemory": { + "primaryProject": "codex-project", + "captureEvents": True, + "eventRetention": 100, + } + } + ), + encoding="utf-8", + ) + transcript = tmp_path / "transcript.jsonl" + transcript.write_text( + json.dumps({"message": {"role": "user", "content": "Vendored codex import"}}) + "\n", + encoding="utf-8", + ) + + note_log = tmp_path / "notes.log" + fake_cli = tmp_path / "fake_basic_memory.py" + fake_cli.write_text( + "import os, sys\n" + "with open(os.environ['BM_TEST_NOTE_LOG'], 'a', encoding='utf-8') as fh:\n" + " fh.write(sys.stdin.read())\n", + encoding="utf-8", + ) + + config_dir = tmp_path / "bm-config" + env = os.environ.copy() + env.update( + { + # Isolate the event log from the developer's real data dir; the + # explicit config dir also wins over any host XDG_CONFIG_HOME. + "BASIC_MEMORY_CONFIG_DIR": str(config_dir), + "BM_BIN": shlex.join([sys.executable, str(fake_cli)]), + "BM_TEST_NOTE_LOG": str(note_log), + } + ) + result = subprocess.run( + [sys.executable, str(installed_plugin / "hooks/pre-compact.py")], + input=json.dumps( + { + "cwd": str(workdir), + "session_id": "codex-session-1", + "transcript_path": str(transcript), + } + ), + capture_output=True, + check=False, + env=env, + text=True, + ) + + assert result.returncode == 0, result.stderr + note = note_log.read_text(encoding="utf-8") + assert "envelope_source: codex" in note + assert "- [source] codex/codex-session-1" in note + event_logs = sorted((config_dir / "events").rglob("events.jsonl")) + assert len(event_logs) == 1 + event = json.loads(event_logs[0].read_text(encoding="utf-8").splitlines()[0]) + assert event["source"] == "codex" + assert event["project_hint"] == "codex-project" From e5c05903658d40bfd2fb9a3926dc9764b574196b Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 15 Jul 2026 09:09:34 -0500 Subject: [PATCH 7/8] fix(plugins): match redaction deny paths across path separators The Windows CI run exposed that path redaction never matched there: os.path.expanduser("~/.ssh/") yields mixed separators on Windows (C:\Users\x/.ssh/) while native payload values use backslashes, so the startswith deny check was dead on that platform. Deny paths and payload values now both normalize to forward slashes before comparison, including user-supplied redactPaths. Adds a platform-independent regression test exercising both separator styles. Co-Authored-By: Claude Fable 5 Signed-off-by: phernandez --- plugins/claude-code/hooks/harness_envelope.py | 20 +++++++++++++------ plugins/codex/hooks/harness_envelope.py | 20 +++++++++++++------ plugins/shared/harness_envelope.py | 20 +++++++++++++------ tests/test_harness_envelope.py | 10 ++++++++++ 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/plugins/claude-code/hooks/harness_envelope.py b/plugins/claude-code/hooks/harness_envelope.py index b8bd5b473..775abbcba 100644 --- a/plugins/claude-code/hooks/harness_envelope.py +++ b/plugins/claude-code/hooks/harness_envelope.py @@ -66,11 +66,19 @@ re.compile(r"(?i)(?:^|[_.])(?:API[_.]KEY|ACCESS[_.]KEY|PRIVATE[_.]KEY)(?:[_.]|$)"), ) -# Paths that should never appear in payload summaries. + +# Paths that should never appear in payload summaries. Deny paths and payload +# values are both compared with forward slashes — os.path.expanduser("~/.ssh/") +# yields mixed separators on Windows (C:\Users\x/.ssh/) while native payload +# values use backslashes, so an un-normalized startswith never matches there. +def _normalize_path(path: str) -> str: + return path.replace("\\", "/") + + DEFAULT_REDACT_PATHS = ( - os.path.expanduser("~/.ssh/"), - os.path.expanduser("~/.aws/"), - os.path.expanduser("~/.gnupg/"), + _normalize_path(os.path.expanduser("~/.ssh/")), + _normalize_path(os.path.expanduser("~/.aws/")), + _normalize_path(os.path.expanduser("~/.gnupg/")), ) # Values that look like environment secrets: KEY= @@ -134,7 +142,7 @@ def _redact_str(value: str, deny_paths: list[str]) -> str: """Apply the string-level redaction rules: secret values, paths, truncation.""" if SECRET_VALUE_RE.match(value): return "[REDACTED]" - if any(value.startswith(p) for p in deny_paths): + if any(_normalize_path(value).startswith(p) for p in deny_paths): return "[REDACTED_PATH]" if len(value) > MAX_PAYLOAD_VALUE_LEN: return value[:MAX_PAYLOAD_VALUE_LEN] + "…[truncated]" @@ -189,7 +197,7 @@ def redact_payload( deny_paths = list(DEFAULT_REDACT_PATHS) if extra_redact_paths: - deny_paths.extend(extra_redact_paths) + deny_paths.extend(_normalize_path(path) for path in extra_redact_paths) return _redact_dict(payload, deny_key_patterns, deny_paths) diff --git a/plugins/codex/hooks/harness_envelope.py b/plugins/codex/hooks/harness_envelope.py index b8bd5b473..775abbcba 100644 --- a/plugins/codex/hooks/harness_envelope.py +++ b/plugins/codex/hooks/harness_envelope.py @@ -66,11 +66,19 @@ re.compile(r"(?i)(?:^|[_.])(?:API[_.]KEY|ACCESS[_.]KEY|PRIVATE[_.]KEY)(?:[_.]|$)"), ) -# Paths that should never appear in payload summaries. + +# Paths that should never appear in payload summaries. Deny paths and payload +# values are both compared with forward slashes — os.path.expanduser("~/.ssh/") +# yields mixed separators on Windows (C:\Users\x/.ssh/) while native payload +# values use backslashes, so an un-normalized startswith never matches there. +def _normalize_path(path: str) -> str: + return path.replace("\\", "/") + + DEFAULT_REDACT_PATHS = ( - os.path.expanduser("~/.ssh/"), - os.path.expanduser("~/.aws/"), - os.path.expanduser("~/.gnupg/"), + _normalize_path(os.path.expanduser("~/.ssh/")), + _normalize_path(os.path.expanduser("~/.aws/")), + _normalize_path(os.path.expanduser("~/.gnupg/")), ) # Values that look like environment secrets: KEY= @@ -134,7 +142,7 @@ def _redact_str(value: str, deny_paths: list[str]) -> str: """Apply the string-level redaction rules: secret values, paths, truncation.""" if SECRET_VALUE_RE.match(value): return "[REDACTED]" - if any(value.startswith(p) for p in deny_paths): + if any(_normalize_path(value).startswith(p) for p in deny_paths): return "[REDACTED_PATH]" if len(value) > MAX_PAYLOAD_VALUE_LEN: return value[:MAX_PAYLOAD_VALUE_LEN] + "…[truncated]" @@ -189,7 +197,7 @@ def redact_payload( deny_paths = list(DEFAULT_REDACT_PATHS) if extra_redact_paths: - deny_paths.extend(extra_redact_paths) + deny_paths.extend(_normalize_path(path) for path in extra_redact_paths) return _redact_dict(payload, deny_key_patterns, deny_paths) diff --git a/plugins/shared/harness_envelope.py b/plugins/shared/harness_envelope.py index b8bd5b473..775abbcba 100644 --- a/plugins/shared/harness_envelope.py +++ b/plugins/shared/harness_envelope.py @@ -66,11 +66,19 @@ re.compile(r"(?i)(?:^|[_.])(?:API[_.]KEY|ACCESS[_.]KEY|PRIVATE[_.]KEY)(?:[_.]|$)"), ) -# Paths that should never appear in payload summaries. + +# Paths that should never appear in payload summaries. Deny paths and payload +# values are both compared with forward slashes — os.path.expanduser("~/.ssh/") +# yields mixed separators on Windows (C:\Users\x/.ssh/) while native payload +# values use backslashes, so an un-normalized startswith never matches there. +def _normalize_path(path: str) -> str: + return path.replace("\\", "/") + + DEFAULT_REDACT_PATHS = ( - os.path.expanduser("~/.ssh/"), - os.path.expanduser("~/.aws/"), - os.path.expanduser("~/.gnupg/"), + _normalize_path(os.path.expanduser("~/.ssh/")), + _normalize_path(os.path.expanduser("~/.aws/")), + _normalize_path(os.path.expanduser("~/.gnupg/")), ) # Values that look like environment secrets: KEY= @@ -134,7 +142,7 @@ def _redact_str(value: str, deny_paths: list[str]) -> str: """Apply the string-level redaction rules: secret values, paths, truncation.""" if SECRET_VALUE_RE.match(value): return "[REDACTED]" - if any(value.startswith(p) for p in deny_paths): + if any(_normalize_path(value).startswith(p) for p in deny_paths): return "[REDACTED_PATH]" if len(value) > MAX_PAYLOAD_VALUE_LEN: return value[:MAX_PAYLOAD_VALUE_LEN] + "…[truncated]" @@ -189,7 +197,7 @@ def redact_payload( deny_paths = list(DEFAULT_REDACT_PATHS) if extra_redact_paths: - deny_paths.extend(extra_redact_paths) + deny_paths.extend(_normalize_path(path) for path in extra_redact_paths) return _redact_dict(payload, deny_key_patterns, deny_paths) diff --git a/tests/test_harness_envelope.py b/tests/test_harness_envelope.py index 86cd00d8c..7a199f77f 100644 --- a/tests/test_harness_envelope.py +++ b/tests/test_harness_envelope.py @@ -109,6 +109,16 @@ def test_redact_payload_applies_paths_and_truncation_at_depth() -> None: assert len(entry["preview"]) < 600 +def test_redact_payload_matches_deny_paths_across_separators() -> None: + # Windows payload values carry backslashes while deny paths are usually + # written with forward slashes; both sides normalize before comparison. + payload = {"path": "C:\\Users\\dev\\vault\\key.txt"} + + redacted = he.redact_payload(payload, extra_redact_paths=["C:/Users/dev/vault/"]) + + assert redacted["path"] == "[REDACTED_PATH]" + + def test_redact_payload_extra_keys_apply_at_depth() -> None: payload = {"outer": {"internal_id": "abc"}} From 736848fd35d48cbbd6e50b411271dd3ceb05e503 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 15 Jul 2026 09:13:23 -0500 Subject: [PATCH 8/8] fix(plugins): fail closed on non-boolean captureEvents captureEvents gated event capture with a truthiness check, so a hand-edited string value like "false" (truthy in Python) silently enabled recording. A privacy gate must fail closed: every gate site in the Claude Code and Codex SessionStart and PreCompact hooks now requires the JSON boolean true exactly. Tests cover string "true", string "false", and numeric values staying off in both plugins while checkpoint notes still write. Addresses Codex review on #1064. Co-Authored-By: Claude Fable 5 Signed-off-by: phernandez --- plugins/claude-code/hooks/pre-compact.sh | 4 ++- plugins/claude-code/hooks/session-start.sh | 4 ++- plugins/codex/hooks/pre-compact.py | 4 ++- plugins/codex/hooks/session-start.py | 4 ++- tests/test_claude_plugin_hooks.py | 24 +++++++++++++ tests/test_harness_envelope.py | 40 +++++++++++++++++----- 6 files changed, 68 insertions(+), 12 deletions(-) diff --git a/plugins/claude-code/hooks/pre-compact.sh b/plugins/claude-code/hooks/pre-compact.sh index c2599ddbc..2bcffd7e1 100755 --- a/plugins/claude-code/hooks/pre-compact.sh +++ b/plugins/claude-code/hooks/pre-compact.sh @@ -151,7 +151,9 @@ def load_settings(directory): cfg = load_settings(cwd) primary_project = (cfg.get("primaryProject") or "").strip() capture_folder = (cfg.get("captureFolder") or "sessions").strip() -capture_events = bool(cfg.get("captureEvents", False)) +# Strict boolean: captureEvents is a privacy gate, so it fails closed. A +# hand-edited string like "false" (truthy in Python) must not enable capture. +capture_events = cfg.get("captureEvents") is True redact_keys = cfg.get("redactKeys") or [] redact_paths = cfg.get("redactPaths") or [] # Approximate retention cap for the local event log; the envelope module diff --git a/plugins/claude-code/hooks/session-start.sh b/plugins/claude-code/hooks/session-start.sh index e729f7e00..76b00d38d 100755 --- a/plugins/claude-code/hooks/session-start.sh +++ b/plugins/claude-code/hooks/session-start.sh @@ -180,7 +180,9 @@ recall_prompt = cfg.get("recallPrompt") or default_prompt # Without this, setup writes them but they never reach Claude (dead config). placement_conventions = (cfg.get("placementConventions") or "").strip() capture_folder = (cfg.get("captureFolder") or "sessions").strip() -capture_events = bool(cfg.get("captureEvents", False)) +# Strict boolean: captureEvents is a privacy gate, so it fails closed. A +# hand-edited string like "false" (truthy in Python) must not enable capture. +capture_events = cfg.get("captureEvents") is True # Approximate retention cap for the local event log; the envelope module # validates it (non-positive/junk values fall back to its default). event_retention = cfg.get("eventRetention") diff --git a/plugins/codex/hooks/pre-compact.py b/plugins/codex/hooks/pre-compact.py index 0d9ef1417..967282a79 100755 --- a/plugins/codex/hooks/pre-compact.py +++ b/plugins/codex/hooks/pre-compact.py @@ -152,7 +152,9 @@ def main() -> int: cfg = load_config(cwd) primary_project = str(cfg.get("primaryProject") or "").strip() capture_folder = str(cfg.get("captureFolder") or "codex-sessions").strip() - capture_events = bool(cfg.get("captureEvents", False)) + # Strict boolean: captureEvents is a privacy gate, so it fails closed. A + # hand-edited string like "false" (truthy in Python) must not enable capture. + capture_events = cfg.get("captureEvents") is True redact_keys = cfg.get("redactKeys") or [] redact_paths_cfg = cfg.get("redactPaths") or [] # Approximate retention cap for the local event log; the envelope module diff --git a/plugins/codex/hooks/session-start.py b/plugins/codex/hooks/session-start.py index 61fc50a9a..f08d21c38 100755 --- a/plugins/codex/hooks/session-start.py +++ b/plugins/codex/hooks/session-start.py @@ -159,7 +159,9 @@ def main() -> int: capture_folder = str(cfg.get("captureFolder") or "codex-sessions").strip() placement = str(cfg.get("placementConventions") or "").strip() focus = str(cfg.get("focus") or "").strip() - capture_events = bool(cfg.get("captureEvents", False)) + # Strict boolean: captureEvents is a privacy gate, so it fails closed. A + # hand-edited string like "false" (truthy in Python) must not enable capture. + capture_events = cfg.get("captureEvents") is True # Approximate retention cap for the local event log; the envelope module # validates it (non-positive/junk values fall back to its default). event_retention = cfg.get("eventRetention") diff --git a/tests/test_claude_plugin_hooks.py b/tests/test_claude_plugin_hooks.py index 7a0e7f659..f6c1fde84 100644 --- a/tests/test_claude_plugin_hooks.py +++ b/tests/test_claude_plugin_hooks.py @@ -416,6 +416,30 @@ def test_pre_compact_event_log_is_opt_in_and_lives_in_data_dir( assert event["idempotency_key"] +@pytest.mark.parametrize("capture_events", ["true", "false", 1]) +def test_pre_compact_capture_events_gate_fails_closed_on_non_booleans( + hook_harness: HookHarness, + tmp_path: Path, + capture_events: object, +) -> None: + # captureEvents is a privacy gate: only JSON `true` may enable capture. A + # hand-edited string "false" (or "true", or a number) is truthy in Python + # and must NOT switch event logging on. + hook_harness.write_settings( + hook_harness.home, + "settings.json", + {"primaryProject": "envelope-project", "captureEvents": capture_events}, + ) + payload = _pre_compact_payload(hook_harness, tmp_path, "Gate must fail closed") + + result = hook_harness.run_hook("pre-compact.sh", payload) + + assert result.returncode == 0, result.stderr + # The checkpoint itself still writes — only event capture stays off. + assert len(hook_harness.written_notes()) == 1 + assert hook_harness.event_log_files() == [] + + def test_pre_compact_event_retention_caps_local_event_log( hook_harness: HookHarness, tmp_path: Path, diff --git a/tests/test_harness_envelope.py b/tests/test_harness_envelope.py index 7a199f77f..bee3bc634 100644 --- a/tests/test_harness_envelope.py +++ b/tests/test_harness_envelope.py @@ -239,12 +239,12 @@ def test_event_log_rotation_keeps_newest_entries( # --- Installed Codex plugin layout --- -def test_installed_codex_plugin_stamps_envelope_from_vendored_module( - tmp_path: Path, -) -> None: - # Codex installs copy only plugins/codex/ — run the pre-compact hook from a - # bare copy (no plugins/shared sibling) and prove the vendored module loads - # and stamps the checkpoint. +def _run_codex_pre_compact(tmp_path: Path, capture_events: object) -> tuple[Path, Path]: + """Run the codex pre-compact hook from a bare plugin copy. + + The copy has no plugins/shared sibling, mirroring a marketplace install. + Returns (note_log, config_dir) for the caller's assertions. + """ installed_plugin = tmp_path / "installed/codex" shutil.copytree(REPO_ROOT / "plugins/codex", installed_plugin) assert not (tmp_path / "installed/shared").exists() @@ -256,7 +256,7 @@ def test_installed_codex_plugin_stamps_envelope_from_vendored_module( { "basicMemory": { "primaryProject": "codex-project", - "captureEvents": True, + "captureEvents": capture_events, "eventRetention": 100, } } @@ -303,8 +303,18 @@ def test_installed_codex_plugin_stamps_envelope_from_vendored_module( env=env, text=True, ) - assert result.returncode == 0, result.stderr + return note_log, config_dir + + +def test_installed_codex_plugin_stamps_envelope_from_vendored_module( + tmp_path: Path, +) -> None: + # Codex installs copy only plugins/codex/ — run the pre-compact hook from a + # bare copy (no plugins/shared sibling) and prove the vendored module loads + # and stamps the checkpoint. + note_log, config_dir = _run_codex_pre_compact(tmp_path, capture_events=True) + note = note_log.read_text(encoding="utf-8") assert "envelope_source: codex" in note assert "- [source] codex/codex-session-1" in note @@ -313,3 +323,17 @@ def test_installed_codex_plugin_stamps_envelope_from_vendored_module( event = json.loads(event_logs[0].read_text(encoding="utf-8").splitlines()[0]) assert event["source"] == "codex" assert event["project_hint"] == "codex-project" + + +@pytest.mark.parametrize("capture_events", ["true", "false", 1]) +def test_codex_capture_events_gate_fails_closed_on_non_booleans( + tmp_path: Path, capture_events: object +) -> None: + # captureEvents is a privacy gate: only JSON `true` may enable capture. A + # hand-edited string "false" (or "true", or a number) is truthy in Python + # and must NOT switch event logging on. + note_log, config_dir = _run_codex_pre_compact(tmp_path, capture_events=capture_events) + + # The checkpoint itself still writes — only event capture stays off. + assert "envelope_source: codex" in note_log.read_text(encoding="utf-8") + assert not (config_dir / "events").exists()