From e21cee7ffa24be14bcc693dc6062a332e6ccdc42 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 10:12:37 -0700 Subject: [PATCH 01/34] Keep each section's raw Markdown in the parser Each parsed section now keeps its own raw Markdown: everything after its heading line up to the next heading of any level, so a child section's text is not included. Blank lines before the first line of content are dropped as whole lines rather than trimmed, so an indented first line keeps its indentation, and trailing whitespace is removed. Code that walks the parsed tree can now read a section's text as the author wrote it, without its heading or its children. - `Section` gains a crate-visible `markdown` field with a public `markdown()` accessor, the same shape as its other fields. - `trim_leading_blank_lines` becomes `pub(crate)` so section building reuses its whole-line trim; a plain trim would strip the first line's indentation. - `build_sections` fills `markdown` from the current heading's content, so a section's text stops at its first child heading and an empty section reads as an empty string. - `section_markdown_is_the_sections_own_content` pins one section's text byte for byte through an indented first line, a `lua` fence, a `---` break with a blank line above it, and `{{ x }}`, then checks its child's text and an empty section. - `synthetic_section` sets `markdown` to an empty string, so synthetic test sections carry no Markdown. - `markdown()` has no caller in this change other than the new test. Design: extends oversized-unit @ crates/promptforge-internal/parser/src/build.rs::build_sections deps: &Emitter,&[Heading],&mut usize,u32,u8 Design: extends pure-function @ crates/promptforge-internal/parser/src/fence.rs::trim_leading_blank_lines deps: &str Design: extends pure-function @ crates/promptforge-internal/parser/src/test_support.rs::synthetic_section deps: &str,Vec,Vec,u8 Plan: vibe/2026-10-10-2-prompt-namespace-reads.md --- .../promptforge-internal/parser/src/build.rs | 3 +- .../promptforge-internal/parser/src/fence.rs | 2 +- .../promptforge-internal/parser/src/prompt.rs | 9 + .../parser/src/test_support.rs | 1 + .../parser/src/tests/headings.rs | 17 + vibe/2026-10-10-2-prompt-namespace-reads.md | 468 ++++++++++++++++++ vibe/ACTIVE | 1 + 7 files changed, 499 insertions(+), 2 deletions(-) create mode 100644 vibe/2026-10-10-2-prompt-namespace-reads.md create mode 100644 vibe/ACTIVE diff --git a/crates/promptforge-internal/parser/src/build.rs b/crates/promptforge-internal/parser/src/build.rs index a45f15f9d..a293a1a2d 100644 --- a/crates/promptforge-internal/parser/src/build.rs +++ b/crates/promptforge-internal/parser/src/build.rs @@ -9,7 +9,7 @@ use pulldown_cmark::{Event, HeadingLevel, Options, Parser, Tag, TagEnd}; use promptforge_types::emitter::Emitter; -use super::fence::{RawBlock, lua_block_location, split_section_blocks}; +use super::fence::{RawBlock, lua_block_location, split_section_blocks, trim_leading_blank_lines}; use super::list::{is_all_list_markers, parse_bullet_items}; use super::{Block, LuaProgram, ParseErrorKind, Section}; use crate::{Error, Result}; @@ -290,6 +290,7 @@ pub(crate) fn build_sections( blocks, children, items, + markdown: trim_leading_blank_lines(&h.content).trim_end().to_owned(), }); } Ok(result) diff --git a/crates/promptforge-internal/parser/src/fence.rs b/crates/promptforge-internal/parser/src/fence.rs index dbe9a5b18..dfb1dcff4 100644 --- a/crates/promptforge-internal/parser/src/fence.rs +++ b/crates/promptforge-internal/parser/src/fence.rs @@ -119,7 +119,7 @@ pub(super) fn split_h1( } /// Removes complete leading lines that contain only whitespace. -fn trim_leading_blank_lines(content: &str) -> &str { +pub(crate) fn trim_leading_blank_lines(content: &str) -> &str { let mut offset = 0; for line in content.split_inclusive('\n') { if line.trim().is_empty() { diff --git a/crates/promptforge-internal/parser/src/prompt.rs b/crates/promptforge-internal/parser/src/prompt.rs index 26e0980f2..b8b514edc 100644 --- a/crates/promptforge-internal/parser/src/prompt.rs +++ b/crates/promptforge-internal/parser/src/prompt.rs @@ -40,6 +40,8 @@ pub struct Section { /// Pre-parsed bullet items for list-only sections (no lua blocks). /// Empty for non-list sections. pub(crate) items: Vec, + /// The section's own raw Markdown, without its heading line or children. + pub(crate) markdown: String, } impl Section { @@ -73,6 +75,13 @@ impl Section { &self.items } + /// The section's own raw Markdown: from just after its heading line to the next + /// heading of any level, leading blank lines and trailing whitespace trimmed. + #[must_use] + pub fn markdown(&self) -> &str { + &self.markdown + } + /// Classic leading Lua fence when the first block is Lua. #[must_use] pub fn prologue(&self) -> Option<&LuaProgram> { diff --git a/crates/promptforge-internal/parser/src/test_support.rs b/crates/promptforge-internal/parser/src/test_support.rs index 31b4f8b45..52eba17f9 100644 --- a/crates/promptforge-internal/parser/src/test_support.rs +++ b/crates/promptforge-internal/parser/src/test_support.rs @@ -18,6 +18,7 @@ pub fn synthetic_section(name: &str, level: u8, blocks: Vec, items: Vec + +## Product Requirements + +Prompt authors need to read other sections' raw text, and the one cross-section reader they have, `list_from_section`, is a bare global that raises on every miss. This plan adds a `prompt` Lua namespace with `prompt.section(heading)` and `prompt.list(heading)`, both returning `nil` when the heading names no visible section, and removes `list_from_section`. One function answers every heading lookup. Production code grows only by the new read, about 40 lines, and the tests shrink by about as much, ending with fewer tests than they start with. Nothing about task execution changes. + +- Problem and users: + - Prompt authors cannot read another section's text. `list_from_section` returns a list section's items and refuses every other kind of section. + - `list_from_section` raises when the heading isn't found, so an author can't ask "does this section exist?" without `pcall`. + - It is a bare global beside `call`, `jump`, and `prose`, with no namespace to grow into. +- Goals: + - `prompt.section(heading)` returns a visible section's own raw Markdown, byte for byte, or `nil` when the heading names no visible section. + - `prompt.list(heading)` is today's `list_from_section` under the new name and also returns `nil` on a miss, so a heading can serve as an index: with sections `### anthropic`, `### openai`, and `### gemini`, `prompt.list('### grok')` is `nil`. + - `list_from_section` is removed, with no stub. + - One function answers every heading lookup: given a heading and a visible set, the section, or "not in scope". +- Non-goals: + - Anything about task execution: `tasks`, the `fanout` global's behavior, subagents, the scheduler, tool-call dispatch, and the Harness. + - Backward compatibility for `list_from_section`. + - Addressing grandchild sections in one call. + - A `markdown` namespace for parsing arbitrary text. +- Success criteria: + - `prompt.section` returns a section's own content byte for byte, including Lua fences, `---` breaks, and unrendered `{{ }}` placeholders, and `""` for an empty visible section. + - Both reads return `nil`, without raising, for a heading that doesn't exist and for one outside the caller's visible set (a niece, a grandchild, or the caller itself). + - Both reads work from the H1 pass and from a `lua shared` library at load time. + - `call` and `jump` behave exactly as before, error messages included. + - No source file mentions `list_from_section` outside history files, and no VM has that global. + - The change deletes more test code than it adds, and every exit gate passes. +- Constraints: + - Engine crates name the code that steps a run "the caller" and never mention a Host, Workshop, or lowercase "harness". The docs-claims test scans every line of `crates/promptforge` and `crates/promptforge-internal`, comments and strings included. + - No source change under `crates/promptforge-internal/engine/src/execute/scheduler/`. Test files under `crates/promptforge-internal/engine/src/execute/tests/scheduler/` are a different directory and change only as the Testing Plan lists. + - Every `.rs` file stays under 500 lines, and source directories stay flat. + - Prompts are at language version `promptforge: 0`, so removed forms simply stop existing. +- Open questions: + - None. + +## Functional Specification + +Authors read other sections through `prompt`. A read resolves its heading over the section's visible set, the same set `call` and `jump` resolve against. A heading with no match in that set reads as `nil`, and a mistake in the call itself raises. + +- Actors and workflows: + - A *section* is a Markdown heading (H2 to H6) and its content. A section's *visible set* is its siblings minus itself, plus its direct children. The *H1 pass* is the Lua under the prompt's H1 title, and its visible set is the whole top-level slice. The *`lua shared` library* is a fenced Lua block in the H1 that replays at the start of every section VM. + - A prompt author reads another section: + + ```lua + local rules = prompt.section('### Instructions') -- raw Markdown; nil if no such visible section + local extra = prompt.section('### Extras') or '' -- an optional read needs no pcall + local topics = prompt.list('### Topics') -- was list_from_section + local picks = prompt.list('### ' .. vendor) -- headings as an index: nil for a vendor with no section + ``` +- Inputs and outputs: + - `prompt.section(heading)` returns a string, or `nil`. + - `prompt.list(heading)` returns a Lua array of strings (a list section's items, markers stripped), or `nil`. + - `heading` is a string such as `'### Instructions'`: the level markers, whitespace, and the exact name. +- States and validation: + - `prompt.section` span: from just after the heading line to the next heading of any level, so the heading line and nested subsections are excluded. Leading blank lines and trailing whitespace are trimmed; everything between, including the first content line's indentation, is byte-exact. An empty section returns `""`, never `nil`. Any kind of section is valid, list sections included. + - A heading that doesn't exist and one outside the visible set get the same answer, `nil`, because the lookup searches only the visible set. That is also why a miss reveals nothing about the rest of the document. + - `prompt` is a reserved global, and `list_from_section` no longer exists. +- Errors and recovery: + - Both reads raise at the call site, catchable by `pcall`, for a malformed heading (no `#` markers, no name, or no whitespace after the markers) and for a non-string argument. + - `prompt.list` also raises "section `` has no pre-parsed items" for a visible section that isn't a list, since naming a prose section is a mistake in the call. + - A `nil` passed on as a `fanout` collection fails with the collection error, which now points at `prompt.list` and says it returns `nil` when no visible section has that heading. +- Security and privacy behavior: + - The reads expose only text already in the prompt, and only for sections in the caller's visible set. `{{ }}` placeholders come back unrendered, so a read evaluates no template and reads no run data. +- Acceptance criteria: + - Every behavior above has a test in the Testing Plan, and every exit gate passes. + + + + +## Technical Design + +One lookup, `find_visible`, searches a visible set for a heading and answers with the section or `None`. Two thin adapters sit on it: `resolve_sibling`, which keeps today's not-found error for `call` and `jump`, and `read_visible`, which hands a prompt read the section's text. The Lua crate takes one `read` callback in place of today's `list_callback`, the callback behind `list_from_section`, and builds both `prompt` functions over it, so no signature gains a parameter. The parser keeps each section's raw Markdown, the one piece of data that doesn't exist today. + +- Architecture: + + ```mermaid + flowchart LR + reads["prompt reads"] --> readVisible["read_visible"] + callJump["call and jump"] --> resolveSibling["resolve_sibling"] + readVisible -->|"miss is nil"| findVisible["find_visible"] + resolveSibling -->|"miss is an error"| findVisible + findVisible -->|"searches only"| visibleSet["visible set"] + ``` + + - The visible set is built where it is today: `visible_sections` for a walked section, and the top-level slice for H1. Neither changes. + - "Out of scope" and "doesn't exist" are one answer by construction: the lookup only ever sees the visible set. +- Modules and interfaces: + - Paths in the code comments below are relative to `crates/promptforge-internal/`. + + ```rust + // parser/src/prompt.rs + pub struct Section { /* existing fields */ pub(crate) markdown: String } + impl Section { + /// The section's own raw Markdown: from just after its heading line to the next + /// heading of any level, leading blank lines and trailing whitespace trimmed. + pub fn markdown(&self) -> &str; + } + + // parser/src/fence.rs: reused, now crate-visible + pub(crate) fn trim_leading_blank_lines(content: &str) -> &str; + + // engine/src/heading_address.rs + /// The one heading lookup: the section in `visible` with the heading's exact level + /// and name, or `None` when the visible set has none. + /// # Errors: a malformed heading, or two visible sections with one address. + pub(crate) fn find_visible<'a>(heading: &str, visible: &'a [Section]) -> Result>; + /// `find_visible` where a miss is an error (`call`, `jump`); callers and messages unchanged. + pub(crate) fn resolve_sibling<'a>(heading: &str, visible: &'a [Section]) -> Result<&'a Section>; + + // engine/src/execute/walk_target.rs: replaces list_items_from_visible + pub(super) fn read_visible(heading: &str, visible: &[Section]) -> Result>; + + // lua/src/handles.rs, re-exported at the crate root and in engine/src/lua.rs + /// What a prompt read sees of one visible section. + #[derive(Debug, Clone, PartialEq, Eq)] + pub struct SectionText { + /// The section's own raw Markdown. + pub markdown: String, + /// Its pre-parsed list items; empty for a section that isn't a list. + pub items: Vec, + } + + // lua/src/vm/install.rs + impl SectionVm { + pub fn install_scheduler_control_globals(&self, read: R) -> Result<()> // was list_callback + where + R: Fn(String) -> std::result::Result, E> + Send + Sync + 'static, + E: std::error::Error + Send + Sync + 'static; + #[cfg(test)] + pub(crate) fn install_control_globals(&self, call_callback: E, read: R) -> Result<()> + where + E: Fn(Value, Option, Json) -> std::result::Result + Send + 'static, + R: Fn(String) -> std::result::Result, Error> + Send + Sync + 'static; + fn install_prompt_table(&self, globals: &mlua::Table, read: R) -> Result<()>; // replaces install_list_global + } + + // engine/src/execute/section_vm.rs + pub(crate) fn setup_section_vm(vm: &mut SectionVm, setup: &SectionVmSetup<'_>, read: R) -> Result<()> + where + R: Fn(String) -> std::result::Result, Error> + Send + Sync + 'static; + ``` + + The three bodies that hold logic. `find_visible` is today's `resolve_sibling` body without its not-found branch, and the adapter takes that branch over with the same text: + + ```rust + pub(crate) fn find_visible<'a>(heading: &str, visible: &'a [Section]) -> Result> { + let (level, name) = parse_heading_address(heading)?; + let mut matches = visible + .iter() + .filter(|section| usize::from(section.level()) == level && section.name() == name); + let found = matches.next(); + if found.is_some() && matches.next().is_some() { + return Err(Error::Lua(format!( + "section heading `{}` is ambiguous; more than one visible section matches {} {name}", + heading.trim(), + "#".repeat(level) + ))); + } + Ok(found) + } + + pub(crate) fn resolve_sibling<'a>(heading: &str, visible: &'a [Section]) -> Result<&'a Section> { + find_visible(heading, visible)?.ok_or_else(|| { + let available: Vec = visible + .iter() + .map(|s| format!("{} {}", "#".repeat(s.level().into()), s.name())) + .collect(); + Error::Lua(format!( + "section heading `{}` not found; available sections: {}", + heading.trim(), + available.join(", ") + )) + }) + } + + pub(super) fn read_visible(heading: &str, visible: &[Section]) -> Result> { + Ok(heading_address::find_visible(heading, visible)?.map(|section| SectionText { + markdown: section.markdown().to_owned(), + items: section.items().to_vec(), + })) + } + ``` + + `install_prompt_table` builds both functions over one shared callback. The "no pre-parsed items" check moves here from the Engine, because it belongs to the `prompt.list` surface. Every `read` bound gains `Sync` (today's list callback bounds are `Send + 'static`): mlua's `send` feature makes each Lua function require `Send`, and `Arc` is `Send` only when `R` is `Send + Sync`. Every current closure captures plain data (`Vec
`, or nothing in the stubs), so none needs to change: + + ```rust + fn install_prompt_table(&self, globals: &mlua::Table, read: R) -> Result<()> + where + R: Fn(String) -> std::result::Result, E> + Send + Sync + 'static, + E: std::error::Error + Send + Sync + 'static, + { + let read = Arc::new(read); + let read_markdown = Arc::clone(&read); + let section = self + .lua + .create_function(move |_, target: Value| { + let text = read_markdown(resolve_section_target(target)?).map_err(mlua::Error::external)?; + Ok(text.map(|text| text.markdown)) + }) + .map_err(Error::lua)?; + let list = self + .lua + .create_function(move |lua, target: Value| { + let heading = resolve_section_target(target)?; + match read(heading.clone()).map_err(mlua::Error::external)? { + None => Ok(Value::Nil), + Some(text) if text.items.is_empty() => Err(mlua::Error::external(format!( + "section `{}` has no pre-parsed items", + heading.trim() + ))), + Some(text) => pack_sequence(lua, text.items).map(Value::Table), + } + }) + .map_err(Error::lua)?; + let prompt = self.lua.create_table().map_err(Error::lua)?; + prompt.raw_set("section", section).map_err(Error::lua)?; + prompt.raw_set("list", list).map_err(Error::lua)?; + globals.raw_set("prompt", prompt).map_err(Error::lua) + } + ``` +- File and public API changes: + - Parser: + - [crates/promptforge-internal/parser/src/prompt.rs](crates/promptforge-internal/parser/src/prompt.rs): the `markdown` field and accessor. + - [crates/promptforge-internal/parser/src/build.rs](crates/promptforge-internal/parser/src/build.rs): `collect_headings` already slices each heading's content from the end of its heading line to the start of the next heading of any level. `build_sections` sets `markdown: trim_leading_blank_lines(&h.content).trim_end().to_owned()` in its `result.push(Section { ... })` literal, where `h`, the current `Heading`, is still in scope. + - [crates/promptforge-internal/parser/src/fence.rs](crates/promptforge-internal/parser/src/fence.rs): `trim_leading_blank_lines` becomes `pub(crate)`. Plain `trim()` would strip the first content line's indentation. + - [crates/promptforge-internal/parser/src/test_support.rs](crates/promptforge-internal/parser/src/test_support.rs): `synthetic_section` sets `markdown: String::new()`. + - `Section` is not on the facade surface, so `crates/promptforge/public-api.txt` stays the same. + - Engine: + - [crates/promptforge-internal/engine/src/heading_address.rs](crates/promptforge-internal/engine/src/heading_address.rs): `find_visible` and the adapter as above. The adapter's callers (`resolve_chain_target` in `engine/src/execute/scheduler/walk.rs`, `end_live_h1_at_jump` in `engine/src/execute/scheduler/h1.rs`, and `resolve_jump_target` in `walk_target.rs`) don't change. Module docs name `prompt.section` and `prompt.list` in place of `list_from_section` and say `find_visible` is the one lookup. + - [crates/promptforge-internal/engine/src/execute/walk_target.rs](crates/promptforge-internal/engine/src/execute/walk_target.rs): `read_visible` replaces `list_items_from_visible`; module docs. `home_without`'s doc comment keeps only its first paragraph. Its second describes a fanout arm whose slice lacks the caller, and no such slice exists any more: a spawned arm runs on the worker's own slice (`tasks.rs` hands `target_section.slice` to `start_chain`), and both callers of `visible_sections` pass the section's own slice. A doc-only change; the code stays as it is. + - [crates/promptforge-internal/engine/src/execute/section_context-construct.rs](crates/promptforge-internal/engine/src/execute/section_context-construct.rs): in `new` and `new_live_h1`, `list_callback` becomes `read`, calling `read_visible`. The visible set still moves into one closure. Docs (five mentions). + - [crates/promptforge-internal/engine/src/execute/section_vm.rs](crates/promptforge-internal/engine/src/execute/section_vm.rs): `setup_section_vm` takes `read`; docs (two mentions). + - [crates/promptforge-internal/engine/src/lua.rs](crates/promptforge-internal/engine/src/lua.rs): re-export `SectionText`. + - Lua crate: + - [crates/promptforge-internal/lua/src/handles.rs](crates/promptforge-internal/lua/src/handles.rs): `SectionText`, beside `resolve_section_target`, and the module doc's first line, which today names only tool bindings, the tool set, and the output kind, names it too; [lib.rs](crates/promptforge-internal/lua/src/lib.rs) adds it to `pub use handles::{...}`. + - [crates/promptforge-internal/lua/src/vm/install.rs](crates/promptforge-internal/lua/src/vm/install.rs): `install_prompt_table` replaces `install_list_global`; the two install functions take `read`; every doc mention of `list_from_section` names the `prompt` reads instead. + - [crates/promptforge-internal/lua/src/globals.rs](crates/promptforge-internal/lua/src/globals.rs): `("list_from_section", ...)` becomes `("prompt", Reserved::EngineGlobal)` between `plugins` and `prose`; the array stays 61. + - [crates/promptforge-internal/lua/src/collection.rs](crates/promptforge-internal/lua/src/collection.rs): the error becomes "fanout's second parameter is a collection; for a list section use prompt.list(heading), which returns nil when no visible section has that heading", plus its doc comment and test assertion. A message string only. + - Test stubs, return type only (`Ok(None)`, or the existing `Err` reworded to "prompt reads are not needed here"), under `crates/promptforge-internal/`: `lua/src/tests.rs`, `lua/src/prelude-tests.rs`, `lua/src/globals-tests.rs`, `lua/src/tests/shared_replay.rs` (two sites), `engine/src/lua/tests.rs`, and `engine/src/lua/tests/globals.rs`. The two engine stubs bind a local named `list_callback`; it becomes `read`. + - Docs, [../promptforge-design/design/architect-promptforge-prompting-language.md](../promptforge-design/design/architect-promptforge-prompting-language.md), in the separate `promptforge-design` repository that sits beside `promptforge` and is committed on its own: + - line 48 (the L index line) drops `list_from_section(heading)`; line 52 (the P line) adds `prompt.list(heading)` and `prompt.section(heading)` after `pfanout` and before `promptforge: 0`, where they sort + - line 92: the whole sentence is corrected, not just the read. `call`, `jump`, and `list_from_section` leave the list of forms it calls unavailable in H1, which keeps `call_async` and the deferred `execute` and `execute_async`, and the sentence says `call`, `jump`, and both `prompt` reads work in H1 over the whole top-level slice + - line 125, the deferred `markdown` note, says heading-resolved reads live in `prompt` and `markdown` stays for parsing arbitrary text + - lines 362 to 366: the `list_from_section(heading)` entry becomes entries for `prompt.section(heading)` and `prompt.list(heading)`, each opening with an `**Implementation status:**` line like its neighbors, then the same Effect, Return, Failure, Example format. The status lines describe behavior without naming the removed global, because the language is at `promptforge: 0`, where removed forms simply stop existing, and the exit search over the spec must find no `list_from_section`: `prompt.list` reads "Active: the list-section read, now `nil` on a miss", and `prompt.section` reads "Active: new in this plan" +- Data, persistence, failure, security, and privacy constraints: + - Nothing is persisted, and no event or effect changes. A read is a pure function of the prompt and the caller's visible set, so replay is unaffected. + - No scheduler, task runtime, Harness, or public API code changes. + + + + +## Testing Plan + +The test suite gets smaller. One parser test pins the span, one unit test pins the lookup's miss, and one engine file covers both reads end to end. That file has three tests over one shared document, replacing six, and three tests elsewhere that only repeat coverage are deleted. Suites that already use the list read keep running under the new name. + +- Unit: + - `crates/promptforge-internal/parser/src/tests/headings.rs`, one new test, `section_markdown_is_the_sections_own_content`. One prompt checks three values: + - a section whose content has an indented first line, a Lua fence, a `---` break, and `{{ x }}` comes back exactly: heading line excluded, surrounding blank lines trimmed, stopping at its child heading. The `---` has a blank line above it: directly under a text line it is a setext heading underline (see `rule_ranges` in `fence.rs`), which would turn that line into a new section + - the child's own Markdown + - `""` for an empty section + - `crates/promptforge-internal/engine/src/heading_address-tests.rs`, one new test, `find_visible_returns_none_for_an_address_not_in_the_set`: a missing name and a level mismatch are `Ok(None)`, and the exact address is `Some`. The seven existing `resolve_sibling` tests stay, and they now cover `find_visible`'s error paths through the adapter. + - Reserved names need no new test: `crates/promptforge-internal/engine/src/lua/tests/globals.rs` already compares `RESERVED_NAMES` with a set-up VM's globals in both directions, so it pins `prompt` present and `list_from_section` absent. +- Integration and end-to-end: + - Engine test paths in this list are relative to `crates/promptforge-internal/engine/src/execute/tests/`. That directory holds two different `exec_flow.rs` files: `exec_flow.rs` at its top, and `suite/exec_flow.rs`, which declares the `suite/exec_flow/` modules and defines the `flow_prompt!` macro (it prepends only frontmatter). + - `suite/exec_flow/list_from_section.rs` becomes `suite/exec_flow/prompt_reads.rs`, with its module line and doc in `suite/exec_flow.rs`. One shared document, written once as a `const` and joined after each test's `## Main` block at runtime (the `[..].concat()` form `suite/exec_flow/fanout_arms.rs` uses, since `flow_prompt!` takes only a literal), holds: + - `### Kid`, a child of `## Main`, with a grandchild `#### Grand` + - a numbered `## Nums` + - a raw `## Rules` whose body is its own `const RULES`, spliced into the document: a Lua fence, `{{ args }}`, and a `---` with a blank line above it. `RULES` starts on a non-blank line and ends without a trailing newline or space, because the read trims both ends and the test compares byte for byte; the document supplies the blank lines around it + - an empty `## Empty` + - `## Other`, with a niece `### Niece` + - Every test's `## Main` ends with a scalar `return`, which ends the run, so the walk never reaches the shared sections and `## Rules`'s fence never runs. Each Lua `assert` includes a message naming its case, so a failure in the shared loop says which one. + - The three tests: + - `prompt_reads_resolve_over_the_visible_set`: numbered items with markers stripped, a direct child's items, a list section's raw bullets, `""` for the empty section, and `nil` from both reads for a missing heading, the niece, the grandchild, and the caller, checked in one Lua loop. `## Main` then returns `prompt.section('## Rules')`, and Rust compares the run's output to `RULES` with `assert_eq!`. The expected text is never written inside a Lua string: a line that is exactly the closing fence marker would end `## Main`'s own fence, because the fence scanner closes at the first such line. + - `prompt_reads_raise_only_for_mistakes_in_the_call`: under `pcall`, a bare-name heading and a non-string argument raise from both reads, and `prompt.list('## Rules')` raises "has no pre-parsed items". A `call` to a missing heading still raises, and its listing (the text after "available sections:") names a sibling and a direct child but not the caller, the niece, or the grandchild. That assertion keeps all three of the old not-found-listing checks; no other engine test asserts what the listing leaves out. + - `prompt_reads_work_in_h1_and_the_shared_library`: the H1 block reads a top-level section and list and gets `nil` for `### Kid`. It asserts and returns nothing, because a scalar return from H1 ends the run before `## Main` executes. A `lua shared` library reads `prompt.list('### Kid')` into a global at load time, so the replay resolves over each VM's own visible set: the H1 block sees that global as `nil`, and `## Main`, whose child `### Kid` is, sees its items (`suite/exec_flow/store_failures.rs` shows how a test writes the `lua shared` fence) + - Kept under the new name, test function names included (`list_from_section_inside_a_fanout_arm_reads_items` becomes `prompt_list_inside_a_fanout_arm_reads_items`, and its doc comment, which describes the arm's visible set as "the set the worker was resolved from, minus the worker, plus its children", says instead that the arm resolves like any section: the worker's siblings minus the worker, plus its children): `suite/exec_flow/fanout_arms.rs` reading items inside an arm, `suite/exec_flow/fanout_collections.rs` feeding `fanout` from `prompt.list`, `live_infer.rs`, `model_and_reply-globals.rs`, and the fixtures `crates/promptforge-internal/engine/tests/prompts/execution/fanout-basic.md`, `fanout-epilog.md`, `fanout-store-writes.md`, and `fanout-arm-failure.md`. + - Changed for the `nil` rule: + - the list half of `suite/exec_flow/fanout_arms.rs`'s `call_and_list_inside_a_fanout_arm_reject_non_visible_sections` asserts `prompt.list('## Parent') == nil`; its doc comment and `expect` message stop saying both error not-found + - `suite/exec_flow/fanout_collections.rs`'s two-string test becomes `fanout_string_second_parameter_errors_pointing_at_prompt_list` and asserts on `prompt.list` + - Deleted as redundant: + - `exec_flow.rs` `list_from_section_ambiguous_error_is_loud`: `heading_address-tests.rs` `resolve_sibling_rejects_more_than_one_match` pins the same guard + - `suite/exec_flow/h1_control.rs` `list_from_section_from_h1_to_an_unknown_section_fails_the_run`: a miss no longer fails, and `prompt_reads_work_in_h1_and_the_shared_library` covers H1 + - `scheduler/live_h1/control_flow.rs` `list_from_section_works_on_the_h1`: covered by the same H1 test + - Module docs that list `list_from_section` are updated in `exec_flow.rs`, `suite/exec_flow.rs`, `suite/exec_flow/h1_control.rs`, `suite/exec_flow/fanout_arms.rs`, and `scheduler/live_h1/control_flow.rs`, and the doc comment on the `suite/exec_flow/fanout_collections.rs` two-string test. +- Regression, security, and performance: + - The `call` and `jump` suites pass unchanged, which pins that the adapter kept their behavior and messages. + - A search over `crates/` and the language spec finds no `list_from_section` outside history files. + - The docs-claims test passes. +- Exit criteria: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` + - both clippy runs with `CARGO_BUILD_WARNINGS=deny`: `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets` + - `cargo fmt --all --check` + - the docs build (`cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api`) and the facade docs build (`cargo doc -p promptforge --no-deps`), both with `RUSTDOCFLAGS="-D warnings"` + - `cargo +nightly-2026-09-05 xtask api --check`; the nightly is the one pinned in `crates/build-xtask/src/api/toolchain.rs`, so use whatever that file names if it has moved + - `cargo test -p build-xtask` + - the docs-claims test: `node --test test/docs-claims.mjs`, run from `crates/workshop/ui` (`npm test` there runs it with the rest of the UI tests) + - `rg -n list_from_section crates` and `rg -n list_from_section ../promptforge-design/design/architect-promptforge-prompting-language.md` both print nothing. This is the only gate that catches a leftover test name or doc comment. + + + + +## Decision Record + +- Decisions: + - `prompt.section(heading)` returns a section's own raw Markdown: no heading line, no subsections, fences and `---` included, placeholders unrendered. Rationale: it is exactly what the section parses into blocks, and raw text is the only form in which Lua fences survive. User: "I want something like `section('### Instructions')` to return the section's markdown in its entirety including any fenced blocks or areas delineated with ---", then chose "Its own content only (no heading line, no subsections)". + - Heading resolution follows `list_from_section`: siblings and direct children only, never grandchildren. Rationale: every heading surface agrees on what a section may name, and a miss can't expose deeper structure. User: "I want the scope of the passed section heading to follow the scoping rules of list_from_section. You can only name a sibling or a child" and "I agree. This is a good scoping rule". + - The reads live in a `prompt` namespace, `prompt.section` and `prompt.list`, and `call`, `jump`, `prose`, and `sys` stay where they are. Rationale: one reserved name, the same namespace convention as `tools` and `models`, and a clear split from the deferred `markdown` namespace. A local named `prompt` shadows the namespace in its scope, an accepted tradeoff. User: "I like prompt.list and prompt.section". + - Both reads return `nil` when the heading doesn't exist or names a section outside the caller's visible set. Rationale: a read is naturally optional, a heading used as an index needs a miss that isn't an error, and an empty visible section stays distinct as `""`. User: "prompt.section(x) should return nil if x doesn't exist or names an out of scope section", then "prompt.list should return nil. this lets you use section headings as indexes for example" with sections `### anthropic`, `### openai`, and `### gemini`, where `prompt.list("### grok")` gives `nil`. + - `list_from_section` is removed outright, with no stub. User: "it should remove list_from_section". + - One function, `find_visible`, answers every heading lookup, and two adapters decide what a miss means: `nil` for the reads, today's not-found error for `call` and `jump`. Rationale: every heading surface already searches only the visible set through `resolve_sibling`, so splitting its not-found branch out gives one lookup with no new rule. "Out of scope" and "doesn't exist" are one answer, because telling them apart would need the whole document and would let a miss reveal what the visible-set rule hides. User: "I want to make sure that there's only one function that calculates the scope, right? There should just be one utility function, given a section name, it tells you, you know, it gives you the section, or it tells you that it's out of scope, or it tells you that it, that it doesn't exist, maybe, something." + - One `read` callback returning `SectionText` replaces the list callback, rather than a second callback beside it. Rationale: no signature gains a parameter, the Engine shares no visible set between closures, the seven test stubs change only their return type, and the "no pre-parsed items" check moves to the `prompt.list` surface where its error belongs. User: "find the minimum change. Can we reuse existing code? Can any existing code be removed?" and "make this a beautiful, crisp, concise, scoped change that reduces technical debt rather than increasing it". + - The tests consolidate into one file of three tests over one shared document, and three tests that only repeat coverage are deleted. User: "How do we make the tests comprehensive and compact without bloating them, right? We want to avoid bloat." + - The plan stays out of task execution. User: "we dont want to touch anything related to task execution". The only contact is text: `list_from_section` calls inside `fanout` test prompts and fixtures are renamed, one `fanout` test asserts `nil`, and the `fanout` collection error names `prompt.list`. + - Defaults chosen in planning without discussion, open to override: + - both reads raise for a malformed heading and a non-string argument, and return `nil` only for no match + - `prompt.list` still raises for a visible section with no pre-parsed items, and the message names the heading the author wrote, trimmed as `resolve_sibling`'s messages trim it, rather than the bare section name + - `resolve_sibling` keeps its name, since its other callers live in the scheduler + - the new accessor and field are `markdown`, not `source`, because `LuaProgram::source` in the same crate means Lua code + - `SectionText` derives no `Default`, since nothing constructs an empty one +- Rejected alternatives: + - Two callbacks, one per read, sharing an `Arc` visible set. Reason: parallel plumbing through three layers and a new parameter at every call site. No revisit. + - Telling "out of scope" apart from "doesn't exist". Reason: it needs the whole document, both read as `nil` anyway, and it would leak structure. No revisit. + - Removing the lookup's ambiguity check. Reason: a parsed prompt can't reach it (the parser rejects duplicate sibling names, a section's children sit one level below its siblings, and H1's set is the top-level slice alone; fanout arms and `call` chains run on the target's own slice through `resolve_chain_target`, so they follow the same rule), but it also guards `call` and `jump`, which this plan leaves alone. Revisit in a cleanup of those paths. + - Renaming `resolve_sibling`, which resolves children too. Reason: its callers are in the scheduler. Revisit with the H1 cleanup under Deferred. + - Returning the heading line or the whole subtree from `prompt.section`. Reason: the user chose the section's own content. Revisit if authors need a subsection's text with its parent's. + - Letting a section name its grandchildren. Reason: it breaks the one scoping rule. Revisit as a one-level-per-step path form if authors need it. + - A bare `section` global, or a `doc` or `document` namespace. Reason: the user chose `prompt`. Revisit if shadowing by locals named `prompt` causes bugs. + - Moving `sys.section_name` and `sys.section_count`, or `call`, `jump`, and `prose`, under `prompt`. Reason: `sys` is the sealed run-metadata table, `call` and `jump` move through the document rather than read it, and `prose` is the current block's text. No revisit. + - Raising on a miss, or returning `nil` for a malformed heading too. Reason: the user chose `nil` for misses, and a malformed heading is a mistake in the call. Revisit if authors want a strict read; `assert(prompt.section(h), msg)` is the one-line form. + - Keeping `list_from_section` as a deprecated alias. Reason: the user chose removal. No revisit. +- Assumptions, risks, and notes: + - Assumption: no Plugin prelude in `crates/` defines a `prompt` or `section` global (searched at planning time), so the new reserved name collides with nothing. + - Assumption: `list_from_section` already works from H1, though the language spec's line 92 says it doesn't; `crates/promptforge-internal/engine/src/execute/tests/scheduler/live_h1/control_flow.rs` pins it today. The same sentence also calls `call` and `jump` unavailable in H1, which is wrong too: `resolve_chain_target`'s H1 branch in `engine/src/execute/scheduler/walk.rs` and `end_live_h1_at_jump` in `engine/src/execute/scheduler/h1.rs` implement both. This plan rewrites that sentence anyway, so it corrects all three forms rather than leave a known-false claim in a sentence it edits. + - Risk: because both reads return `nil` on a miss, a misspelled heading fails silently and surfaces later, for example as "attempt to concatenate a nil value" or as the `fanout` collection error. Authors who want a strict read write `assert(prompt.section(h), 'missing section: ' .. h)`. + - Risk: authors who name a local `prompt` hide the namespace for the rest of that scope. + - Note: the spec edit is committed in `promptforge-design` (`c:\Users\Vinnie\cursor\promptforge-design`) as its own commit, staging only the spec file and none of that repository's other untracked files. It goes with the step that removes `list_from_section`, which the spec describes, so no step's `promptforge` commit holds only a plan marker. Sub-agents leave the spec edit unstaged like any change; the session running the step commits it after the coding sub-agent returns and before that step's `promptforge` commit, and amends it after any later round that changes it. Nothing is pushed. + - Note: `SectionText` copies both the Markdown and the items on every read, even though each read uses one. Section text is small, and one callback is simpler than two. + - Note: repository paths are relative to the `promptforge` repository unless they start with `../`, and line numbers are from planning time, so search by name if they have moved. History files that mention `list_from_section` stay as they are: `vibe/` in `promptforge`, and `design-prev/`, `chats/`, and `research/` in `promptforge-design`, along with the prose-lint word list in `promptforge-design/saved-docs/cicerone/scripts/checks.py`. + +### Deferred and Out of Scope + +- Deferred: one helper for H1's visible set. The top-level slice is spelled out in three places under `crates/promptforge-internal/engine/src/execute/`: `section_context-construct.rs`, `scheduler/walk.rs`, and `scheduler/h1.rs`; folding them together touches the scheduler. Revisit in a change that already edits the `call` and `jump` paths. +- Deferred: grandchild addressing as a one-level-per-step path form. Revisit if authors need it. +- Deferred: a `markdown` namespace for parsing arbitrary text. Revisit when a prompt needs to parse text it didn't get from its own sections. +- Out of scope: task execution of any kind, including `tasks`, `fanout` behavior, subagents, the scheduler, and tool-call dispatch. +- Out of scope: the Harness, the package registry, and any Host-facing API. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build --locked -p gateway` (`crates/gateway/app` is the workspace's only default member, so plain `cargo build` builds just the gateway); the desktop app is an explicit `cargo build --locked -p workshop`. Build scripts bundle the UIs into `OUT_DIR`, so a fresh clone first runs `npm ci --prefix crates/workshop` and `npm ci --prefix crates/gateway/config-ui/ui`, as `.github/workflows/ci.yml` does. +- Focused test command pattern: `cargo nextest run --locked -p --all-features `, dropping `--all-features` for `workshop`, `workshop-server`, and `workshop-server-api`. A single JS test file runs with `node --test ` from its package directory. +- Component test command pattern: `cargo nextest run --locked -p --all-features` for every crate except the three workshop crates above, which run without `--all-features`; CI adds separate runs for `workshop-workspace --all-features` and `workshop-server --features headless`. JS packages: `npm test --workspace ` from `crates/workshop`, or `npm test` from `crates/gateway/config-ui/ui`. +- Full-suite test command: `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api`. The first run includes `build-xtask`, whose tests are the boundary and structural checks (`cargo xtask tidy` prints the same report). Also run `node --test crates/workshop/ui/test/docs-claims.mjs` (wording scan, no `npm ci` needed), `npm test --workspaces --if-present` in `crates/workshop`, and `npm test` in `crates/gateway/config-ui/ui`. Nextest config is `.config/nextest.toml`: 60 s slow timeout, terminated after 3 periods, and the STT crates run in a capped `heavy` group. +- Linter command: `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, both with `CARGO_BUILD_WARNINGS=deny` (in PowerShell, `$env:CARGO_BUILD_WARNINGS='deny'`). Also `cargo check -p gateway --no-default-features` for the headless build shape (the only allowed standalone `cargo check`), `npm run typecheck --workspaces --if-present` in `crates/workshop`, `npm run typecheck` in `crates/gateway/config-ui/ui`, and `cargo deny check` plus `cargo hakari verify` for dependency policy. The `.githooks/pre-push` hook runs the wording scan, the headless check, the first clippy run, and `cargo deny` when installed. +- Formatter check command: `cargo fmt --all --check` (stable toolchain per `rust-toolchain.toml`, `style_edition = "2024"` in `rustfmt.toml`; `.githooks/pre-commit` runs it). +- Docs command: `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` with `RUSTDOCFLAGS="-D warnings"`. Under the same flags CI also runs `cargo doc -p promptforge --no-deps` and `cargo doc -p harness --no-deps` with default features, and `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items`. Facade surface: `cargo +nightly-2026-09-05 xtask api --check` against the committed `crates/promptforge/public-api.txt` (the nightly is pinned in `crates/build-xtask/src/api/toolchain.rs`; `--bless` rewrites the listing and refuses while violations remain), plus `cargo nextest run --locked -p build-xtask --run-ignored only` on that nightly for the rustdoc JSON fixtures. +- Test placement and naming conventions: Unit tests live in a sibling file wired with `#[cfg(test)] #[path = "-tests.rs"] mod tests;` (for example `crates/promptforge-internal/lua/src/handles.rs` beside `handles-tests.rs`), or in a `tests/` subdirectory of the module once it has three or more test files (for example `crates/promptforge-internal/engine/src/execute/tests/`). Integration tests compile as one binary per crate from `tests/it/main.rs` (most crates) or `tests/suite/main.rs` (`promptforge`, `harness`), with shared helpers in `support.rs` and one file per topic (`cancel.rs`, `replay.rs`, `vfs.rs`). Test function names are snake_case sentences describing the behavior (`a_run_can_move_between_threads_between_calls`). Plugin crates enable `promptforge-plugin`'s `test-support` feature from `[dev-dependencies]` only, which `build-xtask`'s `test_support_leak` guard enforces. Doctests are banned in every workspace crate (`build-xtask` `no_doctests`). `build-xtask` fixtures that need rustdoc JSON are `#[ignore]`d on stable. JS tests are `node:test` files in `test/**/*.mjs` (workshop `ui`, `look`, `platform`) and `src/**/*.test.mjs` (workshop `ui` and the gateway config UI). +- Directory map: `crates/` holds every crate. `promptforge` is the Engine facade (with `public-api.txt`), and the manifestless `promptforge-internal/` container holds the private Engine crates `types`, `vfs`, `parser`, `lua`, `model-client`, and `engine`. `promptforge-plugin` is the Plugin contract, implemented by `plugin-web`, `plugin-mcp`, and `plugin-user-input`. `harness` is the Harness facade, `harness-internal/runner` its private runner, and `harness-gateway-client` the gateway client. `gateway/` holds the gateway family (`app` binary, `config`, `config-ui`, `local`, `routing`, `protocol`, `cloud-providers`, `web-search`, `logging`, `progress`, and the `stt/` speech subsystem with `api`, `engine`, `backend-whisper`, `whisper-ffi`); `gateway-api-types` and `gateway-api-discovery` are its public pair. `workshop/` holds the Workshop Host: the `desktop` Tauri app, `server`, `server-api`, the tiered library crates, and the `ui`, `look`, `platform` npm workspaces. Also in `crates/`: `shared-error-source`, `shared-loopback`, and `shared-ui` (a TypeScript and CSS package, not a crate); `build-*` tooling (`build-xtask` for structural checks and `cargo xtask`, `build-workshop` behind `cargo workshop`, `build-ceiling` for the file-size limit, `build-ui`, `build-user-guide`, `build-llama-cuda`); and `workspace-hack` (cargo-hakari, configured in `.config/hakari.toml`). Top level: `guide/books/` (mdBook books `language`, `gateway`, `workshop`) with `guide/chrome/` and `guide/landing/` site assets, `prompts/` (sample prompts), `tools/` (Node scripts for gateway sidecar staging and live TTS, with tests), `.github/workflows/` (CI, release, nightly), `.githooks/`, `.cursor/rules/` (`workshop-architecture.mdc`, `workshop-spa.mdc`), `vibe/` (past plans, tracked), `local/` (gitignored machine-local config and fixtures), and `images/`. +- Component boundaries: `cargo test -p build-xtask` enforces these (`crates/build-xtask/src/product.rs` and `tidy.rs`). Engine crates (`promptforge*`) depend on no gateway, workshop, Harness, or Plugin crate, and their manifests may not name `tokio`, `tokio-util`, `async-trait`, or `reqwest` outside `[dev-dependencies]` (`engine_deps.rs`). Inside the Engine, `types` and `vfs` are leaves (`vfs` has no dependencies at all), `model-client` depends on `types`, `lua` on `types`, `vfs`, and `model-client`, `parser` on `lua` and `types`, `engine` on all five, and the `promptforge` facade on all six. `promptforge-plugin` depends on `types` and `vfs`. Plugin crates depend only on `promptforge-plugin`, `shared-*`, `workspace-hack`, and outside libraries. Harness crates name only `promptforge`, `promptforge-plugin`, and `workspace-hack` outside their family, except that `harness-gateway-client` may name `plugin-web` to implement its `SearchProvider`. Gateway crates depend on no Engine, Harness, or workshop crate. Workshop crates reach the gateway only through `gateway-api-types` and `gateway-api-discovery`, and the Engine and Harness only through `promptforge`, `promptforge-plugin`, `harness`, and `harness-gateway-client`. Containers are private: only `promptforge` and `promptforge-plugin` may enter `promptforge-internal/`, only `harness` may enter `harness-internal/`, and only `gateway-stt` is public within `gateway/stt/`. Workshop tiers run vocabulary (`protocol`, `registry`, `support`), then services (`gateway`, `menu`, `status`), then features (`agents`, `run-log`, `user-state`, `workspace`), then `workshop-server`, each depending only downward. The `workshop` desktop app depends only on `workshop-server-api`, which depends only on `workshop-server`. `shared-*` crates depend on no product crate. +- Conventions summary: The four defined terms in `AGENTS.md` (Engine, Harness, Host, Plugin) are capitalized with one meaning each, and Engine crates say "the caller" and never mention the Host (`crates/workshop/ui/test/docs-claims.mjs` scans for this). All 52 crate manifests set `publish = false` and `[lints] workspace = true`, inherit `version`, `edition`, `license`, and `repository`, and take dependencies as `.workspace = true`; every crate except `promptforge-vfs` and `build-ceiling` depends on `workspace-hack`. Every crate's build script runs `build_ceiling::check()`, which fails any file over 500 lines. Manifests comment why each dependency, pin, or feature exists. Lints are strict: clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, bare `#[allow]` is denied so suppressions use `#[expect(..., reason = "...")]`, `unsafe_code` is denied outside the allowlisted boundaries, `missing_docs` and `unreachable_pub` warn (fatal under the gate), and `clippy.toml` bans process-global installers outside binary entry points. Workshop, Harness, and Plugin crates must carry a `## Invariants` section in their crate docs. Source directories stay flat until a group reaches three files, using kebab siblings with `#[path]` below that. Comments explain only non-obvious constraints and cite upstream issue URLs for workarounds. Error messages are written for model consumption, naming required versus actual. JSON that reaches a recorder round-trips exactly, with sorted keys and `float_roundtrip`. Behavior changes ship with tests in the same change. Cargo features gate only real build constraints. Changes to the facade surface update `public-api.txt`. Commits are one-line imperative summaries (`Resolve model tool names by wire name only`), and a finished plan closes with `Close plan: `. + + + + +## Execution Instructions + + + +### Step 1: Keep each section's raw Markdown in the parser [completed] + +- Component: section-markdown +- Component placement: first. It depends on nothing, and Step 3's `prompt.section` reads `Section::markdown()`. The parser sits below the Engine in the crate graph, so it lands before the Engine-side lookup. +- Pieces: one, the `markdown` field with its accessor and test. Construction: sequential, since there is only one piece. +- Work item: `parser-markdown` +- Changes, under `crates/promptforge-internal/parser/src/`: + - `prompt.rs`: `pub(crate) markdown: String` on `Section` and `pub fn markdown(&self) -> &str`, with the doc comment from the Technical Design + - `fence.rs`: `trim_leading_blank_lines` becomes `pub(crate)`; plain `trim()` would strip the first content line's indentation + - `build.rs`: `build_sections` sets `markdown` in its `Section { ... }` literal from the current `Heading`'s `content`, through `trim_leading_blank_lines` and then `trim_end` + - `test_support.rs`: `synthetic_section` sets `markdown` to `String::new()` + - `crates/promptforge/public-api.txt` stays the same, since `Section` is not on the facade surface +- Tests: `crates/promptforge-internal/parser/src/tests/headings.rs` gains `section_markdown_is_the_sections_own_content`, one prompt checking three values: a section with an indented first line, a Lua fence, a `---` with a blank line above it (directly under text it would be a setext underline), and `{{ x }}`, returned byte for byte without its heading line and stopping at its child heading; the child's own Markdown; and `""` for an empty section +- Gate: `cargo nextest run --locked -p promptforge-parser --all-features`, the first clippy run with `CARGO_BUILD_WARNINGS=deny`, and `cargo fmt --all --check` +- Commit: `Keep each section's raw Markdown in the parser` + + + + + +### Step 2: Split find_visible out of resolve_sibling + +- Component: heading-lookup +- Component placement: second. It is logically independent of Step 1, but each gate builds the whole workspace, so in one checkout they run in sequence; running it beside Step 1 needs a separate worktree. Step 3's `read_visible` sits on `find_visible`. +- Pieces: one, the lookup and its adapter. Construction: sequential, since there is only one piece. +- Work item: `scope-lookup` +- Changes, under `crates/promptforge-internal/engine/src/`: + - `heading_address.rs`: `pub(crate) fn find_visible` is today's `resolve_sibling` body without its not-found branch, returning `Ok(None)` when no visible section has the heading's exact level and name and keeping the malformed-heading and ambiguity errors; `resolve_sibling` becomes a thin adapter that turns `None` into today's "not found; available sections" error with unchanged text + - `heading_address.rs` module docs say `find_visible` is the one heading lookup; their `list_from_section` mention changes in Step 3 + - the adapter's callers (`resolve_chain_target` in `execute/scheduler/walk.rs`, `end_live_h1_at_jump` in `execute/scheduler/h1.rs`, and `resolve_jump_target` in `execute/walk_target.rs`) don't change, and no source under `execute/scheduler/` is edited +- Tests: `heading_address-tests.rs` gains `find_visible_returns_none_for_an_address_not_in_the_set`: a missing name and a level mismatch are `Ok(None)`, and the exact address is `Some`. The seven existing `resolve_sibling` tests stay as they are and pin the adapter's messages. +- Gate: `cargo nextest run --locked -p promptforge-engine --all-features`, the first clippy run with `CARGO_BUILD_WARNINGS=deny`, and `cargo fmt --all --check` +- Commit: `Split find_visible out of resolve_sibling` + + + + + +### Step 3: Replace list_from_section with prompt.section and prompt.list + +- Component: prompt-namespace +- Component placement: last. It needs `Section::markdown()` from Step 1 and `find_visible` from Step 2, and it is the only component prompt authors see. +- Pieces: the table, the removal, the tests, and the language spec. Construction: joint. Removing `list_items_from_visible` and the `list_from_section` global, and moving every user of both (including the `exec_flow.rs` test that calls `list_items_from_visible` directly), must land together to keep the suite green, and the spec describes the removed global, so it lands with the removal. +- Work items: `prompt-change` and `docs`, closing with the full `verify` gate +- The table, under `crates/promptforge-internal/`: + - `lua/src/handles.rs`: `SectionText { markdown, items }` deriving `Debug, Clone, PartialEq, Eq` (no `Default`), beside `resolve_section_target`, and the module doc's first line names it; `lua/src/lib.rs` adds it to `pub use handles::{...}`, and `engine/src/lua.rs` re-exports it + - `lua/src/vm/install.rs`: `install_prompt_table` replaces `install_list_global` and builds `prompt.section` and `prompt.list` over one `Arc`-shared `read` callback; `prompt.list` raises "section `` has no pre-parsed items" with the trimmed heading; `install_scheduler_control_globals` and the test-only `install_control_globals` take `read` with `Send + Sync + 'static` bounds; every doc mention of `list_from_section` names the `prompt` reads + - `lua/src/globals.rs`: `("list_from_section", ...)` becomes `("prompt", Reserved::EngineGlobal)` between `plugins` and `prose`, and the array stays 61 + - `engine/src/execute/walk_target.rs`: `read_visible` over `heading_address::find_visible` replaces `list_items_from_visible`; `home_without`'s doc comment keeps only its first paragraph (doc-only); module docs + - `engine/src/execute/section_vm.rs`: `setup_section_vm` takes `read`; docs (two mentions) + - `engine/src/execute/section_context-construct.rs`: `new` and `new_live_h1` pass a `read` closure calling `read_visible`, with the visible set still moving into one closure; docs (five mentions) + - the seven test stubs change only their return type (`Ok(None)`, or the existing `Err` reworded to "prompt reads are not needed here"): `lua/src/tests.rs`, `lua/src/prelude-tests.rs`, `lua/src/globals-tests.rs`, `lua/src/tests/shared_replay.rs` (two sites), `engine/src/lua/tests.rs`, and `engine/src/lua/tests/globals.rs`; the two engine stubs rename their local `list_callback` to `read` +- The removal, under `crates/promptforge-internal/`: + - `lua/src/collection.rs`: the error becomes "fanout's second parameter is a collection; for a list section use prompt.list(heading), which returns nil when no visible section has that heading", with its doc comment and test assertion + - `engine/src/heading_address.rs`: module docs name `prompt.section` and `prompt.list` + - every remaining `list_from_section` call, test function name, and doc comment under `engine/src/execute/tests/`: `live_infer.rs`, `model_and_reply-globals.rs`, `suite/exec_flow/fanout_collections.rs`, and `suite/exec_flow/fanout_arms.rs`, where `list_from_section_inside_a_fanout_arm_reads_items` becomes `prompt_list_inside_a_fanout_arm_reads_items` and its doc comment says the arm resolves like any section (the worker's siblings minus the worker, plus its children); module docs in `exec_flow.rs`, `suite/exec_flow.rs`, `suite/exec_flow/h1_control.rs`, `suite/exec_flow/fanout_arms.rs`, and `scheduler/live_h1/control_flow.rs` + - the four fixtures under `engine/tests/prompts/execution/`: `fanout-basic.md`, `fanout-epilog.md`, `fanout-store-writes.md`, and `fanout-arm-failure.md` +- The tests, under `crates/promptforge-internal/engine/src/execute/tests/`: + - `suite/exec_flow/prompt_reads.rs` replaces `suite/exec_flow/list_from_section.rs`, with its module line and doc in `suite/exec_flow.rs`. One shared `const` document (`### Kid` with `#### Grand`, a numbered `## Nums`, a raw `## Rules` spliced from its own `const RULES`, an empty `## Empty`, and `## Other` with `### Niece`) is joined after each test's `## Main` with the `[..].concat()` form from `suite/exec_flow/fanout_arms.rs`. Three tests, built as the Testing Plan describes: `prompt_reads_resolve_over_the_visible_set`, `prompt_reads_raise_only_for_mistakes_in_the_call`, and `prompt_reads_work_in_h1_and_the_shared_library` + - the `nil`-rule edits: `call_and_list_inside_a_fanout_arm_reject_non_visible_sections` in `suite/exec_flow/fanout_arms.rs` asserts `prompt.list('## Parent') == nil`, with its doc comment and `expect` message no longer saying both error not-found; the two-string test in `suite/exec_flow/fanout_collections.rs` becomes `fanout_string_second_parameter_errors_pointing_at_prompt_list`, asserting on `prompt.list`, with its doc comment updated + - the deletions: `list_from_section_ambiguous_error_is_loud` in `exec_flow.rs`, `list_from_section_from_h1_to_an_unknown_section_fails_the_run` in `suite/exec_flow/h1_control.rs`, and `list_from_section_works_on_the_h1` in `scheduler/live_h1/control_flow.rs` +- The language spec, `../promptforge-design/design/architect-promptforge-prompting-language.md`: + - the L index line (line 48) drops `list_from_section(heading)`, and the P line (line 52) adds `prompt.list(heading)` and `prompt.section(heading)` after `pfanout` and before `promptforge: 0` + - the H1 availability sentence (line 92) is rewritten: the unavailable list keeps `call_async`, deferred `execute`, and deferred `execute_async`, and the sentence says `call`, `jump`, and both `prompt` reads work in H1 over the whole top-level slice + - the deferred `markdown` note (line 125) says heading-resolved reads live in `prompt` and `markdown` stays for parsing arbitrary text + - the `list_from_section(heading)` entry (lines 362 to 366) becomes `prompt.section(heading)` and `prompt.list(heading)` entries, each opening with an `**Implementation status:**` line ("Active: new in this plan" and "Active: the list-section read, now `nil` on a miss"), then Effect, Return, Failure, and Example, without naming the removed global +- Gate, the full exit criteria from the Testing Plan: both nextest runs; both clippy runs with `CARGO_BUILD_WARNINGS=deny`; `cargo fmt --all --check`; the docs build and the facade docs build with `RUSTDOCFLAGS="-D warnings"`; `cargo +nightly-2026-09-05 xtask api --check` (or whichever nightly `crates/build-xtask/src/api/toolchain.rs` names); `cargo test -p build-xtask`; `node --test test/docs-claims.mjs` from `crates/workshop/ui`; and `rg -n list_from_section crates` plus `rg -n list_from_section ../promptforge-design/design/architect-promptforge-prompting-language.md`, both printing nothing +- Commits, two, spec first, per the Decision Record note: in `promptforge-design`, staging only the spec file, `Specify prompt.section and prompt.list in place of list_from_section`, committed by the session after the coding sub-agent returns and amended after any later round that changes it; then in `promptforge`, `Replace list_from_section with prompt.section and prompt.list`. Nothing is pushed. + + + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..498efdf21 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-10-10-2-prompt-namespace-reads.md From a567e41a1353bf2bd0e5b7eb5b064fd3a1d853ef Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 10:25:48 -0700 Subject: [PATCH 02/34] Split find_visible out of resolve_sibling Split heading lookup into a search that reports a miss as an empty answer and a resolver that treats a miss as an error. The search looks only at the caller's visible sections, so a heading outside that set and a heading that names no section at all get the same empty answer, while a malformed or ambiguous heading still fails. The resolver now delegates to the search and keeps its not-found error and message text unchanged, so every heading lookup shares one matching rule. - `find_visible` is the shared lookup: it parses the heading, filters the given slice on exact level and name, and returns `None` on a miss while still erroring on a malformed or ambiguous heading. It takes the heading as a raw `&str` and parses it itself, as `resolve_sibling` and `parse_heading_address` do. - `resolve_sibling` is now a thin adapter that maps `None` from `find_visible` to the not-found error listing only the visible sections. Its signature and both error texts are unchanged, so no caller is edited. - `find_visible_returns_none_for_an_address_not_in_the_set` pins that a missing name and a level mismatch both yield `None` without error, and that the exact address yields the section. Design: new stringly-typed @ crates/promptforge-internal/engine/src/heading_address.rs::find_visible deps: Section,str Design: new pure-function @ crates/promptforge-internal/engine/src/heading_address.rs::find_visible deps: Section,str Design: extends pure-function @ crates/promptforge-internal/engine/src/heading_address.rs::resolve_sibling deps: Section,str Plan: vibe/2026-10-10-2-prompt-namespace-reads.md --- .../engine/src/heading_address-tests.rs | 12 ++++ .../engine/src/heading_address.rs | 68 +++++++++++-------- vibe/2026-10-10-2-prompt-namespace-reads.md | 2 +- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/crates/promptforge-internal/engine/src/heading_address-tests.rs b/crates/promptforge-internal/engine/src/heading_address-tests.rs index b6c9ac89c..ca191b259 100644 --- a/crates/promptforge-internal/engine/src/heading_address-tests.rs +++ b/crates/promptforge-internal/engine/src/heading_address-tests.rs @@ -66,3 +66,15 @@ fn resolve_sibling_rejects_more_than_one_match() { .expect_err("two identical siblings must be rejected as ambiguous"); assert!(err.to_string().contains("ambiguous"), "error was: {err}"); } + +#[test] +fn find_visible_returns_none_for_an_address_not_in_the_set() { + let sections = vec![sibling("Worker", 3)]; + let missing = find_visible("### Missing", §ions).expect("a missing name is not an error"); + assert!(missing.is_none(), "a missing name must be None"); + let wrong_level = + find_visible("## Worker", §ions).expect("a level mismatch is not an error"); + assert!(wrong_level.is_none(), "a level mismatch must be None"); + let found = find_visible("### Worker", §ions).expect("exact address resolves"); + assert_eq!(found.map(Section::name), Some("Worker")); +} diff --git a/crates/promptforge-internal/engine/src/heading_address.rs b/crates/promptforge-internal/engine/src/heading_address.rs index 4f4c90889..0f3c61dca 100644 --- a/crates/promptforge-internal/engine/src/heading_address.rs +++ b/crates/promptforge-internal/engine/src/heading_address.rs @@ -11,10 +11,10 @@ //! per member - every arm up front, before any join, so no arm is ordered //! after a sibling - and waits on the live set while the scheduler's //! admission limit bounds how many arms run at once. This module defines -//! [`resolve_sibling`], the one -//! heading resolution those surfaces share; the collection enumeration sits -//! in the `promptforge-lua` crate, beside the VM and the coroutine protocol -//! that consume it. +//! [`find_visible`], the one heading lookup those surfaces share, and +//! [`resolve_sibling`], its adapter for surfaces where a miss is an error; +//! the collection enumeration sits in the `promptforge-lua` crate, beside the +//! VM and the coroutine protocol that consume it. use crate::parser::Section; use crate::{Error, Result}; @@ -56,38 +56,30 @@ fn parse_heading_address(heading: &str) -> Result<(usize, String)> { Ok((level, name.to_owned())) } -/// Resolves a heading string like `"### Name"` against a caller's visible -/// sections, returning the single matching section. +/// Looks up a heading string like `"### Name"` in a caller's visible +/// sections, returning the single matching section or `None`. /// /// The heading is parsed into an exact `(level, name)` address; a section -/// matches only when BOTH its level and name are equal. Zero matches and more -/// than one match are both rejected, so an ambiguous or level-mismatched -/// heading never resolves to an arbitrary first hit. +/// matches only when BOTH its level and name are equal. A heading outside the +/// visible set and a heading that names no section anywhere are the same +/// `None`, because only the visible set is ever searched. More than one match +/// is rejected, so an ambiguous heading never resolves to an arbitrary first +/// hit. /// /// # Errors /// Returns [`Error::Lua`] when the heading is malformed (see -/// [`parse_heading_address`]), when no visible section matches the exact -/// address, or when more than one matches. The error message lists the -/// visible sections and nothing else, so the error channel cannot leak the -/// rest of the document's structure. -pub(crate) fn resolve_sibling<'a>(heading: &str, visible: &'a [Section]) -> Result<&'a Section> { +/// [`parse_heading_address`]) or when more than one visible section matches +/// the exact address. +pub(crate) fn find_visible<'a>( + heading: &str, + visible: &'a [Section], +) -> Result> { let (level, name) = parse_heading_address(heading)?; - let mut matches = visible .iter() .filter(|section| usize::from(section.level()) == level && section.name() == name); - let Some(found) = matches.next() else { - let available: Vec = visible - .iter() - .map(|s| format!("{} {}", "#".repeat(s.level().into()), s.name())) - .collect(); - return Err(Error::Lua(format!( - "section heading `{}` not found; available sections: {}", - heading.trim(), - available.join(", ") - ))); - }; - if matches.next().is_some() { + let found = matches.next(); + if found.is_some() && matches.next().is_some() { return Err(Error::Lua(format!( "section heading `{}` is ambiguous; more than one visible section matches {} {name}", heading.trim(), @@ -97,6 +89,28 @@ pub(crate) fn resolve_sibling<'a>(heading: &str, visible: &'a [Section]) -> Resu Ok(found) } +/// [`find_visible`] where a heading with no visible match is an error, for +/// the surfaces that must reach a section, such as `call` and `jump`. +/// +/// # Errors +/// Returns [`Error::Lua`] for every [`find_visible`] error, and when no +/// visible section matches the exact address. The not-found message lists the +/// visible sections and nothing else, so the error channel cannot leak the +/// rest of the document's structure. +pub(crate) fn resolve_sibling<'a>(heading: &str, visible: &'a [Section]) -> Result<&'a Section> { + find_visible(heading, visible)?.ok_or_else(|| { + let available: Vec = visible + .iter() + .map(|s| format!("{} {}", "#".repeat(s.level().into()), s.name())) + .collect(); + Error::Lua(format!( + "section heading `{}` not found; available sections: {}", + heading.trim(), + available.join(", ") + )) + }) +} + #[cfg(test)] #[path = "heading_address-tests.rs"] mod tests; diff --git a/vibe/2026-10-10-2-prompt-namespace-reads.md b/vibe/2026-10-10-2-prompt-namespace-reads.md index 177590a2b..89104271e 100644 --- a/vibe/2026-10-10-2-prompt-namespace-reads.md +++ b/vibe/2026-10-10-2-prompt-namespace-reads.md @@ -414,7 +414,7 @@ The test suite gets smaller. One parser test pins the span, one unit test pins t -### Step 2: Split find_visible out of resolve_sibling +### Step 2: Split find_visible out of resolve_sibling [completed] - Component: heading-lookup - Component placement: second. It is logically independent of Step 1, but each gate builds the whole workspace, so in one checkout they run in sequence; running it beside Step 1 needs a separate worktree. Step 3's `read_visible` sits on `find_visible`. From d24da1d754a5d8a421f9257da1ac901971bbef42 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 10:42:50 -0700 Subject: [PATCH 03/34] Replace list_from_section with prompt.section/list Section code now reads other sections through two Lua read functions. One returns a section's own raw Markdown, and the other returns a list section's items. Both search only the caller's visible set and return nil when no visible section has the heading, so a missing heading and an out-of-scope heading give one answer. They raise only when the call itself is wrong: a non-string target, a malformed or ambiguous heading, or a list read of a section with no list items. The old list-reading global is gone, and one lookup callback serves both reads. - `install_prompt_table` builds `prompt.section` and `prompt.list` over one `read` callback shared through an `Arc`. Every `read` bound therefore requires `Send + Sync + 'static`, and the "no pre-parsed items" check moves out of the Engine into `prompt.list`. - `read_visible` replaces `list_items_from_visible` as a pure adapter over `heading_address::find_visible`. It maps a visible match to a `SectionText` and a miss to `None`. - `SectionText` is a new exported type in the Lua crate that holds a section's raw `markdown` and its `items`. The Engine's `lua` module re-exports it. - `prompt.list` returns `nil` for a heading outside the visible set or absent from the document, where the old global raised not-found. It still raises for a section with no pre-parsed items, and that message now names the trimmed heading as written instead of the section's bare name. - `prompt.section` returns a section's Markdown verbatim, without its child sections, and the empty string for an empty section. A `lua` fence or template inside it comes back as text. - `RESERVED_NAMES` reserves `prompt` as an Engine global in place of `list_from_section`, so a Plugin prelude can no longer claim `prompt`. The array length stays 61. - `collection_members` now points a fanout caller that passes a string at `prompt.list(heading)` and states that it returns nil on a miss. - `prompt_reads_work_in_h1_and_the_shared_library` pins that a shared-library read resolves per VM: at load time over each replaying VM's own visible set, and from a shared function over the calling section's. - `list_from_section_ambiguous_error_is_loud` is deleted, and no test in this change drives an ambiguous heading through either read. Design: new value-object @ crates/promptforge-internal/lua/src/handles.rs::SectionText Design: replaces pure-function @ crates/promptforge-internal/engine/src/execute/walk_target.rs::read_visible deps: Section,str was: crates/promptforge-internal/engine/src/execute/walk_target.rs::list_items_from_visible Design: new surface-growth @ crates/promptforge-internal/lua/src/vm/install.rs::SectionVm::install_prompt_table boundary: pub Design: new shotgun-surgery @ crates/promptforge-internal/lua/src/vm/install.rs::SectionVm::install_scheduler_control_globals Plan: vibe/2026-10-10-2-prompt-namespace-reads.md --- .../src/execute/section_context-construct.rs | 34 ++-- .../engine/src/execute/section_vm.rs | 17 +- .../engine/src/execute/tests/exec_flow.rs | 16 +- .../engine/src/execute/tests/live_infer.rs | 2 +- .../execute/tests/model_and_reply-globals.rs | 2 +- .../tests/scheduler/live_h1/control_flow.rs | 28 +-- .../src/execute/tests/suite/exec_flow.rs | 4 +- .../tests/suite/exec_flow/fanout_arms.rs | 26 ++- .../suite/exec_flow/fanout_collections.rs | 13 +- .../tests/suite/exec_flow/h1_control.rs | 25 +-- .../suite/exec_flow/list_from_section.rs | 177 ------------------ .../tests/suite/exec_flow/prompt_reads.rs | 171 +++++++++++++++++ .../engine/src/execute/walk_target.rs | 38 ++-- .../engine/src/heading_address.rs | 16 +- crates/promptforge-internal/engine/src/lua.rs | 4 +- .../engine/src/lua/tests.rs | 7 +- .../engine/src/lua/tests/globals.rs | 7 +- .../prompts/execution/fanout-arm-failure.md | 2 +- .../tests/prompts/execution/fanout-basic.md | 2 +- .../tests/prompts/execution/fanout-epilog.md | 2 +- .../prompts/execution/fanout-store-writes.md | 2 +- .../lua/src/collection.rs | 8 +- .../lua/src/globals-tests.rs | 4 +- .../promptforge-internal/lua/src/globals.rs | 2 +- .../promptforge-internal/lua/src/handles.rs | 11 +- crates/promptforge-internal/lua/src/lib.rs | 2 +- .../lua/src/prelude-tests.rs | 6 +- crates/promptforge-internal/lua/src/tests.rs | 2 +- .../lua/src/tests/shared_replay.rs | 12 +- .../lua/src/vm/install.rs | 62 +++--- vibe/2026-10-10-2-prompt-namespace-reads.md | 7 +- 31 files changed, 325 insertions(+), 386 deletions(-) delete mode 100644 crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/list_from_section.rs create mode 100644 crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/prompt_reads.rs diff --git a/crates/promptforge-internal/engine/src/execute/section_context-construct.rs b/crates/promptforge-internal/engine/src/execute/section_context-construct.rs index 7db26046f..878fb3bf9 100644 --- a/crates/promptforge-internal/engine/src/execute/section_context-construct.rs +++ b/crates/promptforge-internal/engine/src/execute/section_context-construct.rs @@ -5,8 +5,8 @@ //! a live [`SectionContext`] whose `Drop` is the teardown boundary. The //! setup half (Engine injection, Engine globals, the control surface, the //! Plugin preludes, the shared replay, the store yield shims) is shared; -//! only the seed, the `sys` extras, and the `list_from_section` visible -//! set differ. +//! only the seed, the `sys` extras, and the visible set behind the +//! `prompt` reads differ. use std::sync::Arc; @@ -15,7 +15,7 @@ use promptforge_types::ids::{ChainId, TaskId}; use crate::Result; use crate::execute::context::RunState; use crate::execute::section_vm::{VmSeed, setup_section_vm}; -use crate::execute::walk_target::{list_items_from_visible, visible_sections}; +use crate::execute::walk_target::{read_visible, visible_sections}; use crate::lua::SectionVm; use crate::parser::Section; use promptforge_types::event::lifecycle; @@ -26,15 +26,15 @@ use super::{SectionContext, TaskSeed}; impl SectionContext { /// Constructs the frame for one walked section and runs its setup /// preamble: the `sys` JSON, the section-started observation, VM - /// construction and limits, the control surface (the `jump` and - /// `list_from_section` callbacks resolved over the section's visible - /// set, plus the coroutine yield shims for the suspending calls), and + /// construction and limits, the control surface (`jump`, and the + /// `prompt` reads resolved over the section's visible set, plus the + /// coroutine yield shims for the suspending calls), and /// the rest of the shared setup half (Engine injection, Engine globals, the /// Plugin preludes, the shared replay, the store yield shims). /// /// `siblings` is the caller's own walk slice, from which the section's /// visible set (its siblings minus itself, plus its direct children) is - /// built for the `list_from_section` callback. `section_id` is the + /// built for the `prompt` reads. `section_id` is the /// section's `sys.id`: the entering chain's hierarchical id extended /// by its local entry counter, allocated by the scheduler; `task_id` /// is the entering chain's `sys.taskid`. `var` is the walk's current @@ -85,11 +85,11 @@ impl SectionContext { ctx.limits().lua_memory().get(), ctx.limits().lua_logs().get(), )?; - // The `list_from_section` callback resolves over the section's - // visible set; the suspending calls (`call`, `fanout`, - // `models.infer`) are the yield shims the setup half installs. + // The `prompt` reads resolve over the section's visible set; the + // suspending calls (`call`, `fanout`, `models.infer`) are the + // yield shims the setup half installs. let visible = visible_sections(siblings, section); - let list_callback = move |heading: String| list_items_from_visible(&heading, &visible); + let read = move |heading: String| read_visible(&heading, &visible); // The setup half of the section lifecycle - Engine injection, Engine // globals, the control surface, the shared replay, and the captured // alias bindings - is shared with the H1 pass; only the seed, the @@ -105,7 +105,7 @@ impl SectionContext { ); // Setup runs on the bare VM so a failure tears it down here: the // frame does not exist yet, so its `Drop` cannot own this path. - if let Err(error) = setup_section_vm(&mut vm, &setup, list_callback) { + if let Err(error) = setup_section_vm(&mut vm, &setup, read) { vm.teardown(ctx.emitter(), section.name()); return Err(error); } @@ -134,9 +134,9 @@ impl SectionContext { /// /// H1's only deltas from a walked section: no `SECTION_STARTED` /// observation (the pass is not a walked section), an empty `var` seed - /// (it runs first and is never re-entered), and a `list_from_section` - /// visible set spanning the whole top-level slice - section 0 excludes - /// nothing and has no children. + /// (it runs first and is never re-entered), and a visible set behind + /// the `prompt` reads spanning the whole top-level slice - section 0 + /// excludes nothing and has no children. /// /// # Errors /// Returns the [`Error`](crate::Error) of whichever step failed. A VM @@ -169,7 +169,7 @@ impl SectionContext { // H1's visible set is the whole top-level slice: section 0 // excludes nothing and has no children. let visible = promptforge_parser::detail::sections(ctx.prompt()).to_vec(); - let list_callback = move |heading: String| list_items_from_visible(&heading, &visible); + let read = move |heading: String| read_visible(&heading, &visible); // H1's one privilege: `argv` installs writable, so the repair // pattern can assign it; the executor reads the value back at the // freeze (see the scheduler's H1-to-walk handoff). @@ -177,7 +177,7 @@ impl SectionContext { setup.argv_writable = true; // Setup runs on the bare VM so a failure tears it down here: the // frame does not exist yet, so its `Drop` cannot own this path. - if let Err(error) = setup_section_vm(&mut vm, &setup, list_callback) { + if let Err(error) = setup_section_vm(&mut vm, &setup, read) { vm.teardown(ctx.emitter(), title); return Err(error); } diff --git a/crates/promptforge-internal/engine/src/execute/section_vm.rs b/crates/promptforge-internal/engine/src/execute/section_vm.rs index 45f8daa3d..ce4928fc7 100644 --- a/crates/promptforge-internal/engine/src/execute/section_vm.rs +++ b/crates/promptforge-internal/engine/src/execute/section_vm.rs @@ -11,7 +11,8 @@ //! per-fanout `index`), picks the [`VmSeed`] (the walk's rolled-forward //! `var`; an arm //! adds its collection `item` to the caller's cloned `var`), and supplies -//! the `list_from_section` callback resolved over its own visible set. +//! the `read` callback behind the `prompt` reads, resolved over its own +//! visible set. //! //! VM construction and the Lua limits install stay with the driver. A //! construction failure's handling differs (the walk propagates, the arm @@ -26,7 +27,7 @@ use std::sync::Arc; use promptforge_types::emitter::Emitter; use promptforge_types::plugins::Prelude; -use crate::lua::{LuaProgram, SectionVm}; +use crate::lua::{LuaProgram, SectionText, SectionVm}; use crate::{Error, Result}; use promptforge_vfs::Access; @@ -97,8 +98,8 @@ pub(crate) struct SectionVmSetup<'a> { /// The sequence is fixed and shared: Engine injection with the driver's /// [`VmSeed`], [`SectionVm::install_engine_globals`], the `item` global when the /// seed includes one, the control surface -/// ([`SectionVm::install_scheduler_control_globals`] for `jump` and -/// `list_from_section`, plus [`SectionVm::install_coro_shims`] for the +/// ([`SectionVm::install_scheduler_control_globals`] for `jump` and the +/// `prompt` reads, plus [`SectionVm::install_coro_shims`] for the /// suspending calls, which the scheduler drives as yield shims), the run's /// Plugin preludes, and [`SectionVm::replay_shared`]. The caller applies /// the Lua limits itself before calling, so a limits failure propagates @@ -112,13 +113,13 @@ pub(crate) struct SectionVmSetup<'a> { /// global install, `item` install, control-global install, the prelude install (a /// prelude that fails to load, or whose global collides), the shared /// replay, or the store shims' install. -pub(crate) fn setup_section_vm( +pub(crate) fn setup_section_vm( vm: &mut SectionVm, setup: &SectionVmSetup<'_>, - list_callback: L, + read: R, ) -> Result<()> where - L: Fn(String) -> std::result::Result, Error> + Send + 'static, + R: Fn(String) -> std::result::Result, Error> + Send + Sync + 'static, { // The raw-model-id fallback reads its flag during Engine injection, so // the snapshot's opt-in lands first. @@ -138,7 +139,7 @@ where if let Some(item) = setup.seed.item { vm.set_global_json("item", item)?; } - vm.install_scheduler_control_globals(list_callback)?; + vm.install_scheduler_control_globals(read)?; vm.install_coro_shims(setup.max_tool_iterations)?; crate::lua::install_section_loop_shim(vm.lua())?; // The preludes read `tools` and `store` from `_G` as they install, so diff --git a/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs b/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs index a7d43cce2..f73e70db9 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs @@ -1,4 +1,4 @@ -//! Tests for section walk control flow: `call`, `jump`, `fanout`, `list_from_section`, and `var`. +//! Tests for section walk control flow: `call`, `jump`, `fanout`, the `prompt` reads, and `var`. use super::run; use super::*; @@ -125,20 +125,6 @@ Args: {{ args }}\n\n\ ); } -/// Two visible sections sharing one `(level, name)` address error loudly as -/// ambiguous rather than silently resolving to the first. -#[test] -fn list_from_section_ambiguous_error_is_loud() { - let visible = vec![ - synthetic_section("Dup", 3, Vec::new(), vec!["x".to_string()]), - synthetic_section("Dup", 3, Vec::new(), vec!["x".to_string()]), - ]; - let error = super::super::walk_target::list_items_from_visible("### Dup", &visible) - .expect_err("two visible sections with one address must be ambiguous"); - let rendered = error.to_string(); - assert!(rendered.contains("ambiguous"), "error was: {rendered}"); -} - /// Two top-level sections sharing one name error loudly as ambiguous rather /// than silently resolving to the first. #[test] diff --git a/crates/promptforge-internal/engine/src/execute/tests/live_infer.rs b/crates/promptforge-internal/engine/src/execute/tests/live_infer.rs index 6e43ac980..86d01a3ce 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/live_infer.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/live_infer.rs @@ -163,7 +163,7 @@ async fn tool_objects_and_roles_reach_section_call_and_fanout_vms() { ```lua\n\ local direct = binding_names()\n\ local called = call('## Called')\n\ - local arms = fanout('### Worker', list_from_section('### Items'))\n\ + local arms = fanout('### Worker', prompt.list('### Items'))\n\ return direct .. '|' .. called .. '|' .. table.concat(arms, ',')\n\ ```\n\n\ ### Worker\n\n\ diff --git a/crates/promptforge-internal/engine/src/execute/tests/model_and_reply-globals.rs b/crates/promptforge-internal/engine/src/execute/tests/model_and_reply-globals.rs index eb5075ed4..3dae953c1 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/model_and_reply-globals.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/model_and_reply-globals.rs @@ -83,7 +83,7 @@ async fn fanout_arm_sees_sys_model_catalog_id_after_the_scope_install() { let md = "---\nname: t\ndescription: d\npromptforge: 0\nplugins:\n - tools\nmodels:\n writer: {}\n---\n\n\ # Test prompt\n\n```lua shared\n\ models.default('writer')\n```\n\n\ -## Parent\n\n```lua\nlocal r = fanout('### Worker', list_from_section('### Items'))\nreturn table.concat(r, ',')\n```\n\n\ +## Parent\n\n```lua\nlocal r = fanout('### Worker', prompt.list('### Items'))\nreturn table.concat(r, ',')\n```\n\n\ ### Worker\n\n```lua\ntools.call('tools/echo', { value = item })\n```\n\n\ ```lua\nreturn sys.model .. ':' .. item\n```\n\n\ ### Items\n\n- a\n"; diff --git a/crates/promptforge-internal/engine/src/execute/tests/scheduler/live_h1/control_flow.rs b/crates/promptforge-internal/engine/src/execute/tests/scheduler/live_h1/control_flow.rs index 104f23b53..0e13286a7 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/scheduler/live_h1/control_flow.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/scheduler/live_h1/control_flow.rs @@ -1,7 +1,6 @@ //! Control flow out of the live H1 pass: `call` as a contained chain and //! the child ids it takes, `jump` to start the walk at a target, `fanout`, -//! the decision-tool idiom, and `list_from_section`, with the -//! unknown-section failures. +//! and the decision-tool idiom, with the unknown-section failures. use super::*; @@ -180,28 +179,3 @@ async fn the_h1_decision_tool_idiom_runs_before_the_walk() { "the loop runs the tool-call round and the terminal text round" ); } - -#[tokio::test(flavor = "current_thread")] -async fn list_from_section_works_on_the_h1() { - // `list_from_section` resolves over H1's visible set - the whole - // top-level slice. - let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ - # Test prompt\n\n\ - ```lua\n\ - local items = list_from_section('## Items')\n\ - var.answer = table.concat(items, ',')\n\ - ```\n\n\ - ## Result\n\n\ - ```lua\nreturn var.answer\n```\n\n\ - ## Items\n\n\ - - one\n\ - - two\n"; - let prompt = parse(md); - let (ctx, fixture) = h1_context(&prompt); - let out = TokioDriver::new(&ctx, fixture, None) - .drive() - .await - .expect("list_from_section from H1 reads the target's items"); - - assert_eq!(out, "one,two"); -} diff --git a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow.rs b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow.rs index 5fa1e00db..dab5a2dd7 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow.rs @@ -1,5 +1,5 @@ //! Section walk control flow over the offline fixtures: `call`, `jump`, -//! `fanout`, `list_from_section`, and `var`. The cases run on the suite's +//! `fanout`, the `prompt` reads, and `var`. The cases run on the suite's //! `run_fixture` runner through the thin fixture support below: the //! `run`/`fixture`/`TestStore`/`silent`/`run_offline` helpers. Each //! submodule holds one topic and reaches that support through `super`. @@ -98,6 +98,6 @@ mod fanout_arms; mod fanout_collections; mod globals; mod h1_control; -mod list_from_section; +mod prompt_reads; mod run_setup; mod store_failures; diff --git a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_arms.rs b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_arms.rs index 512f7192f..cfcc26975 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_arms.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_arms.rs @@ -1,4 +1,4 @@ -//! Control flow inside a fanout arm: `list_from_section`, `call`, nested +//! Control flow inside a fanout arm: `prompt.list`, `call`, nested //! `fanout`, and `jump` over the worker's visible set, the depth cap //! across the arm boundary, and an arm's text when it replies nothing. @@ -16,16 +16,16 @@ return r[1].text\n\ ```\n\n" ); -/// `list_from_section` inside a fanout arm reads a list section's items, -/// resolving over the worker's visible set (the set the worker was resolved -/// from, minus the worker, plus its children). +/// `prompt.list` inside a fanout arm reads a list section's items: the arm +/// resolves like any section (the worker's siblings minus the worker, plus +/// its children). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn list_from_section_inside_a_fanout_arm_reads_items() { +async fn prompt_list_inside_a_fanout_arm_reads_items() { let md = [ ARM_FANOUT_PARENT, "### Worker\n\n\ ```lua\n\ -local items = list_from_section('### Items')\n\ +local items = prompt.list('### Items')\n\ return item .. ':' .. table.concat(items, ',')\n\ ```\n\n\ ### Items\n\n\ @@ -35,7 +35,7 @@ return item .. ':' .. table.concat(items, ',')\n\ .concat(); let out = run_offline(&md) .await - .expect("list_from_section inside an arm must read items"); + .expect("prompt.list inside an arm must read items"); assert_eq!(out, "alpha:x,y"); } @@ -251,9 +251,9 @@ jump('## Parent')\n\ ); } -/// `call` and `list_from_section` inside an arm naming a section outside -/// the worker's visible set both error not-found; the arm catches them with -/// `pcall` and asserts on the messages. +/// `call` and `prompt.list` inside an arm naming a section outside the +/// worker's visible set both reject it: `call` errors not-found, which the +/// arm catches with `pcall` and asserts on, and `prompt.list` reads `nil`. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn call_and_list_inside_a_fanout_arm_reject_non_visible_sections() { let md = [ @@ -263,9 +263,7 @@ async fn call_and_list_inside_a_fanout_arm_reject_non_visible_sections() { local ok_call, err_call = pcall(call, '## Parent')\n\ assert(not ok_call, 'call of a non-visible section must fail')\n\ assert(tostring(err_call):find('not found', 1, true), 'call error must be not-found: ' .. tostring(err_call))\n\ -local ok_list, err_list = pcall(list_from_section, '## Parent')\n\ -assert(not ok_list, 'list_from_section of a non-visible section must fail')\n\ -assert(tostring(err_list):find('not found', 1, true), 'list error must be not-found: ' .. tostring(err_list))\n\ +assert(prompt.list('## Parent') == nil, 'prompt.list of a non-visible section must read nil')\n\ return item .. ':rejected'\n\ ```\n\n\ ### Items\n\n\ @@ -274,7 +272,7 @@ return item .. ':rejected'\n\ .concat(); let out = run_offline(&md) .await - .expect("non-visible call/list inside an arm must error not-found"); + .expect("a non-visible call must error not-found and a non-visible list must read nil"); assert_eq!(out, "alpha:rejected"); } diff --git a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_collections.rs b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_collections.rs index be3569542..7052764b0 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_collections.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_collections.rs @@ -10,7 +10,7 @@ async fn fanout_returns_structured_results() { "\ ## Parent\n\n\ ```lua\n\ -local r = fanout('### Worker', list_from_section('### Items'))\n\ +local r = fanout('### Worker', prompt.list('### Items'))\n\ assert(r[1].text == 'alpha-1')\n\ assert(r[1].ok == true)\n\ assert(r[1].item == 'alpha')\n\ @@ -43,7 +43,7 @@ async fn off_walk_worker_runs_as_a_fanout_arm() { "\ ## Parent\n\n\ ```lua\n\ -local r = fanout('### Worker', list_from_section('### Items'))\n\ +local r = fanout('### Worker', prompt.list('### Items'))\n\ return r[1].text .. ',' .. r[2].text\n\ ```\n\n\ ### Worker\n\n\ @@ -259,10 +259,9 @@ fanout('### Niece', {'x'})\n\ ); } -/// The two-string form errors at the boundary, pointing at -/// `list_from_section`. +/// The two-string form errors at the boundary, pointing at `prompt.list`. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn fanout_string_second_parameter_errors_pointing_at_list_from_section() { +async fn fanout_string_second_parameter_errors_pointing_at_prompt_list() { let md = flow_prompt!( "\ ## Parent\n\n\ @@ -283,8 +282,8 @@ fanout('### Worker', '### Items')\n\ "error was: {rendered}" ); assert!( - rendered.contains("list_from_section"), - "the error must point at list_from_section: {rendered}" + rendered.contains("prompt.list(heading)"), + "the error must point at prompt.list: {rendered}" ); } diff --git a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/h1_control.rs b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/h1_control.rs index b48ed02f0..5a6174680 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/h1_control.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/h1_control.rs @@ -1,5 +1,5 @@ -//! The control globals from H1: `call`, `jump`, `fanout`, and -//! `list_from_section` naming an unknown section fail the run. +//! The control globals from H1: `call`, `jump`, and `fanout` naming an +//! unknown section fail the run. use super::*; @@ -66,24 +66,3 @@ fanout('## Nope', {'a'})\n\ "the resolution error names the missing section: {rendered}" ); } - -/// `list_from_section` from H1 resolves over the whole top-level slice; an -/// unknown target fails the run with the resolution error. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn list_from_section_from_h1_to_an_unknown_section_fails_the_run() { - let md = flow_prompt!( - "\ -# Test prompt\n\n\ -```lua\n\ -list_from_section('## Nope')\n\ -```\n" - ); - let error = run_offline(md) - .await - .expect_err("list_from_section from H1 to an unknown section must fail"); - let rendered = error.to_string(); - assert!( - rendered.contains("## Nope"), - "the resolution error names the missing section: {rendered}" - ); -} diff --git a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/list_from_section.rs b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/list_from_section.rs deleted file mode 100644 index f9ec140ee..000000000 --- a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/list_from_section.rs +++ /dev/null @@ -1,177 +0,0 @@ -//! `list_from_section`: the items it returns, the sections it resolves, -//! and the sections it refuses. - -use super::*; - -/// `list_from_section` returns a sibling list section's pre-parsed bullet -/// items as a Lua array of strings, addressed by heading string. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn list_from_section_returns_bullet_items() { - let md = flow_prompt!( - "\ -## Main\n\n\ -```lua\n\ -local items = list_from_section('## List')\n\ -assert(type(items) == 'table')\n\ -assert(#items == 2)\n\ -assert(items[1] == 'alpha')\n\ -assert(items[2] == 'beta')\n\ -return 'ok'\n\ -```\n\n\ -## List\n\n\ -- alpha\n\ -- beta\n" - ); - let out = run_offline(md) - .await - .expect("a sibling list section's bullet items must be returned"); - assert_eq!(out, "ok"); -} - -/// A numbered list section's items come back in order, markers stripped. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn list_from_section_returns_numbered_items() { - let md = flow_prompt!( - "\ -## Main\n\n\ -```lua\n\ -local items = list_from_section('## Nums')\n\ -assert(#items == 3)\n\ -assert(items[1] == 'one' and items[2] == 'two' and items[3] == 'three')\n\ -return 'ok'\n\ -```\n\n\ -## Nums\n\n\ -1. one\n\ -2. two\n\ -3. three\n" - ); - let out = run_offline(md) - .await - .expect("a numbered list section's items must be returned"); - assert_eq!(out, "ok"); -} - -/// The caller's direct children are in the visible set. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn list_from_section_resolves_a_direct_child() { - let md = flow_prompt!( - "\ -## Main\n\n\ -```lua\n\ -local items = list_from_section('### Sub')\n\ -assert(#items == 2 and items[1] == 'x' and items[2] == 'y')\n\ -return 'ok'\n\ -```\n\n\ -### Sub\n\n\ -- x\n\ -- y\n" - ); - let out = run_offline(md) - .await - .expect("a direct child list section must be visible"); - assert_eq!(out, "ok"); -} - -/// A sibling's child (niece), a child's child (grandchild), and the caller -/// itself are all outside the visible set: each resolves as not-found. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn list_from_section_hides_nieces_grandchildren_and_the_caller() { - let md = flow_prompt!( - "\ -## Main\n\n\ -```lua\n\ -local ok_niece = pcall(list_from_section, '### Niece')\n\ -assert(not ok_niece, 'a niece is not visible')\n\ -local ok_grand = pcall(list_from_section, '#### Grand')\n\ -assert(not ok_grand, 'a grandchild is not visible')\n\ -local ok_self = pcall(list_from_section, '## Main')\n\ -assert(not ok_self, 'the caller itself is not visible')\n\ -return 'ok'\n\ -```\n\n\ -### Kid\n\n\ -- kid-item\n\n\ -#### Grand\n\n\ -- grand-item\n\n\ -## Other\n\n\ -```lua\nlocal x = 1\n```\n\n\ -### Niece\n\n\ -- niece-item\n" - ); - let out = run_offline(md) - .await - .expect("nieces, grandchildren, and the caller must all be invisible"); - assert_eq!(out, "ok"); -} - -/// The not-found error lists exactly the visible sections - siblings plus -/// direct children - so the error channel cannot leak the rest of the -/// document's structure. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn list_from_section_not_found_lists_only_the_visible_sections() { - let md = flow_prompt!( - "\ -## Main\n\n\ -```lua\n\ -list_from_section('## Missing')\n\ -```\n\n\ -### Kid\n\n\ -- kid-item\n\n\ -#### Grand\n\n\ -- grand-item\n\n\ -## Sibling\n\n\ -```lua\nlocal x = 1\n```\n\n\ -### Niece\n\n\ -- niece-item\n" - ); - let error = run_offline(md) - .await - .expect_err("an unknown heading must fail"); - let rendered = error.to_string(); - assert!(rendered.contains("not found"), "error was: {rendered}"); - assert!( - rendered.contains("## Sibling"), - "a sibling is visible: {rendered}" - ); - assert!( - rendered.contains("### Kid"), - "a direct child is visible: {rendered}" - ); - let (_, available) = rendered - .split_once("available sections:") - .expect("the not-found error must list the visible sections"); - assert!( - !available.contains("## Main"), - "the caller is not listed: {rendered}" - ); - assert!( - !available.contains("Niece"), - "a niece is not listed: {rendered}" - ); - assert!( - !available.contains("Grand"), - "a grandchild is not listed: {rendered}" - ); -} - -/// Naming a prose section (no pre-parsed items) is the mistake the no-items -/// error exists to catch. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn list_from_section_rejects_a_prose_section() { - let md = flow_prompt!( - "\ -## Main\n\n\ -```lua\n\ -list_from_section('## Prose')\n\ -```\n\n\ -## Prose\n\n\ -Just prose, no list items here.\n" - ); - let error = run_offline(md) - .await - .expect_err("a prose section has no pre-parsed items"); - let rendered = error.to_string(); - assert!( - rendered.contains("section `Prose` has no pre-parsed items"), - "error was: {rendered}" - ); -} diff --git a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/prompt_reads.rs b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/prompt_reads.rs new file mode 100644 index 000000000..b45bff361 --- /dev/null +++ b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/prompt_reads.rs @@ -0,0 +1,171 @@ +//! `prompt.section` and `prompt.list`: what each returns over the caller's +//! visible set, the only calls that raise, and the reads from H1 and the +//! shared library. + +use super::*; + +/// The raw Markdown `## Rules` holds: an indented first line, inline markup, +/// a `lua` fence, a thematic break, and a template, all of which +/// `prompt.section` returns verbatim, neither run nor rendered. The break +/// resets the section's prose and list capture but not its Markdown. +const RULES: &str = " Indented first line.\n\n\ +- **Bold** and `code` stay raw.\n\n\ +```lua\n\ +var.never = 'run'\n\ +```\n\n\ +---\n\n\ +Closing line with {{ args }}."; + +/// The sections every test joins after its own `## Main`: a child list +/// `### Kid` with a grandchild `#### Grand`, a numbered `## Nums`, the raw +/// `## Rules`, an empty `## Empty`, and `## Other` with a niece `### Niece`. +const DOCUMENT: [&str; 3] = [ + "### Kid\n\n\ +- kid-one\n\ +- kid-two\n\n\ +#### Grand\n\n\ +- grand-item\n\n\ +## Nums\n\n\ +1. one\n\ +2. two\n\ +3. three\n\n\ +## Rules\n\n", + RULES, + "\n\n\ +## Empty\n\n\ +## Other\n\n\ +Other prose.\n\n\ +### Niece\n\n\ +- niece-item\n", +]; + +/// Both reads search only the caller's visible set (its siblings minus +/// itself, plus its direct children): `prompt.section` returns a visible +/// section's own raw Markdown, `prompt.list` its pre-parsed items in order, +/// and a niece, a grandchild, the caller itself, and a heading no section +/// has all read as `nil` from both. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prompt_reads_resolve_over_the_visible_set() { + let md = [ + flow_prompt!( + "\ +## Main\n\n\ +```lua\n\ +assert(prompt.section('### Kid') == '- kid-one\\n- kid-two', 'a child reads its own Markdown, without its children')\n\ +local kids = prompt.list('### Kid')\n\ +assert(#kids == 2 and kids[1] == 'kid-one' and kids[2] == 'kid-two', 'a child list reads its items')\n\ +local nums = prompt.list('## Nums')\n\ +assert(#nums == 3 and nums[1] == 'one' and nums[2] == 'two' and nums[3] == 'three', 'a sibling list reads its items in order')\n\ +assert(prompt.section('## Nums') == '1. one\\n2. two\\n3. three', 'a list section reads its raw Markdown too')\n\ +assert(prompt.section('## Empty') == '', 'an empty section reads as the empty string')\n\ +for _, heading in ipairs({'### Niece', '#### Grand', '## Main', '## Missing'}) do\n\ + assert(prompt.section(heading) == nil, heading .. ' must read nil from prompt.section')\n\ + assert(prompt.list(heading) == nil, heading .. ' must read nil from prompt.list')\n\ +end\n\ +return prompt.section('## Rules')\n\ +```\n\n" + ), + DOCUMENT.concat().as_str(), + ] + .concat(); + let out = run_offline(&md) + .await + .expect("both reads must resolve over the visible set"); + assert_eq!(out, RULES, "a section's Markdown comes back verbatim"); +} + +/// A read raises only when the call itself is wrong: a target that is not a +/// string, a malformed heading, or `prompt.list` naming a section with no +/// pre-parsed items, whose message names the trimmed heading. A heading no +/// section has and a heading outside the visible set are not mistakes, so +/// both read as `nil`. A `call` to a heading no section has still raises, +/// and its listing of the available sections names only the visible set. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prompt_reads_raise_only_for_mistakes_in_the_call() { + let md = [ + flow_prompt!( + "\ +## Main\n\n\ +```lua\n\ +local function expect_raise(read, target, needle)\n\ + local ok, err = pcall(read, target)\n\ + assert(not ok, 'expected a raise for ' .. tostring(target))\n\ + assert(tostring(err):find(needle, 1, true), tostring(target) .. ' must raise with ' .. needle .. ', got: ' .. tostring(err))\n\ +end\n\ +for _, read in ipairs({prompt.section, prompt.list}) do\n\ + expect_raise(read, 42, 'section target must be a string, got integer')\n\ + expect_raise(read, 'Nums', 'got bare name: Nums')\n\ + expect_raise(read, '##Nums', 'must have whitespace after the ## markers')\n\ + for _, heading in ipairs({'## Missing', '### Niece'}) do\n\ + local ok, value = pcall(read, heading)\n\ + assert(ok and value == nil, heading .. ' must read as nil, not raise: ' .. tostring(value))\n\ + end\n\ +end\n\ +expect_raise(prompt.list, ' ## Rules ', 'section `## Rules` has no pre-parsed items')\n\ +expect_raise(prompt.list, '## Empty', 'section `## Empty` has no pre-parsed items')\n\ +assert(prompt.section('## Rules') ~= nil, 'prompt.section reads a section prompt.list refuses')\n\ +local ok, err = pcall(call, '## Missing')\n\ +assert(not ok, 'a call to a missing heading still raises')\n\ +local message = tostring(err)\n\ +local at = message:find('available sections:', 1, true)\n\ +assert(at, 'the call error lists the available sections: ' .. message)\n\ +local listed = message:sub(at)\n\ +for _, heading in ipairs({'## Other', '### Kid'}) do\n\ + assert(listed:find(heading, 1, true), 'the listing names the visible ' .. heading .. ': ' .. message)\n\ +end\n\ +for _, hidden in ipairs({'## Main', 'Niece', 'Grand'}) do\n\ + assert(not listed:find(hidden, 1, true), 'the listing leaks ' .. hidden .. ': ' .. message)\n\ +end\n\ +return 'ok'\n\ +```\n\n" + ), + DOCUMENT.concat().as_str(), + ] + .concat(); + let out = run_offline(&md) + .await + .expect("every raise must be caught and every miss must read nil"); + assert_eq!(out, "ok"); +} + +/// Both reads work in H1, whose visible set is the whole top-level slice +/// (every top-level section and no child), and in the shared library: at +/// load time, where each VM's replay resolves over that VM's own visible +/// set, and from a shared function, which resolves over the visible set of +/// whichever section calls it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prompt_reads_work_in_h1_and_the_shared_library() { + let md = [ + flow_prompt!( + "\ +# Test prompt\n\n\ +```lua shared\n\ +kids_at_load = prompt.list('### Kid')\n\ +function kid_items()\n\ + return prompt.list('### Kid')\n\ +end\n\ +```\n\n\ +```lua\n\ +assert(prompt.section('## Main') ~= nil, 'H1 sees every top-level section')\n\ +assert(prompt.section('### Kid') == nil, 'H1 sees no child')\n\ +assert(kids_at_load == nil, 'the load-time read in the H1 replay resolves over the H1 visible set')\n\ +assert(kid_items() == nil, 'a shared function reads over the H1 visible set')\n\ +var.nums = table.concat(prompt.list('## Nums'), ',')\n\ +```\n\n\ +## Main\n\n\ +```lua\n\ +assert(kids_at_load ~= nil, 'the load-time read in the Main replay resolves over the Main visible set')\n\ +return var.nums .. '|' .. table.concat(kids_at_load, ',') .. '|' .. table.concat(kid_items(), ',')\n\ +```\n\n" + ), + DOCUMENT.concat().as_str(), + ] + .concat(); + let out = run_offline(&md) + .await + .expect("both reads must work in H1 and the shared library"); + assert_eq!( + out, "one,two,three|kid-one,kid-two|kid-one,kid-two", + "H1 reads the top-level list, and both the load-time read and the shared function read the caller's child" + ); +} diff --git a/crates/promptforge-internal/engine/src/execute/walk_target.rs b/crates/promptforge-internal/engine/src/execute/walk_target.rs index e3ac57418..7c7ca169f 100644 --- a/crates/promptforge-internal/engine/src/execute/walk_target.rs +++ b/crates/promptforge-internal/engine/src/execute/walk_target.rs @@ -6,11 +6,12 @@ //! nieces/nephews, and grandchildren are never in the set, so a resolution //! error that lists the set cannot leak the rest of the document's //! structure. The scheduler's chains resolve jumps, `call` targets, and -//! fanout workers through these helpers, and the `list_from_section` -//! callback resolves through them too, so every control surface agrees on +//! fanout workers through these helpers, and the `prompt.section` and +//! `prompt.list` reads resolve through them too, so every surface agrees on //! what a heading may name. use crate::heading_address; +use crate::lua::SectionText; use crate::parser::Section; use crate::{Error, Result}; @@ -24,11 +25,6 @@ pub(super) fn section_position(slice: &[Section], target: &Section) -> Option Vec
{ let caller_index = section_position(home, caller); home.iter() @@ -47,24 +43,20 @@ pub(super) fn visible_sections(home: &[Section], caller: &Section) -> Vec Result> { - let section = heading_address::resolve_sibling(heading, visible)?; - if section.items().is_empty() { - return Err(Error::Lua(format!( - "section `{}` has no pre-parsed items", - section.name() - ))); - } - Ok(section.items().to_vec()) +/// Returns [`Error::Lua`] when the heading is malformed or matches more +/// than one visible section (see [`heading_address::find_visible`]). +pub(super) fn read_visible(heading: &str, visible: &[Section]) -> Result> { + Ok( + heading_address::find_visible(heading, visible)?.map(|section| SectionText { + markdown: section.markdown().to_owned(), + items: section.items().to_vec(), + }), + ) } /// Where a jump transfers control, resolved against the jumper's visible set. diff --git a/crates/promptforge-internal/engine/src/heading_address.rs b/crates/promptforge-internal/engine/src/heading_address.rs index 0f3c61dca..74a4462a5 100644 --- a/crates/promptforge-internal/engine/src/heading_address.rs +++ b/crates/promptforge-internal/engine/src/heading_address.rs @@ -2,14 +2,14 @@ //! surface resolves through. //! //! A section's Lua names other sections by heading string - `call`, `jump`, -//! `list_from_section`, `tasks.spawn`, and `fanout(worker, collection)` -//! (which runs the worker template once per collection member: the array -//! part in order, then the hash part as `{ key, value }` pairs sorted by -//! key, a list section's pre-parsed items feeding in through -//! `list_from_section`). The `fanout` shim itself is Lua over the task -//! protocol (`promptforge-lua`'s `__impl_fanout.lua`): it spawns one task -//! per member - every arm up front, before any join, so no arm is ordered -//! after a sibling - and waits on the live set while the scheduler's +//! `prompt.section`, `prompt.list`, `tasks.spawn`, and +//! `fanout(worker, collection)` (which runs the worker template once per +//! collection member: the array part in order, then the hash part as +//! `{ key, value }` pairs sorted by key, a list section's pre-parsed items +//! feeding in through `prompt.list`). The `fanout` shim itself is Lua over +//! the task protocol (`promptforge-lua`'s `__impl_fanout.lua`): it spawns +//! one task per member - every arm up front, before any join, so no arm is +//! ordered after a sibling - and waits on the live set while the scheduler's //! admission limit bounds how many arms run at once. This module defines //! [`find_visible`], the one heading lookup those surfaces share, and //! [`resolve_sibling`], its adapter for surfaces where a miss is an error; diff --git a/crates/promptforge-internal/engine/src/lua.rs b/crates/promptforge-internal/engine/src/lua.rs index 7b920f2d8..626fca61a 100644 --- a/crates/promptforge-internal/engine/src/lua.rs +++ b/crates/promptforge-internal/engine/src/lua.rs @@ -16,8 +16,8 @@ // model-facing message renderer a store failure carries. pub(crate) use promptforge_lua::{ Argv, CoroStep, LuaBlockResult, LuaProgram, MessageList, ModelReport, OverflowReason, - ProseState, ScriptReport, SectionVm, TaskAllowlist, ToolBinding, ToolCallCounts, ToolSet, - ToolView, UsageAnchor, current_tool_bindings, enrich_sys_model, install_preludes, + ProseState, ScriptReport, SectionText, SectionVm, TaskAllowlist, ToolBinding, ToolCallCounts, + ToolSet, ToolView, UsageAnchor, current_tool_bindings, enrich_sys_model, install_preludes, install_section_loop_shim, install_store_shims, install_ui, output_reserve, precheck, prepare_dispatch, prepare_model_dispatch, project_messages, render_item, resolve_model_binding, }; diff --git a/crates/promptforge-internal/engine/src/lua/tests.rs b/crates/promptforge-internal/engine/src/lua/tests.rs index 621548a08..eca070dbe 100644 --- a/crates/promptforge-internal/engine/src/lua/tests.rs +++ b/crates/promptforge-internal/engine/src/lua/tests.rs @@ -30,7 +30,7 @@ use promptforge_lua::Error; use crate::execute::protocol::{Answer, Request, YieldParse}; use crate::execute::section_vm::{SectionVmSetup, VmSeed, setup_section_vm}; -use crate::lua::{LuaProgram, SectionVm, ToolBinding, ToolSet}; +use crate::lua::{LuaProgram, SectionText, SectionVm, ToolBinding, ToolSet}; use crate::model::{ModelBinding, ModelSet}; use crate::test_support::recording::null_emitter; use crate::tools::ToolId; @@ -121,9 +121,8 @@ fn scheduler_vm_with_tools( preludes: &[], raw_shims: false, }; - let list_callback = - |_: String| -> std::result::Result, crate::Error> { Ok(Vec::new()) }; - setup_section_vm(&mut vm, &setup, list_callback).expect("the setup installs"); + let read = |_: String| -> std::result::Result, crate::Error> { Ok(None) }; + setup_section_vm(&mut vm, &setup, read).expect("the setup installs"); vm } diff --git a/crates/promptforge-internal/engine/src/lua/tests/globals.rs b/crates/promptforge-internal/engine/src/lua/tests/globals.rs index 535a3785a..c3f37d2e0 100644 --- a/crates/promptforge-internal/engine/src/lua/tests/globals.rs +++ b/crates/promptforge-internal/engine/src/lua/tests/globals.rs @@ -21,7 +21,7 @@ use promptforge_types::plugins::PluginId; use promptforge_types::tools::{ToolDescriptor, ToolId}; use crate::execute::section_vm::{SectionVmSetup, VmSeed, setup_section_vm}; -use crate::lua::{LuaProgram, SectionVm, ToolBinding, ToolSet}; +use crate::lua::{LuaProgram, SectionText, SectionVm, ToolBinding, ToolSet}; use crate::model::{ModelBinding, ModelSet}; use crate::test_support::recording::null_emitter; use crate::untrusted::GuardNonce; @@ -123,9 +123,8 @@ fn fully_bound_vm(argv_writable: bool) -> SectionVm { preludes: &[], raw_shims: false, }; - let list_callback = - |_: String| -> std::result::Result, crate::Error> { Ok(Vec::new()) }; - setup_section_vm(&mut vm, &setup, list_callback).expect("the setup installs"); + let read = |_: String| -> std::result::Result, crate::Error> { Ok(None) }; + setup_section_vm(&mut vm, &setup, read).expect("the setup installs"); vm.install_lazy_prose(|_| Ok("the block's prose".to_owned())) .expect("the prose guard installs"); vm diff --git a/crates/promptforge-internal/engine/tests/prompts/execution/fanout-arm-failure.md b/crates/promptforge-internal/engine/tests/prompts/execution/fanout-arm-failure.md index fdb434257..b4e2f0af8 100644 --- a/crates/promptforge-internal/engine/tests/prompts/execution/fanout-arm-failure.md +++ b/crates/promptforge-internal/engine/tests/prompts/execution/fanout-arm-failure.md @@ -9,7 +9,7 @@ promptforge: 0 ## Research ```lua -local replies = fanout("### Worker", list_from_section("### Topics")) +local replies = fanout("### Worker", prompt.list("### Topics")) return table.concat(replies, "\n") ``` diff --git a/crates/promptforge-internal/engine/tests/prompts/execution/fanout-basic.md b/crates/promptforge-internal/engine/tests/prompts/execution/fanout-basic.md index 2fcb15470..dd8e122a4 100644 --- a/crates/promptforge-internal/engine/tests/prompts/execution/fanout-basic.md +++ b/crates/promptforge-internal/engine/tests/prompts/execution/fanout-basic.md @@ -9,7 +9,7 @@ promptforge: 0 ## Research ```lua -local replies = fanout("### Worker", list_from_section("### Topics")) +local replies = fanout("### Worker", prompt.list("### Topics")) return table.concat(replies, "\n") ``` diff --git a/crates/promptforge-internal/engine/tests/prompts/execution/fanout-epilog.md b/crates/promptforge-internal/engine/tests/prompts/execution/fanout-epilog.md index 1053a631a..9aae9e71d 100644 --- a/crates/promptforge-internal/engine/tests/prompts/execution/fanout-epilog.md +++ b/crates/promptforge-internal/engine/tests/prompts/execution/fanout-epilog.md @@ -12,7 +12,7 @@ promptforge: 0 ``` ```lua -local replies = fanout("### Worker", list_from_section("### Items")) +local replies = fanout("### Worker", prompt.list("### Items")) return table.concat(replies, ",") ``` diff --git a/crates/promptforge-internal/engine/tests/prompts/execution/fanout-store-writes.md b/crates/promptforge-internal/engine/tests/prompts/execution/fanout-store-writes.md index bd118a9d2..c8eb6967c 100644 --- a/crates/promptforge-internal/engine/tests/prompts/execution/fanout-store-writes.md +++ b/crates/promptforge-internal/engine/tests/prompts/execution/fanout-store-writes.md @@ -9,7 +9,7 @@ promptforge: 0 ## Research ```lua -local replies = fanout("### Worker", list_from_section("### Topics")) +local replies = fanout("### Worker", prompt.list("### Topics")) local files = store.glob("arm-*.md") -- The ordered merge: the join delivers arm results in collection order, -- never finish order, so the parent's merge is deterministic by diff --git a/crates/promptforge-internal/lua/src/collection.rs b/crates/promptforge-internal/lua/src/collection.rs index bccb9c152..8a84fdc6a 100644 --- a/crates/promptforge-internal/lua/src/collection.rs +++ b/crates/promptforge-internal/lua/src/collection.rs @@ -25,13 +25,13 @@ pub(crate) use order::{KeyError, SortKey, sort_key}; /// /// # Errors /// Returns [`Error::Lua`] when the value is not a table (the message points -/// at `list_from_section` for the list-section case), when a member is a +/// at `prompt.list` for the list-section case), when a member is a /// function, userdata, or thread (the error names the member's index), or /// when a hash key is not a string, number, or boolean. pub(crate) fn collection_members(lua: &Lua, collection: &Value) -> Result { let Value::Table(table) = collection else { return Err(Error::Lua( - "fanout's second parameter is a collection; for a list section use list_from_section(heading)".to_owned(), + "fanout's second parameter is a collection; for a list section use prompt.list(heading), which returns nil when no visible section has that heading".to_owned(), )); }; let members = lua.create_table().map_err(Error::lua)?; @@ -145,8 +145,8 @@ mod tests { let error = collection_members(&lua, &value).expect_err("a non-table is not a collection"); assert!( - error.to_string().contains("list_from_section"), - "the error must point at list_from_section for {source}: {error}" + error.to_string().contains("prompt.list(heading)"), + "the error must point at prompt.list for {source}: {error}" ); } } diff --git a/crates/promptforge-internal/lua/src/globals-tests.rs b/crates/promptforge-internal/lua/src/globals-tests.rs index 9fe5448cb..759c669eb 100644 --- a/crates/promptforge-internal/lua/src/globals-tests.rs +++ b/crates/promptforge-internal/lua/src/globals-tests.rs @@ -15,7 +15,7 @@ use serde_json::json; use super::{GLOBALS_CHUNK_NAME, GLOBALS_SOURCE, RESERVED_NAMES, Reserved, reserved_name}; use crate::tests::recording::null_emitter; use crate::tests::{assert_chunk_name_resolves, refusal_line}; -use crate::{Argv, SectionVm}; +use crate::{Argv, SectionText, SectionVm}; const SECTION: &str = "Globals"; @@ -51,7 +51,7 @@ fn section_vm(argv: Argv<'_>) -> SectionVm { vm.install_engine_globals(&emitter, SECTION) .expect("the Engine globals install"); vm.install_scheduler_control_globals(|_| { - Ok::, std::convert::Infallible>(Vec::new()) + Ok::, std::convert::Infallible>(None) }) .expect("the control globals install"); vm.install_coro_shims(1) diff --git a/crates/promptforge-internal/lua/src/globals.rs b/crates/promptforge-internal/lua/src/globals.rs index cff5d5e0f..e19510047 100644 --- a/crates/promptforge-internal/lua/src/globals.rs +++ b/crates/promptforge-internal/lua/src/globals.rs @@ -161,11 +161,11 @@ pub const RESERVED_NAMES: [(&str, Reserved); 61] = [ ("fanout", Reserved::EngineGlobal), ("item", Reserved::EngineGlobal), ("jump", Reserved::EngineGlobal), - ("list_from_section", Reserved::EngineGlobal), ("log", Reserved::EngineGlobal), ("messages", Reserved::EngineGlobal), ("models", Reserved::EngineGlobal), ("plugins", Reserved::EngineGlobal), + ("prompt", Reserved::EngineGlobal), ("prose", Reserved::EngineGlobal), ("store", Reserved::EngineGlobal), ("sys", Reserved::EngineGlobal), diff --git a/crates/promptforge-internal/lua/src/handles.rs b/crates/promptforge-internal/lua/src/handles.rs index 2fc8a25c0..4fe96632e 100644 --- a/crates/promptforge-internal/lua/src/handles.rs +++ b/crates/promptforge-internal/lua/src/handles.rs @@ -1,4 +1,4 @@ -//! Tool bindings, the shared tool set, and the per-binding output kind that shape how bound tools reach Lua. +//! Tool bindings, the shared tool set, and the per-binding output kind that shape how bound tools reach Lua, and the section text a prompt read sees. use promptforge_types::plugins::PluginId; use promptforge_types::tools::ToolDescriptor; @@ -139,6 +139,15 @@ pub(crate) fn resolve_section_target(value: Value) -> mlua::Result { } } +/// What a prompt read sees of one visible section. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SectionText { + /// The section's own raw Markdown. + pub markdown: String, + /// Its pre-parsed list items; empty for a section that isn't a list. + pub items: Vec, +} + /// The run's tool set: the declared plugins, the prompt-wide offers, and /// every catalog tool the run can offer. #[derive(Debug, Clone, Default, PartialEq, Eq)] diff --git a/crates/promptforge-internal/lua/src/lib.rs b/crates/promptforge-internal/lua/src/lib.rs index 4fd1cb4a2..9f2a87a5e 100644 --- a/crates/promptforge-internal/lua/src/lib.rs +++ b/crates/promptforge-internal/lua/src/lib.rs @@ -164,7 +164,7 @@ pub use dispatch::{ }; pub use engine_globals::{run_store_op, store_error_message}; pub use globals::{RESERVED_NAMES, Reserved, reserved_name}; -pub use handles::{LuaBlockResult, ToolBinding, ToolOutputKind, ToolSet, ToolView}; +pub use handles::{LuaBlockResult, SectionText, ToolBinding, ToolOutputKind, ToolSet, ToolView}; pub use messages::MessageList; pub use models::ModelRuntime; pub use prelude::install_preludes; diff --git a/crates/promptforge-internal/lua/src/prelude-tests.rs b/crates/promptforge-internal/lua/src/prelude-tests.rs index 2d3f705d6..efe8c345b 100644 --- a/crates/promptforge-internal/lua/src/prelude-tests.rs +++ b/crates/promptforge-internal/lua/src/prelude-tests.rs @@ -13,8 +13,8 @@ use super::install_preludes; use crate::tests::recording::null_emitter; use crate::tests::refusal_line; use crate::{ - Argv, CoroStep, LuaProgram, Request, SectionVm, YieldParse, install_section_loop_shim, - install_ui, + Argv, CoroStep, LuaProgram, Request, SectionText, SectionVm, YieldParse, + install_section_loop_shim, install_ui, }; #[path = "prelude-tests-environment.rs"] @@ -50,7 +50,7 @@ fn section_vm_with_var(var: Option<&serde_json::Value>) -> SectionVm { vm.install_engine_globals(&emitter, SECTION) .expect("the Engine globals install"); vm.install_scheduler_control_globals(|_| { - Ok::, std::convert::Infallible>(Vec::new()) + Ok::, std::convert::Infallible>(None) }) .expect("the control globals install"); vm.install_coro_shims(1) diff --git a/crates/promptforge-internal/lua/src/tests.rs b/crates/promptforge-internal/lua/src/tests.rs index 2220dbac5..536339f87 100644 --- a/crates/promptforge-internal/lua/src/tests.rs +++ b/crates/promptforge-internal/lua/src/tests.rs @@ -292,7 +292,7 @@ fn shim_vm(cancel: Option) -> SectionVm vm.install_engine_globals(&emitter, "Loop") .expect("Engine globals must install"); vm.install_scheduler_control_globals(|_| { - Ok::, std::convert::Infallible>(Vec::new()) + Ok::, std::convert::Infallible>(None) }) .expect("the control globals must install"); vm.install_coro_shims(1).expect("coro shims must install"); diff --git a/crates/promptforge-internal/lua/src/tests/shared_replay.rs b/crates/promptforge-internal/lua/src/tests/shared_replay.rs index ecb5a3707..acddb94de 100644 --- a/crates/promptforge-internal/lua/src/tests/shared_replay.rs +++ b/crates/promptforge-internal/lua/src/tests/shared_replay.rs @@ -17,11 +17,7 @@ fn jump_during_shared_replay_is_a_hard_error() { .expect("Engine globals must install"); vm.install_control_globals( |_, _, _| Err(Error::Lua("call is not needed here".to_owned())), - |_| { - Err(Error::Lua( - "list_from_section is not needed here".to_owned(), - )) - }, + |_| Err(Error::Lua("prompt reads are not needed here".to_owned())), ) .expect("control globals must install"); let error = vm @@ -49,11 +45,7 @@ fn call_with_a_non_string_target_errors() { .expect("Engine globals must install"); vm.install_control_globals( |target, _, _| resolve_section_target(target).map_err(Error::lua), - |_| { - Err(Error::Lua( - "list_from_section is not needed here".to_owned(), - )) - }, + |_| Err(Error::Lua("prompt reads are not needed here".to_owned())), ) .expect("control globals must install"); let out = run_scalar( diff --git a/crates/promptforge-internal/lua/src/vm/install.rs b/crates/promptforge-internal/lua/src/vm/install.rs index c1ab3a6c7..ba9422568 100644 --- a/crates/promptforge-internal/lua/src/vm/install.rs +++ b/crates/promptforge-internal/lua/src/vm/install.rs @@ -5,8 +5,8 @@ use super::{SectionVm, pack_sequence}; #[cfg(test)] use crate::var_to_json; use crate::{ - Access, Arc, Argv, Emitter, Error, Json, Mutex, Ordering, ProseState, Result, Value, - guarded_var, install_compactors, install_log, install_messages, install_models, + Access, Arc, Argv, Emitter, Error, Json, Mutex, Ordering, ProseState, Result, SectionText, + Value, guarded_var, install_compactors, install_log, install_messages, install_models, install_plugins, install_shim_prelude, install_store_table, install_tools, resolve_section_target, seal_sys, }; @@ -151,8 +151,9 @@ impl SectionVm { .and_then(|mut slot| slot.take()) } - /// Installs `call`, `jump`, and `list_from_section` as persistent - /// globals for the section's whole lifecycle. + /// Installs `call`, `jump`, and the `prompt` reads (`prompt.section` + /// and `prompt.list`) as persistent globals for the section's whole + /// lifecycle. /// /// Called once by `promptforge-engine` after Engine injection. The /// callbacks own their run context, so the closures stay valid across @@ -167,14 +168,10 @@ impl SectionVm { /// # Errors /// Returns [`Error::Lua`] if any global cannot be installed. #[cfg(test)] - pub(crate) fn install_control_globals( - &self, - call_callback: E, - list_callback: L, - ) -> Result<()> + pub(crate) fn install_control_globals(&self, call_callback: E, read: R) -> Result<()> where E: Fn(Value, Option, Json) -> std::result::Result + Send + 'static, - L: Fn(String) -> std::result::Result, Error> + Send + 'static, + R: Fn(String) -> std::result::Result, Error> + Send + Sync + 'static, { let globals = self.lua.globals(); let call_fn = self @@ -186,11 +183,11 @@ impl SectionVm { .map_err(Error::lua)?; globals.raw_set("call", call_fn).map_err(Error::lua)?; self.install_jump_global(&globals)?; - self.install_list_global(&globals, list_callback) + self.install_prompt_table(&globals, read) } - /// Installs the scheduler-mode control surface: `jump` and - /// `list_from_section` as Rust callbacks (neither suspends). + /// Installs the scheduler-mode control surface: `jump` and the + /// `prompt` reads as Rust callbacks (none suspends). /// /// The suspending calls (`models.infer`, `call`, `fanout`, /// `tools.call`, `tasks.*`) are the yield shims installed by @@ -198,14 +195,14 @@ impl SectionVm { /// /// # Errors /// Returns [`Error::Lua`] if any global cannot be installed. - pub fn install_scheduler_control_globals(&self, list_callback: L) -> Result<()> + pub fn install_scheduler_control_globals(&self, read: R) -> Result<()> where - L: Fn(String) -> std::result::Result, E> + Send + 'static, + R: Fn(String) -> std::result::Result, E> + Send + Sync + 'static, E: std::error::Error + Send + Sync + 'static, { let globals = self.lua.globals(); self.install_jump_global(&globals)?; - self.install_list_global(&globals, list_callback) + self.install_prompt_table(&globals, read) } /// Installs the coroutine yield shims (`models.infer`, `call`, @@ -248,22 +245,39 @@ impl SectionVm { globals.raw_set("jump", jump_fn).map_err(Error::lua) } - fn install_list_global(&self, globals: &mlua::Table, list_callback: L) -> Result<()> + fn install_prompt_table(&self, globals: &mlua::Table, read: R) -> Result<()> where - L: Fn(String) -> std::result::Result, E> + Send + 'static, + R: Fn(String) -> std::result::Result, E> + Send + Sync + 'static, E: std::error::Error + Send + Sync + 'static, { - let list_fn = self + let read = Arc::new(read); + let read_markdown = Arc::clone(&read); + let section = self + .lua + .create_function(move |_, target: Value| { + let text = read_markdown(resolve_section_target(target)?) + .map_err(mlua::Error::external)?; + Ok(text.map(|text| text.markdown)) + }) + .map_err(Error::lua)?; + let list = self .lua .create_function(move |lua, target: Value| { let heading = resolve_section_target(target)?; - let items = list_callback(heading).map_err(mlua::Error::external)?; - pack_sequence(lua, items) + match read(heading.clone()).map_err(mlua::Error::external)? { + None => Ok(Value::Nil), + Some(text) if text.items.is_empty() => Err(mlua::Error::external(format!( + "section `{}` has no pre-parsed items", + heading.trim() + ))), + Some(text) => pack_sequence(lua, text.items).map(Value::Table), + } }) .map_err(Error::lua)?; - globals - .raw_set("list_from_section", list_fn) - .map_err(Error::lua) + let prompt = self.lua.create_table().map_err(Error::lua)?; + prompt.raw_set("section", section).map_err(Error::lua)?; + prompt.raw_set("list", list).map_err(Error::lua)?; + globals.raw_set("prompt", prompt).map_err(Error::lua) } /// Replaces the sealed Lua `sys` global after scope close. diff --git a/vibe/2026-10-10-2-prompt-namespace-reads.md b/vibe/2026-10-10-2-prompt-namespace-reads.md index 89104271e..00c333b07 100644 --- a/vibe/2026-10-10-2-prompt-namespace-reads.md +++ b/vibe/2026-10-10-2-prompt-namespace-reads.md @@ -432,7 +432,7 @@ The test suite gets smaller. One parser test pins the span, one unit test pins t -### Step 3: Replace list_from_section with prompt.section and prompt.list +### Step 3: Replace list_from_section with prompt.section and prompt.list [completed] - Component: prompt-namespace - Component placement: last. It needs `Section::markdown()` from Step 1 and `find_visible` from Step 2, and it is the only component prompt authors see. @@ -452,7 +452,10 @@ The test suite gets smaller. One parser test pins the span, one unit test pins t - every remaining `list_from_section` call, test function name, and doc comment under `engine/src/execute/tests/`: `live_infer.rs`, `model_and_reply-globals.rs`, `suite/exec_flow/fanout_collections.rs`, and `suite/exec_flow/fanout_arms.rs`, where `list_from_section_inside_a_fanout_arm_reads_items` becomes `prompt_list_inside_a_fanout_arm_reads_items` and its doc comment says the arm resolves like any section (the worker's siblings minus the worker, plus its children); module docs in `exec_flow.rs`, `suite/exec_flow.rs`, `suite/exec_flow/h1_control.rs`, `suite/exec_flow/fanout_arms.rs`, and `scheduler/live_h1/control_flow.rs` - the four fixtures under `engine/tests/prompts/execution/`: `fanout-basic.md`, `fanout-epilog.md`, `fanout-store-writes.md`, and `fanout-arm-failure.md` - The tests, under `crates/promptforge-internal/engine/src/execute/tests/`: - - `suite/exec_flow/prompt_reads.rs` replaces `suite/exec_flow/list_from_section.rs`, with its module line and doc in `suite/exec_flow.rs`. One shared `const` document (`### Kid` with `#### Grand`, a numbered `## Nums`, a raw `## Rules` spliced from its own `const RULES`, an empty `## Empty`, and `## Other` with `### Niece`) is joined after each test's `## Main` with the `[..].concat()` form from `suite/exec_flow/fanout_arms.rs`. Three tests, built as the Testing Plan describes: `prompt_reads_resolve_over_the_visible_set`, `prompt_reads_raise_only_for_mistakes_in_the_call`, and `prompt_reads_work_in_h1_and_the_shared_library` + - `suite/exec_flow/prompt_reads.rs` replaces `suite/exec_flow/list_from_section.rs`, with its module line and doc in `suite/exec_flow.rs`. One shared `const` document (`### Kid` with `#### Grand`, a numbered `## Nums`, a raw `## Rules` spliced from its own `const RULES`, an empty `## Empty`, and `## Other` with `### Niece`) is joined after each test's `## Main` with the `[..].concat()` form from `suite/exec_flow/fanout_arms.rs`. `RULES` holds a Lua fence, `{{ args }}`, and a `---` with a blank line above it; it starts on a non-blank line and ends without a trailing newline or space, because the read trims both ends, and the document supplies the blank lines around it. Every test's `## Main` ends with a scalar `return`, which ends the run, so the walk never reaches the shared sections and `## Rules`'s fence never runs. Each Lua `assert` includes a message naming its case. The three tests: + - `prompt_reads_resolve_over_the_visible_set`: numbered items with markers stripped, a direct child's items, a list section's raw bullets, `""` for the empty section, and `nil` from both reads for a missing heading, the niece, the grandchild, and the caller, checked in one Lua loop. `## Main` then returns `prompt.section('## Rules')`, and Rust compares the run's output to `RULES` with `assert_eq!`. The expected text is never written inside a Lua string, because a line that is exactly the closing fence marker would end `## Main`'s own fence. + - `prompt_reads_raise_only_for_mistakes_in_the_call`: under `pcall`, a bare-name heading and a non-string argument raise from both reads, and `prompt.list('## Rules')` raises "has no pre-parsed items". A `call` to a missing heading still raises, and its listing (the text after "available sections:") names a sibling and a direct child but not the caller, the niece, or the grandchild; this keeps the deleted `list_from_section.rs` file's not-found-listing coverage, which no other engine test has. + - `prompt_reads_work_in_h1_and_the_shared_library`: the H1 block reads a top-level section and list and gets `nil` for `### Kid`; it asserts and returns nothing, because a scalar return from H1 ends the run before `## Main` executes. A `lua shared` library reads `prompt.list('### Kid')` into a global at load time, so each VM's replay resolves over its own visible set: the H1 block sees that global as `nil`, and `## Main`, whose child `### Kid` is, sees its items (`suite/exec_flow/store_failures.rs` shows how a test writes the `lua shared` fence) - the `nil`-rule edits: `call_and_list_inside_a_fanout_arm_reject_non_visible_sections` in `suite/exec_flow/fanout_arms.rs` asserts `prompt.list('## Parent') == nil`, with its doc comment and `expect` message no longer saying both error not-found; the two-string test in `suite/exec_flow/fanout_collections.rs` becomes `fanout_string_second_parameter_errors_pointing_at_prompt_list`, asserting on `prompt.list`, with its doc comment updated - the deletions: `list_from_section_ambiguous_error_is_loud` in `exec_flow.rs`, `list_from_section_from_h1_to_an_unknown_section_fails_the_run` in `suite/exec_flow/h1_control.rs`, and `list_from_section_works_on_the_h1` in `scheduler/live_h1/control_flow.rs` - The language spec, `../promptforge-design/design/architect-promptforge-prompting-language.md`: From 601fcb2edef6fdb4c781442283799a493361e5ce Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 11:06:33 -0700 Subject: [PATCH 04/34] Close plan: prompt-namespace-reads Plan: vibe/2026-10-10-2-prompt-namespace-reads.md --- vibe/ACTIVE | 1 - 1 file changed, 1 deletion(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 498efdf21..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-10-10-2-prompt-namespace-reads.md From 532497ac6761f965b77ce1753903e130b20d8a1e Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 15:31:16 -0700 Subject: [PATCH 05/34] Split build_sections into named check helpers Break the parser's section tree builder into small named checks so its loop reads as a sequence of steps: reject a heading that skips a level, reject an empty heading, detect a list section, and reject a duplicate sibling name. Each check keeps the error kind, location, and message it produced inline, and the checks run in the same order, so parsing behavior does not change. The test helper that builds synthetic sections also drops its list items argument, which every caller changed here passed empty. - `build_sections` keeps its signature and shrinks from 93 to 38 lines. Its loop now calls `check_child_level`, `section_name`, `list_items`, and `check_unique_sibling` in the order the inline checks ran. - `synthetic_section` loses its `items` parameter and always sets `items: Vec::new()`. The `sibling` helper and the three calls in `duplicate_top_level_section_names_error_loudly` drop the trailing `Vec::new()`. - `Error::parse_at` calls move into the helpers with the same `ParseErrorKind::Structure` kind, heading span, and message text, so parse errors are unchanged. - `exec_flow.rs` and `heading_address-tests.rs` change only call shape. No test or assertion changes. Design: removes oversized-unit @ crates/promptforge-internal/parser/src/build.rs::build_sections deps: Emitter,[Heading],u32,u8,usize Design: new pure-function @ crates/promptforge-internal/parser/src/build.rs::check_child_level deps: Heading,u8 Design: new pure-function @ crates/promptforge-internal/parser/src/build.rs::section_name deps: Heading,u8 Design: new stringly-typed @ crates/promptforge-internal/parser/src/build.rs::section_name deps: Heading,u8 Design: new pure-function @ crates/promptforge-internal/parser/src/build.rs::list_items deps: [Block],str Design: new stringly-typed @ crates/promptforge-internal/parser/src/build.rs::list_items deps: [Block],str Design: new pure-function @ crates/promptforge-internal/parser/src/build.rs::check_unique_sibling deps: Heading,[(String,u32)],str,u32 Design: new stringly-typed @ crates/promptforge-internal/parser/src/build.rs::check_unique_sibling deps: Heading,[(String,u32)],str,u32 Design: extends pure-function @ crates/promptforge-internal/parser/src/test_support.rs::synthetic_section deps: Vec,str,u8 Plan: vibe/2026-10-10-3-prompt-reads-debt-removal.md --- .../engine/src/execute/tests/exec_flow.rs | 6 +- .../engine/src/heading_address-tests.rs | 1 - .../promptforge-internal/parser/src/build.rs | 146 +++++---- .../parser/src/test_support.rs | 10 +- .../2026-10-10-3-prompt-reads-debt-removal.md | 276 ++++++++++++++++++ vibe/ACTIVE | 1 + 6 files changed, 370 insertions(+), 70 deletions(-) create mode 100644 vibe/2026-10-10-3-prompt-reads-debt-removal.md create mode 100644 vibe/ACTIVE diff --git a/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs b/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs index f73e70db9..96d15c279 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs @@ -130,9 +130,9 @@ Args: {{ args }}\n\n\ #[test] fn duplicate_top_level_section_names_error_loudly() { let sections = vec![ - synthetic_section("Main", 2, Vec::new(), Vec::new()), - synthetic_section("Dup", 2, Vec::new(), Vec::new()), - synthetic_section("Dup", 2, Vec::new(), Vec::new()), + synthetic_section("Main", 2, Vec::new()), + synthetic_section("Dup", 2, Vec::new()), + synthetic_section("Dup", 2, Vec::new()), ]; let error = super::super::walk_target::resolve_jump_target("## Dup", §ions, §ions[0]) .expect_err("two visible sections with one name must be ambiguous"); diff --git a/crates/promptforge-internal/engine/src/heading_address-tests.rs b/crates/promptforge-internal/engine/src/heading_address-tests.rs index ca191b259..9a8b8587f 100644 --- a/crates/promptforge-internal/engine/src/heading_address-tests.rs +++ b/crates/promptforge-internal/engine/src/heading_address-tests.rs @@ -28,7 +28,6 @@ fn sibling(name: &str, level: u8) -> Section { name, level, vec![promptforge_parser::test_support::prose_block(String::new())], - Vec::new(), ) } diff --git a/crates/promptforge-internal/parser/src/build.rs b/crates/promptforge-internal/parser/src/build.rs index a293a1a2d..093cea4e9 100644 --- a/crates/promptforge-internal/parser/src/build.rs +++ b/crates/promptforge-internal/parser/src/build.rs @@ -189,6 +189,85 @@ fn build_heading_blocks( Ok(blocks) } +/// Rejects an orphan deep heading: one that skips a level (e.g. an H4 under an +/// H2 with no intervening H3, or an H3/H4 top-level section with no parent H2). +/// Such a heading has no well-defined parent, so reject it rather than +/// silently reparenting it to a shallower ancestor. +fn check_child_level(heading: &Heading, parent_level: u8) -> Result<()> { + let level = heading.level; + if level > parent_level + 1 { + return Err(Error::parse_at( + ParseErrorKind::Structure, + heading.span.clone(), + format!( + "section `{}` is an orphan H{level} heading with no parent H{}", + heading.title.trim(), + parent_level + 1 + ), + )); + } + Ok(()) +} + +/// Returns the heading's section name. A section's name is its runtime address +/// (jumps, lookups, fanout), so an empty/whitespace heading is unaddressable +/// and must be rejected at parse. +fn section_name(heading: &Heading, level: u8) -> Result { + let name = heading.title.clone(); + if name.trim().is_empty() { + return Err(Error::parse_at( + ParseErrorKind::Structure, + heading.span.clone(), + format!("an H{level} section heading must not be empty"), + )); + } + Ok(name) +} + +/// Returns the section's list items, or none when it is not a list. A section +/// is a list only when it has no Lua and every nonblank prose line is a list +/// marker; one incidental bullet in mixed prose leaves a non-marker line, so +/// the section stays prose. +fn list_items(blocks: &[Block], name: &str) -> Result> { + let has_no_lua = blocks + .iter() + .all(|block| matches!(block, Block::Prose { .. })); + let prose_for_items = blocks.iter().find_map(|block| match block { + Block::Prose { text, .. } => Some(text.as_str()), + Block::Lua(_) => None, + }); + if has_no_lua + && let Some(prose) = prose_for_items + && is_all_list_markers(prose) + { + parse_bullet_items(prose, name) + } else { + Ok(Vec::new()) + } +} + +/// Rejects a section whose name an earlier sibling already took. Sections are +/// addressed by name (jumps, lookups), so two siblings sharing a name would +/// make the target ambiguous. The error names BOTH heading locations so the +/// author can find each one. +fn check_unique_sibling( + siblings: &[(String, u32)], + name: &str, + heading: &Heading, + heading_abs_line: u32, +) -> Result<()> { + if let Some((_, first_line)) = siblings.iter().find(|(n, _)| n == name) { + return Err(Error::parse_at( + ParseErrorKind::Structure, + heading.span.clone(), + format!( + "duplicate sibling section name `{name}`: first declared at line {first_line}, again at line {heading_abs_line}; sibling section names must be unique" + ), + )); + } + Ok(()) +} + /// Builds a section tree from a flat, document-ordered list of headings. /// /// Recursion consumes headings whose level is deeper than `parent_level`; a @@ -214,74 +293,19 @@ pub(crate) fn build_sections( // a duplicate can name both the first and the offending location. let mut sibling_lines: Vec<(String, u32)> = Vec::new(); while *pos < headings.len() { - let level = headings[*pos].level; + let h = &headings[*pos]; + let level = h.level; if level <= parent_level { break; } - // An orphan deep heading skips a level (e.g. an H4 under an H2 with no - // intervening H3, or an H3/H4 top-level section with no parent H2). Such - // a heading has no well-defined parent, so reject it rather than - // silently reparenting it to a shallower ancestor. - if level > parent_level + 1 { - return Err(Error::parse_at( - ParseErrorKind::Structure, - headings[*pos].span.clone(), - format!( - "section `{}` is an orphan H{level} heading with no parent H{}", - headings[*pos].title.trim(), - parent_level + 1 - ), - )); - } - let h = &headings[*pos]; - let name = h.title.clone(); - // A section's name is its runtime address (jumps, lookups, fanout), so an - // empty/whitespace heading is unaddressable and must be rejected at parse. - if name.trim().is_empty() { - return Err(Error::parse_at( - ParseErrorKind::Structure, - h.span.clone(), - format!("an H{level} section heading must not be empty"), - )); - } + check_child_level(h, parent_level)?; + let name = section_name(h, level)?; let heading_abs_line = line_add(frontmatter_lines, h.source_line)?; - let heading_span = h.span.clone(); let blocks = build_heading_blocks(h, &name, frontmatter_lines, emitter)?; *pos += 1; let children = build_sections(headings, pos, level, frontmatter_lines, emitter)?; - - let has_no_lua = blocks - .iter() - .all(|block| matches!(block, Block::Prose { .. })); - let prose_for_items = blocks.iter().find_map(|block| match block { - Block::Prose { text, .. } => Some(text.as_str()), - Block::Lua(_) => None, - }); - // A section is a list only when it has no Lua and every nonblank prose - // line is a list marker; one incidental bullet in mixed prose leaves a - // non-marker line, so the section stays prose. - let items = if has_no_lua - && let Some(prose) = prose_for_items - && is_all_list_markers(prose) - { - parse_bullet_items(prose, &name)? - } else { - Vec::new() - }; - - // Sibling sections must have unique names: sections are addressed by - // name (jumps, lookups), so two siblings sharing a name would make the - // target ambiguous. Reject the duplicate at parse, naming BOTH heading - // locations so the author can find each one. - if let Some((_, first_line)) = sibling_lines.iter().find(|(n, _)| *n == name) { - return Err(Error::parse_at( - ParseErrorKind::Structure, - heading_span, - format!( - "duplicate sibling section name `{name}`: first declared at line {first_line}, again at line {heading_abs_line}; sibling section names must be unique" - ), - )); - } + let items = list_items(&blocks, &name)?; + check_unique_sibling(&sibling_lines, &name, h, heading_abs_line)?; sibling_lines.push((name.clone(), heading_abs_line)); result.push(Section { diff --git a/crates/promptforge-internal/parser/src/test_support.rs b/crates/promptforge-internal/parser/src/test_support.rs index 52eba17f9..c2d3b2404 100644 --- a/crates/promptforge-internal/parser/src/test_support.rs +++ b/crates/promptforge-internal/parser/src/test_support.rs @@ -7,17 +7,17 @@ use crate::{Block, Section}; -/// Builds a synthetic section with the given blocks and pre-parsed items, so -/// each test fixture states only its own deltas (a prose block, a list of -/// items) instead of restating the parser's `Section` literal. +/// Builds a synthetic section with the given blocks, so each test fixture +/// states only its own deltas (a prose block) instead of restating the +/// parser's `Section` literal. #[must_use] -pub fn synthetic_section(name: &str, level: u8, blocks: Vec, items: Vec) -> Section { +pub fn synthetic_section(name: &str, level: u8, blocks: Vec) -> Section { Section { name: name.to_string(), level, blocks, children: Vec::new(), - items, + items: Vec::new(), markdown: String::new(), } } diff --git a/vibe/2026-10-10-3-prompt-reads-debt-removal.md b/vibe/2026-10-10-3-prompt-reads-debt-removal.md new file mode 100644 index 000000000..e8df07531 --- /dev/null +++ b/vibe/2026-10-10-3-prompt-reads-debt-removal.md @@ -0,0 +1,276 @@ +--- +name: prompt reads debt removal +overview: Remove the one cheap fix the prompt-namespace-reads work left, adopt two rejected candidates, and fix every pre-existing finding the debt collector exposed, including marking closed plans' todos completed going forward and across vibe/ history. +todos: + - id: parser-cleanup + content: "D1-5, C-1: split build_sections into four named checks; drop synthetic_section's items parameter and update its four call sites" + status: pending + - id: scope-module + content: "D1-14, D1-15: add h1_visible_sections beside visible_sections and use it at the three H1 sites; rename resolve_sibling to resolve_visible with its tests and doc links" + status: pending + - id: names-docs + content: "C-2, C-3, X-1: rename the fanout-arm test, fix the reserved tool-alias docs in globals.rs and parse.rs, set the two spec status lines to Active. (promptforge-design commit)" + status: pending + - id: plan-history + content: "D1-16: vibe coder close step marks todos completed (tools-public commit); flip todo statuses in every closed vibe/ plan, preserving line endings" + status: pending + - id: exit-gate + content: "Full exit gate: both nextest runs, both clippy runs, fmt, docs builds, xtask api --check, build-xtask tests, docs-claims, and the rg checks" + status: pending +isProject: false +--- + +# prompt reads debt removal + + + +## Product Requirements + +- Scope and target work + - Debt analysis target: the `vibe/2026-10-10-2-prompt-namespace-reads.md` plan, baseline `abe218a36`, endpoint and disposition ref `460da3f9e`, target commits `475fc39cb`, `4dc155672`, `7a80775b7`, `460da3f9e` in `c:\Users\Vinnie\cursor\promptforge`. Its spec commit `6341ac8` in `promptforge-design` was outside the analyzed repository. + - The user expanded the scope: every unrelated pre-existing finding is fixed here, and the rejected candidates were evaluated for adoption (two adopted). +- Cleanup goals + - Remove the orphaned `items` parameter of the parser's `synthetic_section` test helper (C-1). + - Rename a test whose name is now half false (C-2), and correct the two new spec status lines (X-1). + - Fix the pre-existing findings: split `build_sections` into named checks (D1-5), give H1's visible set one helper (D1-14), rename `resolve_sibling` to `resolve_visible` (D1-15), correct two stale doc comments about reserved tool aliases (C-3), and mark closed plans' todos completed, going forward in the vibe coder and across `vibe/` history (D1-16). +- Non-goals + - Any change to author-visible behavior: `prompt.section`, `prompt.list`, `call`, `jump`, `fanout`, error messages, and the reserved-name list stay as they are. + - Task execution behavior, the Harness, the public facade surface (`crates/promptforge/public-api.txt` stays the same). + - The rejected candidates listed under Deferred. +- Success criteria + - Every work item's check passes, and the full exit gate in the Testing Plan passes. + - `rg -n resolve_sibling crates` prints nothing, and H1's visible set is computed in exactly one function. + - No tracked `vibe/` plan except the active one has a frontmatter todo reading `pending` or `in_progress`, and the vibe coder's close step marks todos completed. + +## Functional Specification + +### Debt Inventory + +- Debt added + - None introduced or worsened. The analysis and challenge found no incorrect behavior, contract contradiction, or prospective debt with a present reversal burden in the target. +- Cheap fixes + - C-1, from `7a80775b7`: deleting `exec_flow.rs::list_from_section_ambiguous_error_is_loud` removed the only callers that passed non-empty `items` to `crates/promptforge-internal/parser/src/test_support.rs::synthetic_section`. All four remaining call sites (the `sibling` helper in `engine/src/heading_address-tests.rs`, three calls in `engine/src/execute/tests/exec_flow.rs::duplicate_top_level_section_names_error_loudly`) pass `Vec::new()`. Dead code the target orphaned; no reservation. Target state: the parameter is gone. +- Adopted rejected candidates + - C-2, residual-but-acceptable: `engine/src/execute/tests/suite/exec_flow/fanout_arms.rs::call_and_list_inside_a_fanout_arm_reject_non_visible_sections` now asserts `prompt.list('## Parent') == nil`, so "reject" is false for its list half. Adopted because the rename is free and the name misleads. + - X-1, the analysis's out-of-scope observation: `promptforge-design/design/architect-promptforge-prompting-language.md` lines 364 and 370 read "Active: new in this plan." and "Active: the list-section read, now `nil` on a miss." In that spec "this plan" means the spec's own plan, so the first wrongly claims `prompt.section` was part of it, and "now" is change-relative. The spec already uses a bare `Active.` elsewhere. Target state: both lines read `**Implementation status:** Active.` +- Exposed pre-existing debt, fixed by user scope expansion + - D1-5: `crates/promptforge-internal/parser/src/build.rs::build_sections` runs from line 205 to `Ok(result)` at line 296, about 92 lines, holding the orphan-level check, the empty-name check, list detection, and the duplicate-sibling check inline. Target state: each check is a named helper and `build_sections` reads as the sequence of steps. + - D1-14: H1's visible set (the whole top-level slice) is spelled out at three sites with the same comment: `engine/src/execute/section_context-construct.rs` line 171 (`new_live_h1`), `engine/src/execute/scheduler/walk.rs` line 290 (`resolve_chain_target`'s H1 branch), and `engine/src/execute/scheduler/h1.rs` line 148 (`end_live_h1_at_jump`). Target state: one helper beside `visible_sections`, so one module computes every visible set. + - D1-15: `engine/src/heading_address.rs::resolve_sibling` resolves over siblings and direct children. Target state: renamed `resolve_visible`, pairing with `find_visible`. + - C-3: `lua/src/globals.rs` module doc (lines 29 to 33) says "A frontmatter tool alias installs as a global of its own name, so the parser refuses one that is reserved", and `parser/src/parse.rs` line 32 lists "a tool alias is a reserved name" under `Frontmatter` errors. The frontmatter has only `plugins`, `args`, and `models`, the parser never calls `reserved_name`, and only `lua/src/prelude.rs` checks reserved names (for Plugin prelude globals). Target state: both docs state what is true. + - D1-16: 133 tracked plan files under `vibe/` carry frontmatter todos reading `status: pending` (5 of them also `status: in_progress`), though every one is closed. The vibe coder (`c:\Users\Vinnie\cursor\tools-public\planning\vibe-coder.md`, Run Sequence step 5) marks progress only in step headings and never touches todos. Target state: the close step marks the plan copy's todos completed, and every closed plan's todos read `completed`. +- Rejected candidate counts (22 total; 2 adopted above) + - residual-but-acceptable 7 (D1-1, D1-2, D1-4, D1-7, D1-8, D1-9, C-2): intended contract or bounded cost with no demonstrated consequence; C-2 adopted anyway. + - weak/speculative 3 (D1-3, C-4, C-6): no consequence shown. + - false 6 (D1-6, D1-10, D1-11, D1-12, D1-13, C-5): the code contradicts the candidate. + - unrelated pre-existing 5 (D1-5, D1-14, D1-15, D1-16, C-3): all fixed here by scope expansion. + + + + +## Technical Design + +- Paths are relative to `crates/promptforge-internal/` unless they start with `crates/`, `vibe/`, `../`, or name a sibling repository. +- Parser (D1-5, C-1): + - `parser/src/build.rs`: `build_sections` keeps its signature and behavior. Its inline checks move to private helpers with the exact error kinds, spans, and message texts they produce today: + - `fn check_child_level(heading: &Heading, parent_level: u8) -> Result<()>`: the orphan-heading error + - `fn section_name(heading: &Heading, level: u8) -> Result`: the empty-heading error, returning the name + - `fn list_items(blocks: &[Block], name: &str) -> Result>`: the list-only detection and `parse_bullet_items` call + - `fn check_unique_sibling(siblings: &[(String, u32)], name: &str, heading: &Heading, heading_abs_line: u32) -> Result<()>`: the duplicate-sibling error naming both lines + - `parser/src/test_support.rs`: `synthetic_section(name: &str, level: u8, blocks: Vec) -> Section` sets `items: Vec::new()`; its doc drops "a list of items". +- Visible sets and lookup (D1-14, D1-15): + - `engine/src/execute/walk_target.rs` gains, beside `visible_sections`: + + ```rust + /// The H1 pass's visible set: the whole top-level slice. H1 is + /// section 0, so it excludes nothing and has no children. + pub(super) fn h1_visible_sections(prompt: &Prompt) -> &[Section] { + promptforge_parser::detail::sections(prompt) + } + ``` + + The three H1 sites call it (`new_live_h1` keeps its `.to_vec()`, since the closure owns its set), and their duplicated "section 0 excludes nothing" comments go. Other `detail::sections` uses (`scheduler.rs` `SlicePath::resolve`, `context.rs` section count, the emptiness checks in `scheduler/drive.rs` and `scheduler/h1.rs` line 101) are not visible sets and stay. + - `engine/src/heading_address.rs`: `resolve_sibling` becomes `resolve_visible` with the same signature, body, and messages. Every caller, doc link, and test follows: `walk_target.rs` (2 mentions), `scheduler/walk.rs` (2), `scheduler/h1.rs` (1), `heading_address-tests.rs` (15, including the seven `resolve_sibling_*` test names, which become `resolve_visible_*`). That test file's module doc, "Tests for resolving a fanout worker heading among sibling sections.", becomes "Tests for heading lookup over a visible set." +- Docs (C-3, X-1): + - `lua/src/globals.rs` module doc: the sentence about frontmatter tool aliases becomes "A Plugin prelude global may not take one." + - `parser/src/parse.rs` `# Errors`: drop "a tool alias is a reserved name," from the `Frontmatter` list. + - `../promptforge-design/design/architect-promptforge-prompting-language.md` lines 364 and 370: `**Implementation status:** Active.` +- Test name (C-2): `call_and_list_inside_a_fanout_arm_reject_non_visible_sections` becomes `call_raises_and_list_reads_nil_for_non_visible_sections_in_a_fanout_arm`; body and doc comment unchanged. +- Plan history (D1-16): + - `c:\Users\Vinnie\cursor\tools-public\planning\vibe-coder.md`, Run Sequence step 5 becomes: "Set every frontmatter todo `status:` in to `completed`, then commit that change with the deletion of `vibe/ACTIVE` with subject `Close plan: `, a blank line, and the plan's `Plan:` trailer." Append one attribution line at the file's end in the existing form: `*2026-10-10 - Claude Opus 5.5 (Cursor agent). Close step marks the plan copy's todos completed.*` + - `vibe/`: in every tracked plan file except the one `vibe/ACTIVE` names, within the YAML frontmatter only (line 1 `---` through the next line that is exactly `---`), lines matching `^(\s+)status: (pending|in_progress)\s*$` become `$1status: completed`. Each file keeps its line endings and its final newline, and no other byte changes. Git stores every tracked file with LF; in the worktree `vibe/2026-10-09-3-models-loop-in-rust.md` is CRLF, and `vibe/2026-09/2026-09-26-4-store-into-vfs.md` holds two lone CR bytes that make git classify it as binary. +- Cross-repository commits: edits to `promptforge-design` and `tools-public` are committed in those repositories as their own commits, staging only the named file (`promptforge-design` holds an unrelated untracked `research/harness-survey-subagent-task-tools.md`). Sub-agents leave them unstaged; the session running the step commits them after the coding sub-agent returns and before that step's `promptforge` commit, and amends them after any later round that changes them. Nothing is pushed. +- No public interface, persisted or wire format, component ownership, dependency direction, or trust boundary changes. + + + + +## Testing Plan + +- Focused + - D1-5: `cargo nextest run --locked -p promptforge-parser --all-features` passes with no test edits; the orphan, empty-heading, duplicate-sibling, list, and line-mapping tests (`parser/src/tests/headings.rs`, `tests/lists.rs`, `tests.rs`, `build-tests.rs`) pin every message, kind, and span. + - C-1: the build is the check; every caller compiles against the three-argument `synthetic_section`. + - D1-15: the seven `resolve_visible_*` tests and `find_visible_returns_none_for_an_address_not_in_the_set` pass; `rg -n resolve_sibling crates` prints nothing. + - D1-14: the H1 suites pass unchanged (`suite/exec_flow/h1_control.rs`, `suite/exec_flow/prompt_reads.rs::prompt_reads_work_in_h1_and_the_shared_library`, `tests/scheduler/live_h1/`); `rg -n "detail::sections\(" crates/promptforge-internal/engine/src --glob "!**/tests/**" --glob "!*-tests.rs"` lists only `h1_visible_sections`, `scheduler.rs`, `context.rs`, `scheduler/drive.rs`, and `scheduler/h1.rs` line 101. + - C-2: `cargo nextest run --locked -p promptforge-engine --all-features call_raises_and_list_reads_nil` runs one test and passes. +- Docs + - C-3: `rg -n "tool alias is a reserved|parser refuses one that is reserved" crates` prints nothing; the docs build passes. + - X-1: `rg -n "Active: new in this plan|now .nil. on a miss" ../promptforge-design/design/architect-promptforge-prompting-language.md` prints nothing, and lines 364 and 370 read `**Implementation status:** Active.` +- Plan history (D1-16) + - `git grep -n -E "^\s+status: (pending|in_progress)\s*$" -- vibe` lists only the active plan's lines. + - `git diff --text --numstat -- vibe` shows equal added and deleted counts per file, and every changed line in `git diff --text -U0 -- vibe`, apart from the `---` and `+++` file headers, matches `^[-+]\s+status: `. Without `--text`, git shows the binary-classified plan file as `-` and hides its changes. + - `git ls-files --eol vibe` still shows every tracked file `i/lf` except `vibe/2026-09/2026-09-26-4-store-into-vfs.md` (`i/-text`), with `w/crlf` only on `vibe/.cursorignore` and `vibe/2026-10-09-3-models-loop-in-rust.md`. + - `rg -n -F "Set every frontmatter todo" c:\Users\Vinnie\cursor\tools-public\planning\vibe-coder.md` finds exactly one line, in Run Sequence step 5. +- Exit gate + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` + - both clippy runs with `CARGO_BUILD_WARNINGS=deny`: `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets` + - `cargo fmt --all --check` + - `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` and `cargo doc -p promptforge --no-deps`, both with `RUSTDOCFLAGS="-D warnings"` + - `cargo +nightly-2026-09-05 xtask api --check` (or the nightly `crates/build-xtask/src/api/toolchain.rs` names), `cargo test -p build-xtask`, and `node --test test/docs-claims.mjs` from `crates/workshop/ui` + + + + +## Decision Record + +- Reversible decisions and consequences + - `resolve_visible` is the new name: it pairs with `find_visible` (`Option` versus not-found error) and says what it searches. Consequence: seven test names and five call or doc sites change in one commit. + - H1's helper returns a borrowed slice, so `resolve_chain_target` and `end_live_h1_at_jump` keep resolving without a clone; only `new_live_h1` clones, as today. + - `build_sections`'s helpers stay private to `build.rs` and keep today's messages, so no test changes and the 500-line ceiling is not approached. + - C-1 removes the parameter rather than adding a list-section constructor; a future test that needs one adds it then, at the same cost. + - The history flip is a one-off data edit by the session, not committed tooling, so no ratchet or parser of plan files enters the repository. +- User-resolved choices + - Scope expansion: "for any pre-existing finding, I want it fixed". + - D1-16: fix it going forward in the vibe coder and also flip every closed plan in `vibe/` (chosen over forward-only and leaving it). +- Rejected alternatives, assumptions, and risks + - Not adopted: D1-1 and D1-2 (the chosen design), D1-3 (one parse function, two callers, no divergence), D1-4 (the compiler's bound check covers every site), D1-6 (forced by `parser -> lua`), D1-7 (a second callback was rejected in the target plan), D1-8 (pre-existing deep-clone pattern; `Arc`-shared sections is a wider refactor with no measured cost), D1-9 (unreachable branch still covered by `resolve_visible_rejects_more_than_one_match` and `duplicate_top_level_section_names_error_loudly`), D1-10 (same as `models` and `messages`; preludes cannot see it), C-4 (no non-test `Prompt` comparison), C-6 (a parameter placeholder in error text; changing it churns pinned messages). + - Risk: flipping history marks every todo `completed`, including any a past run skipped; the step headings and git log stay the authoritative record. + - Risk: the session running this plan may hold the vibe coder's old close step. This plan's own close commit therefore sets this plan copy's todos to `completed` explicitly, whatever version the session read. + - Assumption: `vibe/ACTIVE` names this plan's copy during the run, so the history flip excludes it. + +### Deferred and Out of Scope + +- D1-8 visible-set clone cost: revisit if large prompts show memory pressure, with `Arc`-shared sections or `Arc` Markdown. +- D1-3 typed heading address: revisit if a second heading parser or a parse divergence appears. +- C-6 `models.infer takes (prompt)` wording: revisit if authors report confusing it with the `prompt` namespace. +- C-4 `Prompt` equality including raw Markdown: revisit if a non-test consumer starts comparing prompts. +- The spec commit `6341ac8`'s other content was not analyzed beyond X-1: revisit in a debt run that targets `promptforge-design`. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build --locked -p gateway`. `crates/gateway/app` is the only `default-members` entry, so plain `cargo build` builds just the gateway. Build scripts bundle the UIs into `OUT_DIR`, so a fresh clone first runs `npm ci --prefix crates/workshop` and `npm ci --prefix crates/gateway/config-ui/ui`, as every `.github/workflows/ci.yml` cargo job does. The desktop app is an explicit `cargo build --locked -p workshop`, and CI first builds `cargo build --locked -p gateway --no-default-features` and stages it with `node tools/stage-gateway-sidecar.mjs stage --target --source target/debug/promptforge-gateway[.exe]` (removed afterwards with `node tools/stage-gateway-sidecar.mjs remove --target `). Windows links with `rust-lld` and a static CRT (`.cargo/config.toml`). +- Focused test command pattern: `cargo nextest run --locked -p --all-features `, dropping `--all-features` for `workshop`, `workshop-server`, and `workshop-server-api`. A single JS test file runs with `node --test ` from its package directory. +- Component test command pattern: `cargo nextest run --locked -p --all-features`, without `--all-features` for the three workshop crates above. CI adds `cargo nextest run --locked -p workshop-workspace --all-features` (Windows path jail) and `cargo nextest run --locked -p workshop-server --features headless`. JS packages: `npm test --workspace ` from `crates/workshop`, or `npm test` from `crates/gateway/config-ui/ui`. +- Full-suite test command: `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api`. The first run includes `build-xtask`, whose tests are the boundary and structural checks (`cargo xtask tidy` prints the same report). Also run `node --test crates/workshop/ui/test/docs-claims.mjs` (wording scan, no `npm ci` needed), `npm test --workspaces --if-present` in `crates/workshop`, and `npm test` in `crates/gateway/config-ui/ui`. CI's Windows and Linux workshop jobs also run five named Gateway process-ownership race tests with plain `cargo test` (`gateway-api-discovery`, and `gateway --no-default-features --features test-fixtures --test it`). Nextest config is `.config/nextest.toml`: 60 s slow timeout terminated after 3 periods, 250 ms leak timeout, STT crates in a capped `heavy` group, and longer timeouts for the realtime STT tests. `tools/*.test.mjs` exist but no CI job runs them. +- Linter command: `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, both with `CARGO_BUILD_WARNINGS=deny` (PowerShell: `$env:CARGO_BUILD_WARNINGS='deny'`). Also `cargo check -p gateway --no-default-features` for the headless build shape (the only allowed standalone `cargo check`), `npm run typecheck --workspaces --if-present` in `crates/workshop`, `npm run typecheck` in `crates/gateway/config-ui/ui`, and `cargo deny check`, `cargo audit`, and `cargo hakari verify` for supply chain. CI also fails if `ring` enters the gateway's normal dependency closure (`cargo tree --locked -p gateway -e normal -i ring --depth 0`) or if a build dirties the tree. `.githooks/pre-push` runs the wording scan, the headless check, the first clippy run, and `cargo deny --log-level error check` when installed. +- Formatter check command: `cargo fmt --all --check` (stable toolchain per `rust-toolchain.toml`, `style_edition = "2024"` in `rustfmt.toml`; `.githooks/pre-commit` runs it). +- Docs command: `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` with `RUSTDOCFLAGS="-D warnings"`. Under the same flags CI also runs `cargo doc -p promptforge --no-deps` and `cargo doc -p harness --no-deps` with default features, `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items`, and on Windows `cargo doc --locked --no-deps -p workshop-server --document-private-items`. Facade surface: `cargo +nightly-2026-09-05 xtask api --check` against the committed `crates/promptforge/public-api.txt` (the nightly is pinned in `crates/build-xtask/src/api/toolchain.rs`), plus `cargo nextest run --locked -p build-xtask --run-ignored only` on that nightly for the rustdoc JSON fixtures. The docs site is `cargo xtask site [--books-only]`, which needs `PROMPTFORGE_DOCS` pointing at a `promptforge-docs` checkout and `mdbook`, and writes `target/site/`. +- Test placement and naming conventions: Unit tests live in a sibling file wired with `#[cfg(test)] #[path = "-tests.rs"] mod tests;` (174 files use it, for example `crates/promptforge-internal/lua/src/handles.rs` beside `handles-tests.rs`), or in a `tests/` subdirectory of the module once it has three or more files (for example `crates/promptforge-internal/engine/src/execute/tests/`). Integration tests compile as one binary per crate from `tests/it/main.rs` (most crates) or `tests/suite/main.rs` (`promptforge`, `harness`), with shared helpers in `support.rs` (or `tests/common/`) and one file per topic (`cancel.rs`, `replay.rs`, `vfs.rs`); data sits in `tests/fixtures/` or `tests/prompts/`. Test function names are snake_case sentences describing the behavior (`a_process_lifetime_lease_recovers_after_its_owner_is_terminated`). Plugin crates enable `promptforge-plugin`'s `test-support` feature from `[dev-dependencies]` only, and `build-xtask`'s `test_support_leak` guard fails any non-dev table that enables an Engine or Harness crate's `test-support`. Doctests are banned in every workspace crate (`no_doctests`). `build-xtask` fixtures that need rustdoc JSON are `#[ignore]`d on stable. JS tests are `node:test` files in `test/**/*.mjs` (workshop `ui`, `look`, `platform`) and `src/**/*.test.mjs` (workshop `ui` and the gateway config UI). +- Directory map: `crates/` holds every crate (52 manifests). `promptforge` is the Engine facade (with `public-api.txt`), and the manifestless `promptforge-internal/` container holds the private Engine crates `types`, `vfs`, `parser`, `lua`, `model-client`, and `engine`. `promptforge-plugin` is the Plugin contract, implemented by `plugin-web`, `plugin-mcp`, and `plugin-user-input`. `harness` is the Harness facade, `harness-internal/runner` its private runner, and `harness-gateway-client` the gateway client. `gateway/` holds the gateway family (`app` binary with `admin`, `auth`, `boot`, `commands`, `dialect`, `error`, `runner`, `tray` modules; `config`, `config-ui`, `local`, `routing`, `protocol`, `cloud-providers`, `web-search`, `logging`, `progress`, and the `stt/` speech subsystem with `api`, `engine`, `backend-whisper`, `whisper-ffi`); `gateway-api-types` and `gateway-api-discovery` are its public pair. `workshop/` holds the Workshop Host: the `desktop` Tauri app, `server`, `server-api`, the tiered library crates, and the `ui`, `look`, `platform` npm workspaces. Also in `crates/`: `shared-error-source`, `shared-loopback`, and `shared-ui` (a TypeScript and CSS package, not a crate); `build-*` tooling (`build-xtask` for structural checks and `cargo xtask tidy|api|site|new-crate`, `build-workshop` behind `cargo workshop`, `build-ceiling` for the file-size limit, `build-ui`, `build-user-guide`, `build-llama-cuda`); and `workspace-hack` (cargo-hakari, configured in `.config/hakari.toml`). Top level: `guide/` (`books/{language,gateway,workshop}/book.toml` only, since chapters come from the separate `promptforge-docs` checkout; `chrome/` and `landing/` site assets), `prompts/` (sample prompts), `tools/` (Node scripts for gateway sidecar staging and live TTS, with tests), `.github/workflows/` (CI, release, nightly, site, STT Miri, and others), `.githooks/`, `.cursor/rules/` (`workshop-architecture.mdc`, `workshop-spa.mdc`), `vibe/` (past plans, tracked), `local/` (gitignored machine-local config and fixtures), and `images/`. +- Component boundaries: `cargo test -p build-xtask` enforces these (`crates/build-xtask/src/product.rs`, `tidy.rs`, and the guard modules it runs). Engine crates (`promptforge*`) depend on no gateway, workshop, Harness, or Plugin crate, and their manifests may not name `tokio`, `tokio-util`, `async-trait`, or `reqwest` outside `[dev-dependencies]`, except an optional entry only `test-support` enables (`engine_deps.rs`). Inside the Engine, `types` and `vfs` are leaves, `model-client` depends on `types`, `lua` on `types`, `vfs`, and `model-client`, `parser` on `lua` and `types`, `engine` on all five, and the `promptforge` facade on all six. `promptforge-plugin` depends on `types` and `vfs`. Plugin crates depend only on `promptforge-plugin`, `shared-*`, `workspace-hack`, and outside libraries. Harness crates name only `promptforge`, `promptforge-plugin`, and `workspace-hack` outside their family (`harness` uses `harness-runner`; `harness-gateway-client` may also name `plugin-web` to implement its `SearchProvider`), and no Harness crate declares `tokio` or `tokio-util` as a normal dependency (`harness_bans.rs`). Gateway crates depend on no Engine, Harness, or workshop crate. Workshop crates reach the gateway only through `gateway-api-types` and `gateway-api-discovery`, and the Engine and Harness only through `promptforge`, `promptforge-plugin`, `harness`, and `harness-gateway-client`. `shared-*` crates depend on no product crate. Containers are private: only `promptforge` and `promptforge-plugin` may enter `promptforge-internal/`, only `harness` may enter `harness-internal/`, and only `gateway-stt` is public within `gateway/stt/`. The `promptforge` and `harness` facades hold only doc-commented `pub mod` groups and single-item `pub use ::path::Item;` re-exports (`facade_shape.rs`), and no file in either facade or its container carries `doc(hidden)` (`doc_hidden.rs`). A retired-symbol list may not reappear in live Engine source (`retired_symbols.rs`). Only `gateway/app`, `gateway/stt/whisper-ffi`, `gateway-api-discovery`, and `workshop/desktop` may relax `unsafe_code` (`unsafe_allowlist.rs`). Only the gateway app's `admin/walled/` modules plus `lib.rs`, `registry.rs`, and `test_support.rs` may name `crate::admin::walled::`. Workshop tiers run vocabulary (`protocol`, `registry`, `support`), then services (`gateway`, `menu`, `status`), then features (`agents`, `run-log`, `user-state`, `workspace`), then `workshop-server`, each depending only downward. The `workshop` desktop app depends on no workshop crate except `workshop-server-api`, which depends on none except `workshop-server`. +- Conventions summary: The four defined terms in `AGENTS.md` (Engine, Harness, Host, Plugin) are capitalized with one meaning each, and Engine crates say "the caller" and never mention the Host (`crates/workshop/ui/test/docs-claims.mjs` scans for this). All 52 crate manifests set `publish = false` and `[lints] workspace = true`, inherit workspace package fields, and take dependencies as `.workspace = true`; every crate except `promptforge-vfs` and `build-ceiling` depends on `workspace-hack`. Every crate's build script except `workspace-hack`'s runs `build_ceiling::check()`, which fails any file over 500 lines. Manifests comment why each dependency, pin, or feature exists. Lints are strict: clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, `allow_attributes` is denied so suppressions use `#[expect(..., reason = "...")]`, `unsafe_code` is denied outside the allowlisted crates, `missing_docs` and `unreachable_pub` warn (fatal under the gate), and `clippy.toml` bans process-global installers outside binary entry points. Every `workshop-*`, `harness-*`, and `plugin-*` crate (not the `workshop` app or `harness` facade) carries a `//! ## Invariants` section in its crate docs. Source directories stay flat until a group reaches three files, using kebab siblings with `#[path]` below that. Comments explain only non-obvious constraints and cite upstream issue URLs for workarounds. Error messages are written for model consumption, naming required versus actual. JSON that reaches a recorder round-trips exactly, with sorted keys and `float_roundtrip`. Behavior changes ship with tests in the same change. Cargo features gate only real build constraints. Changes to the facade surface update `public-api.txt`. Commits are one-line imperative summaries (`Replace list_from_section with prompt.section/list`), plans are kept in `vibe/` as `YYYY-MM-DD-N-.md`, and a finished plan closes with a `Close plan: ` commit. + + + + +## Execution Instructions + + + +### Step 1: Split build_sections into named checks and drop synthetic_section's items parameter [completed] + +- Component: Parser cleanup +- Placement: first. Engine scope module depends on it: both edit `engine/src/heading_address-tests.rs`, and this step changes the `synthetic_section` signature that file's `sibling` helper calls. +- Pieces: the `build_sections` split (D1-5) and the `synthetic_section` trim (C-1), built jointly. They share no code, but both are parser-crate edits covered by the same parser and engine test runs. +- Paths are relative to `crates/promptforge-internal/`, as in the Technical Design. +- `parser/src/build.rs`: `build_sections` keeps its signature and behavior. Move its four inline checks into private helpers that produce today's exact error kinds, spans, and message texts, so `build_sections` reads as the sequence of steps: + - `fn check_child_level(heading: &Heading, parent_level: u8) -> Result<()>`: the orphan-heading error + - `fn section_name(heading: &Heading, level: u8) -> Result`: the empty-heading error, returning the name + - `fn list_items(blocks: &[Block], name: &str) -> Result>`: the list-only detection and its `parse_bullet_items` call + - `fn check_unique_sibling(siblings: &[(String, u32)], name: &str, heading: &Heading, heading_abs_line: u32) -> Result<()>`: the duplicate-sibling error naming both lines +- `parser/src/test_support.rs`: `synthetic_section(name: &str, level: u8, blocks: Vec) -> Section` sets `items: Vec::new()`, and its doc drops "a list of items". +- Call sites drop the trailing `Vec::new()`: the `sibling` helper in `engine/src/heading_address-tests.rs` and the three calls in `engine/src/execute/tests/exec_flow.rs::duplicate_top_level_section_names_error_loudly`. +- Tests: no test or assertion changes. The orphan, empty-heading, duplicate-sibling, list, and line-mapping tests in `parser/src/tests/headings.rs`, `parser/src/tests/lists.rs`, `parser/src/tests.rs`, and `parser/src/build-tests.rs` pin every message, kind, and span; for C-1 the build is the check. +- Verify: `cargo nextest run --locked -p promptforge-parser --all-features` and `cargo nextest run --locked -p promptforge-engine --all-features` pass. +- Commit: one `promptforge` commit with the parser edits and the four call sites. + + + + + +### Step 2: Give H1's visible set one helper and rename resolve_sibling to resolve_visible + +- Component: Engine scope module +- Placement: second, after Parser cleanup, because both edit `engine/src/heading_address-tests.rs` (the `sibling` helper there, then the renamed tests), so they run in sequence in one checkout. +- Pieces: `h1_visible_sections` (D1-14) and the `resolve_visible` rename (D1-15), built jointly: both edit `execute/walk_target.rs`, `execute/scheduler/walk.rs`, and `execute/scheduler/h1.rs`, and the engine suite covers both. +- Paths are relative to `crates/promptforge-internal/engine/src/`. +- `execute/walk_target.rs`: add `pub(super) fn h1_visible_sections(prompt: &Prompt) -> &[Section]` beside `visible_sections`, returning the whole top-level slice from `promptforge_parser::detail::sections` with the doc comment given in the Technical Design. Import `crate::parser::Prompt` next to the existing `crate::parser::Section`. +- Call `h1_visible_sections` at the three H1 sites and delete each one's duplicated "section 0 excludes nothing" comment: `execute/section_context-construct.rs::new_live_h1` (keeps its `.to_vec()`, since the closure owns its set), `execute/scheduler/walk.rs::resolve_chain_target`'s H1 branch, and `execute/scheduler/h1.rs::end_live_h1_at_jump`. Leave the other `detail::sections` uses, which are not visible sets: `execute/scheduler.rs` (`SlicePath::resolve`), `execute/context.rs` (the section count), and the emptiness checks in `execute/scheduler/drive.rs` and near line 101 of `execute/scheduler/h1.rs`. +- `heading_address.rs`: rename `resolve_sibling` to `resolve_visible` with the same signature, body, and messages. Update every mention: `heading_address.rs` itself (2), `execute/walk_target.rs` (2), `execute/scheduler/walk.rs` (2), `execute/scheduler/h1.rs` (1), and `heading_address-tests.rs` (15, including the seven `resolve_sibling_*` test names, which become `resolve_visible_*`). That test file's module doc becomes "Tests for heading lookup over a visible set." +- Tests: bodies and assertions stay; only the seven test names and the module doc change. The H1 suites stay unchanged: `execute/tests/suite/exec_flow/h1_control.rs`, `execute/tests/suite/exec_flow/prompt_reads.rs::prompt_reads_work_in_h1_and_the_shared_library`, and `execute/tests/scheduler/live_h1/`. +- Verify: + - `rg -n resolve_sibling crates` prints nothing. + - `rg -n "detail::sections\(" crates/promptforge-internal/engine/src --glob "!**/tests/**" --glob "!*-tests.rs"` lists exactly five sites: `h1_visible_sections` in `walk_target.rs`, `scheduler.rs`, `context.rs`, `scheduler/drive.rs`, and the emptiness check in `scheduler/h1.rs`. + - `cargo nextest run --locked -p promptforge-engine --all-features` passes, including the `resolve_visible_*` tests and `find_visible_returns_none_for_an_address_not_in_the_set`. +- Commit: one `promptforge` commit. + + + + + +### Step 3: Mark closed plans' todos completed, going forward and across vibe/ history + +- Component: Plan history +- Placement: third. It depends on no other component. It follows the code components so the 133-file data edit stays out of the `[WIP] Plan:` seed commit that Step 1 amends, and it precedes the last step so that step's exit gate runs once after every work item lands. +- Pieces: the vibe coder close step (forward) and the `vibe/` history flip (backward), built jointly because they are the two halves of D1-16 and share one set of checks. Inside the step, the `tools-public` commit comes first, then the `promptforge` commit. +- `c:\Users\Vinnie\cursor\tools-public\planning\vibe-coder.md`: + - Run Sequence step 5, today "Commit the deletion of `vibe/ACTIVE` with subject `Close plan: `, a blank line, and the plan's `Plan:` trailer.", becomes "Set every frontmatter todo `status:` in to `completed`, then commit that change with the deletion of `vibe/ACTIVE` with subject `Close plan: `, a blank line, and the plan's `Plan:` trailer." + - Append after the existing `*2026-09-08 - GPT-5.6 Sol (Cursor agent)*` line, separated from it by one blank line: `*2026-10-10 - Claude Opus 5.5 (Cursor agent). Close step marks the plan copy's todos completed.*` + - The coding sub-agent leaves the file unstaged. The session commits `planning/vibe-coder.md` alone in `tools-public` after the sub-agent returns and before this step's `promptforge` commit, and amends that commit after any later round that changes the file. Nothing is pushed. +- `vibe/` history flip, a one-off session data edit with no committed tooling: + - Line endings before the edit, which the edit must leave as they are: every tracked `vibe/` file is `i/lf` except `vibe/2026-09/2026-09-26-4-store-into-vfs.md` (`i/-text`), and only `vibe/.cursorignore` and `vibe/2026-10-09-3-models-loop-in-rust.md` are `w/crlf`. + - In every tracked `vibe/**/*.md` plan except the one `vibe/ACTIVE` names (today 133 files have open todos, 5 of them with `in_progress`), within the YAML frontmatter only (line 1 `---` through the next line that is exactly `---` apart from its terminator), lines matching `^(\s+)status: (pending|in_progress)\s*$` become `$1status: completed`. + - Edit bytes line by line and keep each line's own terminator: `vibe/2026-10-09-3-models-loop-in-rust.md` is CRLF in the working tree, so the trailing `\s*` must not swallow its `\r`. Keep each file's final newline and change no other byte, including the two lone CR bytes in `vibe/2026-09/2026-09-26-4-store-into-vfs.md`. +- Close requirement: the flip skips this plan's own copy. The run's close commit sets that copy's frontmatter todos to `completed` before deleting `vibe/ACTIVE`, whatever close-step version the session read. +- Verify: + - `git grep -n -E "^\s+status: (pending|in_progress)\s*$" -- vibe` lists only the active plan copy's lines. + - `git diff --text --numstat -- vibe` shows equal added and deleted counts per file, and every added or removed line in `git diff --text -U0 -- vibe`, apart from the `---` and `+++` file headers, matches `^[-+]\s+status: `. The `--text` flag is required: git classifies `vibe/2026-09/2026-09-26-4-store-into-vfs.md` as binary because of its lone CR bytes, so without it numstat prints `-` for that file and the line check cannot see its changes. + - `git ls-files --eol vibe` still shows every tracked file `i/lf` except `vibe/2026-09/2026-09-26-4-store-into-vfs.md` (`i/-text`), with `w/crlf` only on `vibe/.cursorignore` and `vibe/2026-10-09-3-models-loop-in-rust.md`. + - `rg -n -F "Set every frontmatter todo" c:\Users\Vinnie\cursor\tools-public\planning\vibe-coder.md` finds exactly one line, in Run Sequence step 5. +- Commit: the `tools-public` commit of `planning/vibe-coder.md` alone, then one `promptforge` commit with the flipped `vibe/` files. + + + + + +### Step 4: Correct the fanout-arm test name, the reserved-alias docs, and the spec status lines, then run the exit gate + +- Component: Names and docs +- Placement: last. It depends on no other component, and the exit gate attached to it runs once after every work item lands, so any gate fix goes into a commit that is not yet finalized. +- Pieces: the test rename (C-2), the doc corrections (C-3), and the spec status lines (X-1), built jointly: three independent text edits covered by one engine run, the docs build, and two `rg` checks. +- Paths are relative to `crates/promptforge-internal/` unless they name a sibling repository. +- `engine/src/execute/tests/suite/exec_flow/fanout_arms.rs`: rename `call_and_list_inside_a_fanout_arm_reject_non_visible_sections` to `call_raises_and_list_reads_nil_for_non_visible_sections_in_a_fanout_arm`; body and doc comment stay. +- `lua/src/globals.rs` module doc (lines 29 to 33): the sentence "A frontmatter tool alias installs as a global of its own name, so the parser refuses one that is reserved" becomes "A Plugin prelude global may not take one." +- `parser/src/parse.rs` `# Errors` (line 32): drop "a tool alias is a reserved name," from the `Frontmatter` list. +- `c:\Users\Vinnie\cursor\promptforge-design\design\architect-promptforge-prompting-language.md` lines 364 and 370: both read `**Implementation status:** Active.` The coding sub-agent leaves the file unstaged. The session commits it alone in `promptforge-design`, staging only that file, after the sub-agent returns and before this step's `promptforge` commit, and amends that commit after any later round that changes it. Nothing is pushed. +- Verify: + - `cargo nextest run --locked -p promptforge-engine --all-features call_raises_and_list_reads_nil` runs one test and passes. + - `rg -n "tool alias is a reserved|parser refuses one that is reserved" crates` prints nothing. + - `rg -n "Active: new in this plan|now .nil. on a miss" ../promptforge-design/design/architect-promptforge-prompting-language.md` prints nothing, and lines 364 and 370 read `**Implementation status:** Active.` +- Exit gate, run once from the `promptforge` root after this step's edits; fixes land in this step's commit: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` + - With `CARGO_BUILD_WARNINGS=deny` (PowerShell: `$env:CARGO_BUILD_WARNINGS='deny'`): `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets` + - `cargo fmt --all --check` + - With `RUSTDOCFLAGS="-D warnings"`: `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` and `cargo doc -p promptforge --no-deps` + - `cargo +nightly-2026-09-05 xtask api --check` (or the nightly `crates/build-xtask/src/api/toolchain.rs` names), which confirms `crates/promptforge/public-api.txt` is unchanged, then `cargo test -p build-xtask` + - `node --test test/docs-claims.mjs` from `crates/workshop/ui` + - Rerun this step's `rg` checks, Step 2's two `rg` checks, and Step 3's `git grep` check. +- Commit: the `promptforge-design` commit of the spec file alone, then one `promptforge` commit. + + + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..5134f338e --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-10-10-3-prompt-reads-debt-removal.md From c47e802d8b469fa5f46cc52922adca619b136aa9 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 15:46:14 -0700 Subject: [PATCH 06/34] Give H1's visible set one helper, rename resolve_sibling The H1 pass sees every top-level section, and three places in the engine each built that set inline with the same comment explaining why. They now call one helper that names the set and states the reason once. The heading resolver behind jumps and calls is renamed to say it resolves a heading within a visible set, which is what its callers hand it, rather than among siblings. Its signature, body, and error messages stay the same, so behavior does not change. - `h1_visible_sections` sits beside `visible_sections` and borrows the whole top-level slice from the parsed prompt. It is `pub(super)`, and the H1 doc comments now link to it instead of restating why section 0 excludes nothing. - `resolve_visible` replaces `resolve_sibling` with the same signature, body, and messages, and its callers and doc links follow the new name. - `read_visible` in the H1 section context still reads an owned copy of the set made with `.to_vec()`, because the `move` closure around it keeps the set. The two scheduler sites borrow the slice. - `promptforge_parser::detail::sections` stays a direct call in the H1 pass's emptiness check, which asks whether any section exists, not which ones are visible. - `resolve_visible_finds_exact_match` and the six other renamed tests keep their bodies and assertions. Only their names and the module doc change. Design: new pure-function @ crates/promptforge-internal/engine/src/execute/walk_target.rs::h1_visible_sections deps: Prompt Design: replaces pure-function @ crates/promptforge-internal/engine/src/heading_address.rs::resolve_visible deps: Section,str was: crates/promptforge-internal/engine/src/heading_address.rs::resolve_sibling Design: replaces stringly-typed @ crates/promptforge-internal/engine/src/heading_address.rs::resolve_visible deps: Section,str was: crates/promptforge-internal/engine/src/heading_address.rs::resolve_sibling Plan: vibe/2026-10-10-3-prompt-reads-debt-removal.md --- .../engine/src/execute/scheduler/h1.rs | 11 +++---- .../engine/src/execute/scheduler/walk.rs | 13 ++++---- .../src/execute/section_context-construct.rs | 11 +++---- .../engine/src/execute/walk_target.rs | 12 +++++-- .../engine/src/heading_address-tests.rs | 32 +++++++++---------- .../engine/src/heading_address.rs | 4 +-- .../2026-10-10-3-prompt-reads-debt-removal.md | 2 +- 7 files changed, 43 insertions(+), 42 deletions(-) diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs b/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs index 8a4655e72..e74e7de61 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use promptforge_types::ids::ChainId; use crate::execute::support::GENERIC_COMPLETION; -use crate::execute::walk_target::section_position; +use crate::execute::walk_target::{h1_visible_sections, section_position}; use crate::heading_address; use crate::{Error, Result}; @@ -129,9 +129,8 @@ impl Scheduler { Ok(()) } - /// Ends the H1 pass on a jump out: the heading resolves against the - /// top-level sections (H1's visible set - section 0 excludes nothing - /// and has no children), then the pass ends and the root walk starts + /// Ends the H1 pass on a jump out: the heading resolves against + /// [`h1_visible_sections`], then the pass ends and the root walk starts /// at the target. /// /// # Errors @@ -145,8 +144,8 @@ impl Scheduler { root_result: &mut Option>, ) -> Result<()> { let prompt = self.prompt(); - let sections = promptforge_parser::detail::sections(&prompt); - let target = heading_address::resolve_sibling(heading, sections)?; + let sections = h1_visible_sections(&prompt); + let target = heading_address::resolve_visible(heading, sections)?; let start = section_position(sections, target).ok_or(Error::internal( "a resolved H1 jump target is absent from the top-level slice", ))?; diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs b/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs index 8ed0a8602..e63fe2bbc 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs @@ -14,7 +14,9 @@ use std::sync::Arc; use promptforge_types::ids::ChainId; use crate::execute::section_context::SectionContext; -use crate::execute::walk_target::{JumpTarget, resolve_jump_target, section_position}; +use crate::execute::walk_target::{ + JumpTarget, h1_visible_sections, resolve_jump_target, section_position, +}; use crate::heading_address; use crate::parser::Block; use crate::{Error, Result}; @@ -275,7 +277,7 @@ impl Scheduler { /// # Errors /// Returns [`Error::Lua`] when the heading is malformed, matches no /// visible section, or matches more than one (see - /// [`heading_address::resolve_sibling`]). + /// [`heading_address::resolve_visible`]). pub(super) fn resolve_chain_target( &self, id: ChainIndex, @@ -284,11 +286,8 @@ impl Scheduler { let prompt = self.ctx.prompt(); let chain = &self.chains[id.index()]; if chain.h1 { - // H1 is section 0: its visible set is the whole top-level - // slice - it excludes nothing and has no children, so every - // target is a flat index into that slice. - let sections = promptforge_parser::detail::sections(prompt); - let target = heading_address::resolve_sibling(heading, sections)?; + let sections = h1_visible_sections(prompt); + let target = heading_address::resolve_visible(heading, sections)?; let index = section_position(sections, target).ok_or(Error::internal( "a resolved H1 target is absent from the top-level slice", ))?; diff --git a/crates/promptforge-internal/engine/src/execute/section_context-construct.rs b/crates/promptforge-internal/engine/src/execute/section_context-construct.rs index 878fb3bf9..94cf9db7c 100644 --- a/crates/promptforge-internal/engine/src/execute/section_context-construct.rs +++ b/crates/promptforge-internal/engine/src/execute/section_context-construct.rs @@ -15,7 +15,7 @@ use promptforge_types::ids::{ChainId, TaskId}; use crate::Result; use crate::execute::context::RunState; use crate::execute::section_vm::{VmSeed, setup_section_vm}; -use crate::execute::walk_target::{read_visible, visible_sections}; +use crate::execute::walk_target::{h1_visible_sections, read_visible, visible_sections}; use crate::lua::SectionVm; use crate::parser::Section; use promptforge_types::event::lifecycle; @@ -134,9 +134,8 @@ impl SectionContext { /// /// H1's only deltas from a walked section: no `SECTION_STARTED` /// observation (the pass is not a walked section), an empty `var` seed - /// (it runs first and is never re-entered), and a visible set behind - /// the `prompt` reads spanning the whole top-level slice - section 0 - /// excludes nothing and has no children. + /// (it runs first and is never re-entered), and [`h1_visible_sections`] + /// as the visible set behind the `prompt` reads. /// /// # Errors /// Returns the [`Error`](crate::Error) of whichever step failed. A VM @@ -166,9 +165,7 @@ impl SectionContext { ctx.limits().lua_memory().get(), ctx.limits().lua_logs().get(), )?; - // H1's visible set is the whole top-level slice: section 0 - // excludes nothing and has no children. - let visible = promptforge_parser::detail::sections(ctx.prompt()).to_vec(); + let visible = h1_visible_sections(ctx.prompt()).to_vec(); let read = move |heading: String| read_visible(&heading, &visible); // H1's one privilege: `argv` installs writable, so the repair // pattern can assign it; the executor reads the value back at the diff --git a/crates/promptforge-internal/engine/src/execute/walk_target.rs b/crates/promptforge-internal/engine/src/execute/walk_target.rs index 7c7ca169f..b5d537ddb 100644 --- a/crates/promptforge-internal/engine/src/execute/walk_target.rs +++ b/crates/promptforge-internal/engine/src/execute/walk_target.rs @@ -12,7 +12,7 @@ use crate::heading_address; use crate::lua::SectionText; -use crate::parser::Section; +use crate::parser::{Prompt, Section}; use crate::{Error, Result}; /// The index of `target` in `slice`, matched on the parser-unique @@ -43,6 +43,12 @@ pub(super) fn visible_sections(home: &[Section], caller: &Section) -> Vec &[Section] { + promptforge_parser::detail::sections(prompt) +} + /// What a prompt read sees of the section `heading` names in a caller's /// visible set: its own raw Markdown and its pre-parsed list items, or /// `None` when no visible section has that heading. @@ -74,7 +80,7 @@ pub(super) enum JumpTarget { /// sibling within the jumper's own slice. /// /// Resolution is an exact `(level, name)` match (see -/// [`heading_address::resolve_sibling`]): two visible sections sharing an +/// [`heading_address::resolve_visible`]): two visible sections sharing an /// address error loudly as ambiguous instead of silently resolving to the /// first. /// @@ -87,7 +93,7 @@ pub(super) fn resolve_jump_target( jumper: &Section, ) -> Result { let visible = visible_sections(siblings, jumper); - let target = heading_address::resolve_sibling(heading, &visible)?; + let target = heading_address::resolve_visible(heading, &visible)?; if let Some(index) = section_position(jumper.children(), target) { return Ok(JumpTarget::Child(index)); } diff --git a/crates/promptforge-internal/engine/src/heading_address-tests.rs b/crates/promptforge-internal/engine/src/heading_address-tests.rs index 9a8b8587f..354c34981 100644 --- a/crates/promptforge-internal/engine/src/heading_address-tests.rs +++ b/crates/promptforge-internal/engine/src/heading_address-tests.rs @@ -1,25 +1,25 @@ -//! Tests for resolving a fanout worker heading among sibling sections. +//! Tests for heading lookup over a visible set. use super::*; #[test] -fn resolve_sibling_finds_exact_match() { +fn resolve_visible_finds_exact_match() { let sections = vec![sibling("Worker", 3), sibling("Topics", 3)]; - let found = resolve_sibling("### Worker", §ions).expect("must resolve"); + let found = resolve_visible("### Worker", §ions).expect("must resolve"); assert_eq!(found.name(), "Worker"); } #[test] -fn resolve_sibling_missing_heading_lists_available() { +fn resolve_visible_missing_heading_lists_available() { let sections = vec![sibling("Worker", 3)]; - let err = resolve_sibling("### Missing", §ions).expect_err("missing heading must error"); + let err = resolve_visible("### Missing", §ions).expect_err("missing heading must error"); assert!(err.to_string().contains("### Worker"), "error was: {err}"); } #[test] -fn resolve_sibling_bare_name_errors() { +fn resolve_visible_bare_name_errors() { let sections = vec![sibling("Worker", 3)]; - let err = resolve_sibling("Worker", §ions).expect_err("bare name without ### must error"); + let err = resolve_visible("Worker", §ions).expect_err("bare name without ### must error"); assert!(err.to_string().contains("### markers"), "error was: {err}"); } @@ -32,36 +32,36 @@ fn sibling(name: &str, level: u8) -> Section { } #[test] -fn resolve_sibling_requires_whitespace_after_markers() { +fn resolve_visible_requires_whitespace_after_markers() { let sections = vec![sibling("Worker", 3)]; - let err = resolve_sibling("###Worker", §ions) + let err = resolve_visible("###Worker", §ions) .expect_err("no whitespace after markers must error"); assert!(err.to_string().contains("whitespace"), "error was: {err}"); } #[test] -fn resolve_sibling_marker_only_heading_errors_as_nameless() { +fn resolve_visible_marker_only_heading_errors_as_nameless() { let sections = vec![sibling("Worker", 3)]; - let err = resolve_sibling("### ", §ions).expect_err("a marker-only heading must error"); + let err = resolve_visible("### ", §ions).expect_err("a marker-only heading must error"); assert!(err.to_string().contains("has no name"), "error was: {err}"); } #[test] -fn resolve_sibling_requires_exact_level() { +fn resolve_visible_requires_exact_level() { let sections = vec![sibling("Worker", 3)]; // Same name, wrong marker level, must not resolve. - let err = resolve_sibling("## Worker", §ions) + let err = resolve_visible("## Worker", §ions) .expect_err("a level mismatch must not resolve by name alone"); assert!(err.to_string().contains("not found"), "error was: {err}"); // The exact address resolves. - let ok = resolve_sibling("### Worker", §ions).expect("exact address resolves"); + let ok = resolve_visible("### Worker", §ions).expect("exact address resolves"); assert_eq!(ok.name(), "Worker"); } #[test] -fn resolve_sibling_rejects_more_than_one_match() { +fn resolve_visible_rejects_more_than_one_match() { let sections = vec![sibling("Worker", 3), sibling("Worker", 3)]; - let err = resolve_sibling("### Worker", §ions) + let err = resolve_visible("### Worker", §ions) .expect_err("two identical siblings must be rejected as ambiguous"); assert!(err.to_string().contains("ambiguous"), "error was: {err}"); } diff --git a/crates/promptforge-internal/engine/src/heading_address.rs b/crates/promptforge-internal/engine/src/heading_address.rs index 74a4462a5..b1b257c88 100644 --- a/crates/promptforge-internal/engine/src/heading_address.rs +++ b/crates/promptforge-internal/engine/src/heading_address.rs @@ -12,7 +12,7 @@ //! ordered after a sibling - and waits on the live set while the scheduler's //! admission limit bounds how many arms run at once. This module defines //! [`find_visible`], the one heading lookup those surfaces share, and -//! [`resolve_sibling`], its adapter for surfaces where a miss is an error; +//! [`resolve_visible`], its adapter for surfaces where a miss is an error; //! the collection enumeration sits in the `promptforge-lua` crate, beside the //! VM and the coroutine protocol that consume it. @@ -97,7 +97,7 @@ pub(crate) fn find_visible<'a>( /// visible section matches the exact address. The not-found message lists the /// visible sections and nothing else, so the error channel cannot leak the /// rest of the document's structure. -pub(crate) fn resolve_sibling<'a>(heading: &str, visible: &'a [Section]) -> Result<&'a Section> { +pub(crate) fn resolve_visible<'a>(heading: &str, visible: &'a [Section]) -> Result<&'a Section> { find_visible(heading, visible)?.ok_or_else(|| { let available: Vec = visible .iter() diff --git a/vibe/2026-10-10-3-prompt-reads-debt-removal.md b/vibe/2026-10-10-3-prompt-reads-debt-removal.md index e8df07531..d81dc089a 100644 --- a/vibe/2026-10-10-3-prompt-reads-debt-removal.md +++ b/vibe/2026-10-10-3-prompt-reads-debt-removal.md @@ -202,7 +202,7 @@ isProject: false -### Step 2: Give H1's visible set one helper and rename resolve_sibling to resolve_visible +### Step 2: Give H1's visible set one helper and rename resolve_sibling to resolve_visible [completed] - Component: Engine scope module - Placement: second, after Parser cleanup, because both edit `engine/src/heading_address-tests.rs` (the `sibling` helper there, then the renamed tests), so they run in sequence in one checkout. From 53f2aed67abc03576f2ad3b7f988937eb37ab0c0 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 15:57:36 -0700 Subject: [PATCH 07/34] Mark every finished record's open tasks completed Archived work records still listed most of their tasks as pending, and a few as in progress, so anything reading their frontmatter saw finished work as open. Every such status in those records now reads completed. The edit touches only frontmatter status lines and changes no other byte, so each file keeps its line count and line endings. The one record still underway keeps its open statuses. - `status: completed` is the single target state. It replaces 970 `status: pending` lines and 5 `status: in_progress` lines across 133 files. - `vibe/` changes outside the active record stay inside the YAML frontmatter. Every hunk ends before the closing `---`, and each file's size moves by exactly the length difference of the swapped words. - `vibe/2026-10-10-3-prompt-reads-debt-removal.md` keeps every `status: pending` in its frontmatter. Its only change is a `[completed]` marker on one heading. Plan: vibe/2026-10-10-3-prompt-reads-debt-removal.md --- .../2026-07-29-2-lua-args-substitution.md | 2 +- .../2026-07/2026-07-31-1-orchestrator-only.md | 18 ++++---- vibe/2026-08/2026-08-02-2-mcp-server.md | 28 ++++++------- .../2026-08-03-1-mcp-server-correction.md | 38 ++++++++--------- ...6-08-04-1-recover-core-design-rationale.md | 18 ++++---- ...26-08-08-11-briefer-evidence-thickening.md | 4 +- .../2026-08-08-3-gateway-local-inference.md | 4 +- .../2026-08-09-2-promptforge-user-guide.md | 42 +++++++++---------- .../2026-08-09-4-store-real-plus-virtual.md | 4 +- .../2026-08/2026-08-09-5-h1-once-no-replay.md | 4 +- vibe/2026-08/2026-08-10-1-crate-review.md | 10 ++--- .../2026-08-11-1-dokuman-all-crates.md | 2 +- .../2026-08-16-1-fanout-scope-refactor.md | 8 ++-- vibe/2026-08/2026-08-18-1-tools-local.md | 24 +++++------ ...2026-08-19-1-collapse-fanout-arm-engine.md | 14 +++---- vibe/2026-08/2026-08-22-1-runcontext-seed.md | 20 ++++----- .../2026-08/2026-08-23-1-dominion-refactor.md | 22 +++++----- .../2026-08-24-2-autogen-config-first-run.md | 14 +++---- .../2026-08-24-5-stop-recording-on-send.md | 2 +- .../2026-08-26-1-remove-voice-status-line.md | 32 +++++++------- .../2026-08-27-1-refresh-tree-on-drop.md | 8 ++-- .../2026-08-27-3-server-driven-menu-state.md | 42 +++++++++---------- .../2026-08-27-4-merged-gateway-workshop.md | 22 +++++----- ...026-08-28-2-coroutine-protocol-executor.md | 28 ++++++------- .../2026-08-28-4-cuda-llama-provisioning.md | 22 +++++----- ...2026-08-29-1-crate-extraction-execution.md | 18 ++++---- ...26-08-31-2-rebuild-mdbook-from-includes.md | 6 +-- ...26-08-31-6-tauri-migration-for-workshop.md | 14 +++---- .../2026-09-01-1-build-simplification.md | 30 ++++++------- ...026-09-02-1-workshop-agent-window-clone.md | 36 ++++++++-------- .../2026-09-02-6-product-user-guides.md | 20 ++++----- .../2026-09-04-1-async-boot-and-progress.md | 10 ++--- ...9-04-3-unlock-inference-during-switches.md | 8 ++-- .../2026-09-05-2-generic-realtime-stt.md | 16 +++---- ...026-09-08-1-remove-unsupported-ratchets.md | 8 ++-- .../2026-09-09-2-async-stt-boot-reset.md | 10 ++--- .../2026-09-10-1-unified-prompt-model.md | 20 ++++----- vibe/2026-09/2026-09-10-2-debt-fixes.md | 12 +++--- .../2026-09-11-1-rulebook-debt-tiers.md | 18 ++++---- vibe/2026-09/2026-09-11-2-unify-toolchain.md | 12 +++--- vibe/2026-09/2026-09-11-3-vfs-foundation.md | 22 +++++----- .../2026-09-12-2-dependency-rules-vfs-hook.md | 14 +++---- ...6-09-12-3-workshop-server-decomposition.md | 26 ++++++------ .../2026-09-12-4-workshop-debt-removal.md | 10 ++--- .../2026-09-12-5-one-door-promptforge-api.md | 18 ++++---- ...2026-09-13-1-capabilities-global-naming.md | 14 +++---- ...-1-debt-removal-capabilities-follow-ups.md | 8 ++-- .../2026-09-14-2-provider-model-sheets.md | 8 ++-- ...2026-09-14-3-fix-previous-sheet-swallow.md | 2 +- ...26-09-14-4-provider-expansion-local-run.md | 18 ++++---- ...026-09-14-5-models-aggregation-workflow.md | 10 ++--- .../2026-09-14-6-model-taxonomy-cloud-ui.md | 20 ++++----- .../2026-09-15-1-cloud-sheet-debt-removal.md | 8 ++-- .../2026-09-15-2-profiles-gate-local.md | 28 ++++++------- .../2026-09-15-3-profiles-debt-removal.md | 8 ++-- vibe/2026-09/2026-09-15-4-flat-sources.md | 6 +-- .../2026-09-15-5-workshop-menu-overhaul.md | 18 ++++---- .../2026-09-16-1-turso-workspace-files.md | 20 ++++----- .../2026-09/2026-09-16-2-workshop-ui-state.md | 12 +++--- .../2026-09-16-3-workspace-debt-removal.md | 12 +++--- .../2026-09-17-1-promptforge-family-reorg.md | 2 +- .../2026-09-17-2-gateway-family-reorg.md | 2 +- .../2026-09-17-3-gateway-app-decomp.md | 14 +++---- .../2026-09-17-4-ci-repair-state-accessors.md | 14 +++---- .../2026-09-17-5-workshop-family-reorg.md | 12 +++--- .../2026-09-17-6-minimal-run-window.md | 16 +++---- .../2026-09-18-1-single-rustls-backend.md | 22 +++++----- .../2026-09-18-2-remove-tool-picker.md | 8 ++-- .../2026-09/2026-09-18-3-empty-zone-groups.md | 4 +- .../2026-09-18-4-sans-io-engine-harness.md | 18 ++++---- .../2026-09-19-1-chatbox-extraction.md | 10 ++--- .../2026-09-20-1-chatbox-debt-removal.md | 4 +- .../2026-09-20-1-harness-debt-removal.md | 4 +- ...2026-09-20-2-gateway-api-types-progress.md | 12 +++--- .../2026-09-20-2-rust-rulebook-sweep.md | 26 ++++++------ vibe/2026-09/2026-09-20-4-debt-removal.md | 14 +++---- .../2026-09-22-1-replayed-tool-calls.md | 12 +++--- vibe/2026-09/2026-09-22-2-emit-infer-reply.md | 14 +++---- .../2026-09-22-3-replay-json-fidelity.md | 16 +++---- vibe/2026-09/2026-09-22-4-api-runtime-debt.md | 22 +++++----- .../2026-09-23-1-promptforge-api-firewall.md | 26 ++++++------ .../2026-09-24-1-debt-removal-api-firewall.md | 8 ++-- vibe/2026-09/2026-09-24-2-issues-69-59-70.md | 6 +-- .../2026-09-24-2-workshop-crates-cleanup.md | 34 +++++++-------- .../2026-09-25-1-vibe2-debt-removal.md | 10 ++--- .../2026-09-25-2-multi-product-docs-site.md | 10 ++--- vibe/2026-09/2026-09-25-2-workshop-fixes.md | 30 ++++++------- .../2026-09-25-3-workshop-structure.md | 42 +++++++++---------- vibe/2026-09/2026-09-26-2-harness-facade.md | 16 +++---- .../2026-09-26-2-local-tools-coroutine.md | 12 +++--- .../2026-09-26-3-harness-dependency-rule.md | 4 +- vibe/2026-09/2026-09-26-4-store-into-vfs.md | 28 ++++++------- .../2026-09-27-1-fork-shared-ui-look.md | 14 +++---- vibe/2026-09/2026-09-27-1-store-vfs-debt.md | 18 ++++---- vibe/2026-09/2026-09-27-2-look-fork-debts.md | 4 +- .../2026-09-27-3-platform-extraction.md | 8 ++-- .../2026-09-28-1-platform-extraction-debts.md | 6 +-- ...026-09-28-2-capability-harness-redesign.md | 30 ++++++------- .../2026-09-28-3-capability-harness-debts.md | 8 ++-- ...-09-28-4-internal-crates-critical-fixes.md | 18 ++++---- .../2026-09-28-5-internal-crates-fix-batch.md | 42 +++++++++---------- ...09-30-1-engine-harness-host-terminology.md | 22 +++++----- vibe/2026-10-01-1-remove-task-event-reads.md | 6 +-- vibe/2026-10-01-2-store-to-vfs-inline.md | 4 +- vibe/2026-10-02-1-gateway-client.md | 12 +++--- vibe/2026-10-02-2-host-services.md | 12 +++--- vibe/2026-10-02-3-debt-removal.md | 8 ++-- vibe/2026-10-02-4-inference-broker.md | 10 ++--- vibe/2026-10-03-1-harness-per-run.md | 16 +++---- vibe/2026-10-03-2-debt-removal-6-9.md | 12 +++--- vibe/2026-10-03-3-agent-context-cleanup.md | 26 ++++++------ vibe/2026-10-04-1-tool-context.md | 16 +++---- vibe/2026-10-04-2-plugin-rename.md | 6 +-- vibe/2026-10-05-1-debt-removal.md | 8 ++-- .../2026-10-05-1-whisper-cuda-backend-debt.md | 2 +- vibe/2026-10-06-1-stt-two-model-upgrade.md | 24 +++++------ vibe/2026-10-07-1-plugin-api-minimal.md | 14 +++---- vibe/2026-10-07-1-silero-speech-detection.md | 20 ++++----- vibe/2026-10-07-2-plugin-api-debt-removal.md | 8 ++-- vibe/2026-10-07-2-stt-debt-removal.md | 18 ++++---- vibe/2026-10-08-1-mcp-client-plugin.md | 12 +++--- vibe/2026-10-08-2-copy-cursor-everywhere.md | 20 ++++----- vibe/2026-10-08-2-stt-report-fixes.md | 14 +++---- vibe/2026-10-08-3-debt-removal-mcp-cursor.md | 6 +-- vibe/2026-10-08-3-stt-fix-debt.md | 6 +-- vibe/2026-10-08-4-mcp-plugin-debt-removal.md | 10 ++--- vibe/2026-10-09-1-finish-search-removal.md | 4 +- ...2026-10-09-2-messages-userdata-refactor.md | 12 +++--- vibe/2026-10-09-3-models-loop-in-rust.md | 14 +++---- vibe/2026-10-09-4-models-loop-debt-removal.md | 8 ++-- vibe/2026-10-09-5-author-api-reshape.md | 10 ++--- vibe/2026-10-10-1-author-api-debt-removal.md | 8 ++-- vibe/2026-10-10-2-prompt-namespace-reads.md | 10 ++--- .../2026-10-10-3-prompt-reads-debt-removal.md | 2 +- 134 files changed, 976 insertions(+), 976 deletions(-) diff --git a/vibe/2026-07/2026-07-29-2-lua-args-substitution.md b/vibe/2026-07/2026-07-29-2-lua-args-substitution.md index 54e7b5169..9b3084c25 100644 --- a/vibe/2026-07/2026-07-29-2-lua-args-substitution.md +++ b/vibe/2026-07/2026-07-29-2-lua-args-substitution.md @@ -4,7 +4,7 @@ overview: "Add {{ }} substitution over args, var, and sys, plus the writable var todos: - id: subst-sys-var content: "subst.rs: resolve {{ args }}, {{ var.x }}, {{ sys.x }} in prose (scalar->string, table->JSON, missing->error, single pass). var writable table. sys namespace (when/now/id) populated by the runtime. Wire: run Lua (sets var) -> substitute -> model turn. prompts/greet.md demo + unit tests + e2e. Commit." - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-07/2026-07-31-1-orchestrator-only.md b/vibe/2026-07/2026-07-31-1-orchestrator-only.md index 6b71628cc..853f3ce1c 100644 --- a/vibe/2026-07/2026-07-31-1-orchestrator-only.md +++ b/vibe/2026-07/2026-07-31-1-orchestrator-only.md @@ -4,31 +4,31 @@ overview: Markdown-driven pipeline runtime (Rust + mlua). A prompt is a function todos: - id: mark-docs-provisional content: Add provisional status to design document headers. This plan overrides them. - status: pending + status: completed - id: fix-reachability content: Sections can be dormant, cyclic, freely organized. Unreachable is not an error. - status: pending + status: completed - id: build-version-gate content: "Rung 1: detection function for the `promptforge:` version; run only promptforge prompts (absent = not ours, caller handles), refuse an unsupported major." - status: pending + status: completed - id: build-store-vfs content: "Rung 2: run-scoped store/VFS in core (write/append/read-numbered/str_replace/delete/glob), Lua + model equal access, capability injection (MemVfs vs RealFs). Files-first." - status: pending + status: completed - id: build-substitution content: "Rung 3: one-pass {{ }} resolver over args, sys (now/date/elapsed/id), var, reply, result (scalars stringify, tables -> JSON, nil hard error, no formulas). Parameterized single-step prompts." - status: pending + status: completed - id: build-control-flow content: "Rung 4: fall-through + goto + pass + task + reference blocks (sect.block/sect.list) + result/injection + the three budgets (nesting/step/tool). Deterministic Lua transitions; cycles legal." - status: pending + status: completed - id: build-fanout content: "Rung 5: fanout (homogeneous over a list, heterogeneous over tasks) + keyed-store reduce + structured store. Runs Diligence/Briefer." - status: pending + status: completed - id: build-cyclic content: "Rung 6: cyclic state-machine pipelines (mentograph-shaped) driven by Lua conditionals. Model-surface transfer stays deferred." - status: pending + status: completed - id: generate-design-doc content: "Final step, after all rungs: spawn one subagent to read this plan, grep , and write design-promptforge-orchestrator.md from the finished build; it supersedes the provisional promptforge-design docs." - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-02-2-mcp-server.md b/vibe/2026-08/2026-08-02-2-mcp-server.md index b936f2dd3..feb494127 100644 --- a/vibe/2026-08/2026-08-02-2-mcp-server.md +++ b/vibe/2026-08/2026-08-02-2-mcp-server.md @@ -4,46 +4,46 @@ overview: Build `promptforge-mcp`, a small MCP server that publishes PromptForge todos: - id: step-1 content: "Step 1: promptforge-core observe module - Observer, Event, NullObserver" - status: pending + status: completed - id: step-2 content: "Step 2: RunOptions and event emission in execute::run; CLI call site updated" - status: pending + status: completed - id: step-3 content: "Step 3: promptforge-mcp crate skeleton and prompts.toml parsing" - status: pending + status: completed - id: step-4 content: "Step 4: catalog and the boot validation pass" - status: pending + status: completed - id: step-5 content: "Step 5: tool definitions - direct prompts plus the built-in trio" - status: pending + status: completed - id: step-6 content: "Step 6: call_tool - run a prompt, build RunResult, map errors" - status: pending + status: completed - id: step-7 content: "Step 7: McpObserver and the progress pump" - status: pending + status: completed - id: step-8 content: "Step 8: run registry, admission, and deferred collect" - status: pending + status: completed - id: step-9 content: "Step 9: the router - streamable HTTP, bearer auth, healthz, stdio" - status: pending + status: completed - id: step-10 content: "Step 10: the watcher - debounce, per-prompt revalidation, swap, list_changed" - status: pending + status: completed - id: step-11 content: "Step 11: picker rebuild that reuses the loaded embedder" - status: pending + status: completed - id: step-12 content: "Step 12: need_prompt - descriptors, shortlist, feature gate, rebuild on save" - status: pending + status: completed - id: step-13 content: "Step 13: shipped example config, STATUS.md, README.md" - status: pending + status: completed - id: step-14 content: "Step 14: generate the design document from the finished work" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-03-1-mcp-server-correction.md b/vibe/2026-08/2026-08-03-1-mcp-server-correction.md index 31dde80e1..cf0dad126 100644 --- a/vibe/2026-08/2026-08-03-1-mcp-server-correction.md +++ b/vibe/2026-08/2026-08-03-1-mcp-server-correction.md @@ -4,61 +4,61 @@ overview: Correct the MCP server from a surface that competes for a model's tool todos: - id: step-1 content: "Step 1: publish only the four built-ins; delete Expose and is_direct; rewrite descriptions and instructions in the command register" - status: pending + status: completed - id: step-2 content: "Step 2: delete the tools/list_changed machinery - Sessions, ListChanged, published_changed" - status: pending + status: completed - id: step-3 content: "Step 3: remove expose and default_expose from the configuration and the shipped prompts.toml" - status: pending + status: completed - id: step-4 content: "Step 4: rename the crate, binary, library, and design document to promptforge-mcp-server" - status: pending + status: completed - id: step-5 content: "Step 5: correct the stale doc comments in promptforge-core and promptforge-gateway" - status: pending + status: completed - id: step-6 content: "Step 6: write the do-more-with-less principle into AGENTS.md and design.md" - status: pending + status: completed - id: step-7 content: "Step 7: reconcile design.md, design-promptforge.md and notes.md with explicit invocation" - status: pending + status: completed - id: step-8 content: "Step 8: refactor design-cli.md in place into built and unbuilt parts" - status: pending + status: completed - id: step-9 content: "Step 9: transfer the built part to crates/promptforge-cli/design-cli.md" - status: pending + status: completed - id: step-10 content: "Step 10: finish the residue as design-cli-residue.md in the design repo" - status: pending + status: completed - id: step-11 content: "Step 11: refactor design-gateway.md in place into built and unbuilt parts" - status: pending + status: completed - id: step-12 content: "Step 12: transfer the built part to crates/promptforge-gateway/design-gateway.md" - status: pending + status: completed - id: step-13 content: "Step 13: finish the residue as design-gateway-residue.md in the design repo" - status: pending + status: completed - id: step-14 content: "Step 14: refactor design-core.md in place into built and unbuilt parts" - status: pending + status: completed - id: step-15 content: "Step 15: transfer the built part to crates/promptforge-core/design-core.md" - status: pending + status: completed - id: step-16 content: "Step 16: finish the residue as design-core-residue.md in the design repo" - status: pending + status: completed - id: step-17 content: "Step 17: rename design-mcp.md to design-mcp-server.md and refactor it in place" - status: pending + status: completed - id: step-18 content: "Step 18: merge the built part into the crate's design-mcp-server.md" - status: pending + status: completed - id: step-19 content: "Step 19: rewrite the crate document from finished work; residue becomes design-mcp-server-residue.md" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-04-1-recover-core-design-rationale.md b/vibe/2026-08/2026-08-04-1-recover-core-design-rationale.md index 18ae06f54..234153ee2 100644 --- a/vibe/2026-08/2026-08-04-1-recover-core-design-rationale.md +++ b/vibe/2026-08/2026-08-04-1-recover-core-design-rationale.md @@ -4,31 +4,31 @@ overview: Recover the design rationale of `promptforge-core` from the code alone todos: - id: step-1 content: "Step 1: create the ledger with the method and the two worked example records" - status: pending + status: completed - id: step-2 content: "Step 2: pass 1 extraction - parser.rs, lib.rs, error.rs" - status: pending + status: completed - id: step-3 content: "Step 3: pass 1 extraction - execute.rs, execute/tests.rs, observe.rs" - status: pending + status: completed - id: step-4 content: "Step 4: pass 1 extraction - lua.rs, subst.rs, store.rs" - status: pending + status: completed - id: step-5 content: "Step 5: pass 1 extraction - client.rs, tools.rs, tools/web_search.rs" - status: pending + status: completed - id: step-6 content: "Step 6: merge, deduplicate, fill reach from call sites, rank" - status: pending + status: completed - id: step-7 content: "Step 7: pass 2 - propose collapses from the archive as measurement, with provenance and dates" - status: pending + status: completed - id: step-8 content: "Step 8: write design-core-recovered.md as a design document organized by the system, not a ledger transcript" - status: pending + status: completed - id: step-9 content: "Step 9: report the recovery ratio - code forced, archive could settle, still open" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-08-11-briefer-evidence-thickening.md b/vibe/2026-08/2026-08-08-11-briefer-evidence-thickening.md index dc84e4d98..931542929 100644 --- a/vibe/2026-08/2026-08-08-11-briefer-evidence-thickening.md +++ b/vibe/2026-08/2026-08-08-11-briefer-evidence-thickening.md @@ -25,10 +25,10 @@ todos: status: completed - id: source-log-late content: "Late try: store.append log.md Source Log; keep only if clean" - status: pending + status: completed - id: commit-steps content: Git commit each kept briefer.md quality step-up (fold later) - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-08-3-gateway-local-inference.md b/vibe/2026-08/2026-08-08-3-gateway-local-inference.md index 230687041..3a42b641f 100644 --- a/vibe/2026-08/2026-08-08-3-gateway-local-inference.md +++ b/vibe/2026-08/2026-08-08-3-gateway-local-inference.md @@ -13,10 +13,10 @@ todos: status: completed - id: local-completions content: Serve local llama.cpp inference through /v1/chat/completions endpoint - status: pending + status: completed - id: model-download content: HTTP download from source URL with cache management in ~/.promptforge/models/ - status: pending + status: completed - id: candle-gpu content: Extend Candle from CPU-only to CUDA/Metal for utility models (classifiers, embedders) status: cancelled diff --git a/vibe/2026-08/2026-08-09-2-promptforge-user-guide.md b/vibe/2026-08/2026-08-09-2-promptforge-user-guide.md index cb8b0d626..e258d1ed2 100644 --- a/vibe/2026-08/2026-08-09-2-promptforge-user-guide.md +++ b/vibe/2026-08/2026-08-09-2-promptforge-user-guide.md @@ -4,67 +4,67 @@ overview: Write a progressive tutorial for PromptForge prompt authoring, startin todos: - id: s01 content: "Section 1: What is a prompt file (skeleton + Hello World)" - status: pending + status: completed - id: s02 content: "Section 2: The model turn (prose in, text out)" - status: pending + status: completed - id: s03 content: "Section 3: Your first Lua block (the prologue)" - status: pending + status: completed - id: s04 content: "Section 4: The preamble (H1 shared Lua)" - status: pending + status: completed - id: s05 content: "Section 5: The epilog" - status: pending + status: completed - id: s06 content: "Section 6: Multiple sections" - status: pending + status: completed - id: s07 content: "Section 7: The store" - status: pending + status: completed - id: s08 content: "Section 8: Template substitution" - status: pending + status: completed - id: s09 content: "Section 9: Tools (search and fetch)" - status: pending + status: completed - id: s10 content: "Section 10: The Tool object" - status: pending + status: completed - id: s11 content: "Section 11: The Model object" - status: pending + status: completed - id: s12 content: "Section 12: Explicit inference with model:infer()" - status: pending + status: completed - id: s13 content: "Section 13: Alternating blocks" - status: pending + status: completed - id: s14 content: "Section 14: Composable tool sets" - status: pending + status: completed - id: s15 content: "Section 15: Sections as subroutines: execute()" - status: pending + status: completed - id: s16 content: "Section 16: Control flow: goto()" - status: pending + status: completed - id: s17 content: "Section 17: Fanout (parallel execution)" - status: pending + status: completed - id: s18 content: "Section 18: The sys table" - status: pending + status: completed - id: s19 content: "Section 19: Error handling and validation" - status: pending + status: completed - id: s20 content: "Section 20: Capstone prompt (everything together)" - status: pending + status: completed - id: s21 content: "Section 21: API Reference (all objects, functions, globals)" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-09-4-store-real-plus-virtual.md b/vibe/2026-08/2026-08-09-4-store-real-plus-virtual.md index 7c03d6d81..ef1d44974 100644 --- a/vibe/2026-08/2026-08-09-4-store-real-plus-virtual.md +++ b/vibe/2026-08/2026-08-09-4-store-real-plus-virtual.md @@ -4,10 +4,10 @@ overview: "First-class Tool/Model objects, model:infer(), alternating blocks, ex todos: - id: doc-status-orig-claim content: "Remove stale design-core-orig.md claim from STATUS.md (file was deleted/archived outside repo)" - status: pending + status: completed - id: doc-readme-or-defer content: "Either add first-class API summary to root README or add a note deferring to user-guide" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-09-5-h1-once-no-replay.md b/vibe/2026-08/2026-08-09-5-h1-once-no-replay.md index 9c83f8621..1f416d0d6 100644 --- a/vibe/2026-08/2026-08-09-5-h1-once-no-replay.md +++ b/vibe/2026-08/2026-08-09-5-h1-once-no-replay.md @@ -4,10 +4,10 @@ overview: "Runtime refactor shipped: bind phase eliminated, H1 runs once with li todos: - id: doc-quick-reference content: "Add Quick Reference rules block to the user guide (before final image)" - status: pending + status: completed - id: doc-status-orig-claim content: "Remove stale design-core-orig.md claim from STATUS.md (file was deleted)" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-10-1-crate-review.md b/vibe/2026-08/2026-08-10-1-crate-review.md index 0f756d8fb..77458daa9 100644 --- a/vibe/2026-08/2026-08-10-1-crate-review.md +++ b/vibe/2026-08/2026-08-10-1-crate-review.md @@ -19,19 +19,19 @@ todos: status: completed - id: review-gateway content: Complete findings, API synthesis, fix-to-green, and verification for promptforge-gateway - status: pending + status: completed - id: review-webfetch content: Complete findings, API synthesis, fix-to-green, and verification for promptforge-webfetch - status: pending + status: completed - id: review-cli content: Complete findings, API synthesis, fix-to-green, and verification for promptforge-cli - status: pending + status: completed - id: review-dev content: Complete findings, API synthesis, fix-to-green, and verification for promptforge-dev - status: pending + status: completed - id: review-mcp-server content: Complete findings, API synthesis, fix-to-green, and verification for promptforge-mcp-server - status: pending + status: completed - id: workspace-closure content: Run final full-workspace gates and reconcile the completed cross-crate API status: completed diff --git a/vibe/2026-08/2026-08-11-1-dokuman-all-crates.md b/vibe/2026-08/2026-08-11-1-dokuman-all-crates.md index 05e910304..f123e4c9d 100644 --- a/vibe/2026-08/2026-08-11-1-dokuman-all-crates.md +++ b/vibe/2026-08/2026-08-11-1-dokuman-all-crates.md @@ -25,7 +25,7 @@ todos: status: completed - id: crate-dev content: Run Dokuman on promptforge-dev -> user-guide-promptforge-dev.md - status: in_progress + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-16-1-fanout-scope-refactor.md b/vibe/2026-08/2026-08-16-1-fanout-scope-refactor.md index 4bf9d81a2..a699b4ea3 100644 --- a/vibe/2026-08/2026-08-16-1-fanout-scope-refactor.md +++ b/vibe/2026-08/2026-08-16-1-fanout-scope-refactor.md @@ -4,16 +4,16 @@ overview: "fanout() accepts a Lua array as second arg + new items() function. Sc todos: - id: fanout-array content: fanout() accepts Lua array as second arg, dispatch on Value type in Lua binding - status: pending + status: completed - id: items-fn content: Add items(heading) Lua function that returns a section's pre-parsed bullet items - status: pending + status: completed - id: fanout-tests content: Tests for fanout with array and items() - status: pending + status: completed - id: fanout-docs content: Update guide/src/fanout.md and design-core.md item 20 with array syntax and items() documentation - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-18-1-tools-local.md b/vibe/2026-08/2026-08-18-1-tools-local.md index 6ec1fb22f..192090edc 100644 --- a/vibe/2026-08/2026-08-18-1-tools-local.md +++ b/vibe/2026-08/2026-08-18-1-tools-local.md @@ -4,40 +4,40 @@ overview: "Add tools.local(alias, description, params, handler) - Lua-backed too todos: - id: install-log-store content: Install log and store once per section via vm.install_host_apis() - status: pending + status: completed - id: install-control content: Install control globals once per section via vm.install_control_globals() - status: pending + status: completed - id: delete-prologue-epilog content: Delete prologue/epilog split, one run_loaded_with_control for every chunk - status: pending + status: completed - id: execute-nil-reply content: execute() passes nil reply to subroutines - status: pending + status: completed - id: models-default content: Rename models.only to models.default, remove models.use foreclosure - status: pending + status: completed - id: local-storage content: Add LocalTools storage to SectionVm - status: pending + status: completed - id: tools-local-fn content: Install tools.local Lua function with params table to JSON schema - status: pending + status: completed - id: schema-rebuild content: Rebuild schemas per prose block, delete ToolScope/snapshot_tool_scope - status: pending + status: completed - id: dispatch-local content: Dispatch local tools in tool loop with sentinel ToolId - status: pending + status: completed - id: models-get-infer content: Add models.get() and models.infer(), keep handle:infer() - status: pending + status: completed - id: tests content: Tests for all new behavior - status: pending + status: completed - id: docs content: Documentation for all new APIs - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-19-1-collapse-fanout-arm-engine.md b/vibe/2026-08/2026-08-19-1-collapse-fanout-arm-engine.md index cd976ad3b..1b51ada6c 100644 --- a/vibe/2026-08/2026-08-19-1-collapse-fanout-arm-engine.md +++ b/vibe/2026-08/2026-08-19-1-collapse-fanout-arm-engine.md @@ -4,25 +4,25 @@ overview: Collapse the duplication between run_one_arm (fanout/arm.rs) and run_o todos: - id: step-1-vm-lifecycle content: "Step 1: carve the VM-lifecycle sequence out of run_one_section (behavior-identical)" - status: pending + status: completed - id: step-2-block-walk content: "Step 2: carve the block walk out of run_one_section (returns SectionFlow), Verify" - status: pending + status: completed - id: step-3-arm-adapter content: "Step 3: the arm becomes an adapter over the one engine (stubs kept), Verify" - status: pending + status: completed - id: step-4-arm-capabilities content: "Step 4: arms gain full control globals and jump (decision 3)" - status: pending + status: completed - id: step-5-cap-removal content: "Step 5: remove the fanout item cap (keep the concurrency bound)" - status: pending + status: completed - id: step-6-consolidation content: "Step 6: consolidation - delete surviving duplication, check against the delta list, Verify" - status: pending + status: completed - id: step-7-design-doc content: "Step 7: generate design-promptforge-arm-engine-collapse.md via the design-doc block, Verify" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-22-1-runcontext-seed.md b/vibe/2026-08/2026-08-22-1-runcontext-seed.md index 259b1cb58..88c199673 100644 --- a/vibe/2026-08/2026-08-22-1-runcontext-seed.md +++ b/vibe/2026-08/2026-08-22-1-runcontext-seed.md @@ -4,34 +4,34 @@ overview: Introduce RunContext (starting with Arc plus a per-run GuardNo todos: - id: context-module content: "Create execute/context.rs with RunContext { prompt: Arc } (prompt ONLY - nonce lands in phase 2 with its readers), constructor, accessors, unit tests" - status: pending + status: completed - id: wire-module content: Register mod context in execute.rs and re-export RunContext; update module-layout doc comment - status: pending + status: completed - id: thread-run content: "Construct RunContext in run (pass &ctx into run_body - prompt.title used after); replace prompt: &Prompt with ctx in execute_live_h1 and run_sections; rename RunFrame ctx bindings to frame in engine.rs" - status: pending + status: completed - id: nonce-signature content: "untrusted.rs: GuardNonce pub(crate) + Clone, fresh() pub(crate) (mlua capture needs owned clone); change wrap to wrap(&GuardNonce, &str); update module docs. NOTE: atomic with the two threading todos - the wrap signature change breaks both callers at once" - status: pending + status: completed - id: nonce-thread-tool-loop content: "Thread the nonce RunContext -> RunFrame -> from_run_fields (covers from_walk AND from_fanout) -> ControlContext -> walk_context + FanoutContext -> SectionProgress -> tool_loop wrap site; fix 6 test constructors of SectionProgress" - status: pending + status: completed - id: nonce-thread-lua content: "Pass &GuardNonce into SectionVm::new (new_for_section delegates) so install_untrusted captures an owned clone; fix all construction sites: h1.rs:55, engine.rs:345, arm.rs:214, vm.rs:1037 test helper, execute/tests 897/969, model/tests:98, 25 lua/tests.rs sites via a new test helper" - status: pending + status: completed - id: nonce-tests content: "Invert the three freshness tests to stability tests; add cross-run distinctness test" - status: pending + status: completed - id: h1-stubs content: "Install clear-error stubs for execute/jump/fanout/list_from_section in the H1 VM setup (one commit with h1-stub-tests; independent of phases 1-2 but shares h1.rs/vm.rs - land after them to avoid churn)" - status: pending + status: completed - id: h1-stub-tests content: "Tests: each control global called from H1 fails with the clear message" - status: pending + status: completed - id: verify content: "cargo build / test / clippy green after EACH commit (vibe-rulebook: scheduled verify gates the next step); full suite at the end" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-23-1-dominion-refactor.md b/vibe/2026-08/2026-08-23-1-dominion-refactor.md index 2ad827039..db0c762ea 100644 --- a/vibe/2026-08/2026-08-23-1-dominion-refactor.md +++ b/vibe/2026-08/2026-08-23-1-dominion-refactor.md @@ -46,37 +46,37 @@ todos: status: completed - id: step-13-model-kinds content: "E1: ModelKind (chat/embedding/classifier) on config + kind-scoped validation; ModelInfo carries kind" - status: pending + status: completed - id: step-14-embeddings-route content: "E2a: POST /v1/embeddings wire types + route + remote passthrough through dominion queues" - status: pending + status: completed - id: step-15-local-embeddings content: "E2b: local kind=embedding launches llama-server --embeddings" - status: pending + status: completed - id: step-16-rerank content: "E3: POST /v1/rerank route + local --reranking launch + remote passthrough of the rerank shape" - status: pending + status: completed - id: n1-catalog-enrichment content: "Zed-derived catalog fields: max_output, default_temperature, images, parallel_tool_calls, effort_levels/default_effort, adaptive_thinking; validation" - status: pending + status: completed - id: n2-dialect-config content: "Move tool_dialect/tools_mode to gateway-internal per-model translation config; drop from catalog response" - status: pending + status: completed - id: n3-request-normalization content: "Outbound translation: effort mapping, emulated tool-schema injection, param filtering; Anthropic upstream step series" - status: pending + status: completed - id: n4-response-normalization content: "Inbound translation: emulated tool-call parsing, thinking normalization, finish reasons, Anthropic responses" - status: pending + status: completed - id: n5-core-dialect-removal content: "Delete promptforge-core client-side dialect machinery; integration suite proves zero-dialects" - status: pending + status: completed - id: step-17-streaming-relay content: "S1a: stream:true SSE relay - ChatChunk wire types, Upstream::stream, permit held for stream lifetime" - status: pending + status: completed - id: step-18-streaming-lifecycle content: "S1b: client-disconnect cancels upstream stream; per-chunk minimal shape validation" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-24-2-autogen-config-first-run.md b/vibe/2026-08/2026-08-24-2-autogen-config-first-run.md index 8ff3af8a7..d7e042aaf 100644 --- a/vibe/2026-08/2026-08-24-2-autogen-config-first-run.md +++ b/vibe/2026-08/2026-08-24-2-autogen-config-first-run.md @@ -4,25 +4,25 @@ overview: Config generation, gateway cache API, status bar with observer and act todos: - id: auto-config content: Auto-generate ~/.promptforge/workbench.toml on first run, remove from_env path - status: pending + status: completed - id: gateway-cache-api content: "Gateway: POST/GET/DELETE /v1/cache with SSE download progress" - status: pending + status: completed - id: observer content: Observer broadcast channel in AppState, status on main WS, instrument all subsystems - status: pending + status: completed - id: status-bar content: "Status bar UI: text left, progress bar right, activity LED, custom scrollbar" - status: pending + status: completed - id: css-skinning content: Consolidate all visuals into CSS custom properties, fix scrollbar, skinnable - status: pending + status: completed - id: gateway-resilience content: Background health heartbeat, reconnect, graceful degradation - status: pending + status: completed - id: wb-cache-integration content: Workbench calls POST /v1/cache at startup for whisper models, pipes progress to status bar - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-24-5-stop-recording-on-send.md b/vibe/2026-08/2026-08-24-5-stop-recording-on-send.md index 1e8b5aad3..f3ba05fe7 100644 --- a/vibe/2026-08/2026-08-24-5-stop-recording-on-send.md +++ b/vibe/2026-08/2026-08-24-5-stop-recording-on-send.md @@ -4,7 +4,7 @@ overview: When the user sends a chat message while voice recording is active, st todos: - id: discard-on-send content: "Single commit: onUserSubmit hook discards active recording, suppresses late frames, closes socket without stop; typing-model splice at cursor with readOnly during takes" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-26-1-remove-voice-status-line.md b/vibe/2026-08/2026-08-26-1-remove-voice-status-line.md index de134c943..ec6d808ea 100644 --- a/vibe/2026-08/2026-08-26-1-remove-voice-status-line.md +++ b/vibe/2026-08/2026-08-26-1-remove-voice-status-line.md @@ -4,52 +4,52 @@ overview: Remove the layout-shifting voice-status line, upgrade the agent window todos: - id: step-01 content: Remove the voice-status line; route voice errors to StatusBar.showLocal - status: pending + status: completed - id: step-02 content: Open chat links externally (target/rel in sanitizer, shell navigation intercept) - status: pending + status: completed - id: step-03 content: Block-memoized streaming markdown with unterminated-construct repair - status: pending + status: completed - id: step-04 content: Three-state thinking block (Planning next moves / Thinking, 4-line preview) - status: pending + status: completed - id: step-05 content: Tool activity block (one-line autoscroll collapsed, full log expanded) - status: pending + status: completed - id: step-06 content: Title-bar markup and window-chrome.ts browser-side wiring - status: pending + status: completed - id: step-07 content: Accessible File, Edit, Window, Help menus - status: pending + status: completed - id: step-08 content: Title-bar and menu CSS in the :root skin system - status: pending + status: completed - id: step-09 content: Decorationless Windows shell with typed wry IPC commands - status: pending + status: completed - id: step-10 content: Install the cold medallion as the program icon - status: pending + status: completed - id: step-11 content: Confined workspace list/read/write APIs in workspace.rs - status: pending + status: completed - id: step-12 content: Native Windows path drops through the shell - status: pending + status: completed - id: step-13 content: Zone registry, panel-type registry, and Workshop file-tree panel - status: pending + status: completed - id: step-14 content: EditorSurface interface and CodeMirror 6 editor panel - status: pending + status: completed - id: step-15 content: Layout boot, lock, persistence, and keyboard shortcuts - status: pending + status: completed - id: verify content: Final full verification (build, tests, manual Windows pass) - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-27-1-refresh-tree-on-drop.md b/vibe/2026-08/2026-08-27-1-refresh-tree-on-drop.md index ad5494f47..1e33ef035 100644 --- a/vibe/2026-08/2026-08-27-1-refresh-tree-on-drop.md +++ b/vibe/2026-08/2026-08-27-1-refresh-tree-on-drop.md @@ -4,16 +4,16 @@ overview: "Fix folder drops doing nothing: the grant succeeds but the Workshop t todos: - id: grant-event content: Dispatch promptforge:workspace-granted from workspace-drops.ts after successful grants - status: pending + status: completed - id: tree-refresh content: "Listen in WorkshopTreePanel: invalidate roots cache, re-render, remove empty hint, dispose listener" - status: pending + status: completed - id: tests content: Cover grant event dispatch and tree refresh in UI tests - status: pending + status: completed - id: verify content: npm test + typecheck, then manual drop test in the desktop app - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-27-3-server-driven-menu-state.md b/vibe/2026-08/2026-08-27-3-server-driven-menu-state.md index 36679fed7..2f39d3956 100644 --- a/vibe/2026-08/2026-08-27-3-server-driven-menu-state.md +++ b/vibe/2026-08/2026-08-27-3-server-driven-menu-state.md @@ -4,67 +4,67 @@ overview: "Make the Model menu fully server-driven: the server owns profiles, ac todos: - id: led-stuck-fix content: Fix LED stuck amber when error frame follows expired pulse - status: pending + status: completed - id: statusbar-group content: REC and LED grouped, hidden together behind progress bar - status: pending + status: completed - id: ci-no-cuda content: check-workshop CI builds without the default cuda feature - status: pending + status: completed - id: editor-save-baseline content: Save baseline is the written text, not the post-write editor text - status: pending + status: completed - id: socket-reasoning-close content: Socket close after reasoning-only rejects instead of resolving empty - status: pending + status: completed - id: macos-dragdrop content: macOS drag-drop handler consumes only Drop, restoring HTML5 DnD - status: pending + status: completed - id: workspace-token-colon content: Full-precision write-conflict token; colon ban gated to Windows - status: pending + status: completed - id: reasoning-synonyms content: delta_fields skips empty reasoning synonyms to the next key - status: pending + status: completed - id: titlebar-glyphs content: Maximize/restore glyph toggle that actually hides SVG - status: pending + status: completed - id: menu-state content: menu.rs with workbench snapshot, bus, chat_ready, per-profile memory file - status: pending + status: completed - id: gateway-sse content: Gateway switch-profile streams stage events; client consumes them - status: pending + status: completed - id: populate content: Boot and heartbeat populate profile state and reachability - status: pending + status: completed - id: session-multiplex content: Session loop multiplexes concurrent chats by id, inbound always read - status: pending + status: completed - id: ws-frames content: Connect snapshot, workbench branch, inbound select/switch events - status: pending + status: completed - id: socket-ui content: WorkshopSocket workbench frame and event send methods - status: pending + status: completed - id: model-service content: WorkbenchService and ModelService command/state split fed by pushes - status: pending + status: completed - id: main-rewire content: main.ts drops boot fetches and local switch logic - status: pending + status: completed - id: menu-render content: Menu disables all rows while switching, rebuilds live - status: pending + status: completed - id: send-gating content: Composer send and mic recording gated on chat_ready - status: pending + status: completed - id: remove-http content: Remove HTTP /profiles routes, tests, and update module ceilings - status: pending + status: completed - id: verify content: Full test suites pass and live switch verified - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-27-4-merged-gateway-workshop.md b/vibe/2026-08/2026-08-27-4-merged-gateway-workshop.md index 3cb2302a2..169f993ed 100644 --- a/vibe/2026-08/2026-08-27-4-merged-gateway-workshop.md +++ b/vibe/2026-08/2026-08-27-4-merged-gateway-workshop.md @@ -7,37 +7,37 @@ todos: status: completed - id: harden-cross-site content: "Step 1: block cross-site requests - Sec-Fetch-Site filter, WS Origin allowlist, JSON content-type guard, URL-decode workspace path params" - status: pending + status: completed - id: harden-atomic-writes content: "Step 2: shared atomic-write helper for workspace.rs and menu.rs plus startup orphan sweep" - status: pending + status: completed - id: harden-timeouts content: "Step 3: TimeoutLayer tiers, explicit gateway HTTP timeout, bounded heartbeat probes" - status: pending + status: completed - id: harden-delta-decoder content: "Step 4: gateway delta-decoder conformance audit with random-split tests" - status: pending + status: completed - id: harden-backoff content: "Step 5: gateway reconnect backoff - reset on useful work, jitter, total budget" - status: pending + status: completed - id: harden-traversal content: "Step 6: verify debug asset path enforces the same traversal guarantee as the release embed path" - status: pending + status: completed - id: gateway-spawn content: "Step 7: refactor gateway run() into spawn() returning GatewayHandle with shutdown" - status: pending + status: completed - id: workshop-config content: "Step 8: add optional [workshop] section to gateway config with boot-only profile validation" - status: pending + status: completed - id: host-workshop content: "Step 9: gateway workshop feature - spawn ws-server on second loopback listener, derive client creds, shutdown order" - status: pending + status: completed - id: shell-boot content: "Step 10: desktop shell boots merged gateway; first-run generates gateway.toml + default profile" - status: pending + status: completed - id: docs content: "Step 11: update README and design log; run full verify" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-28-2-coroutine-protocol-executor.md b/vibe/2026-08/2026-08-28-2-coroutine-protocol-executor.md index 9c710dce9..a81214f50 100644 --- a/vibe/2026-08/2026-08-28-2-coroutine-protocol-executor.md +++ b/vibe/2026-08/2026-08-28-2-coroutine-protocol-executor.md @@ -4,46 +4,46 @@ overview: "Replace promptforge-core's block_in_place bridge with a coroutine yie todos: - id: design-doc content: "Step 1: write design-ws/coroutine-executor-design.md capturing the protocol and scheduler design" - status: pending + status: completed - id: upgrade-mlua content: "Step 2: upgrade mlua 0.10 -> 0.12+ and lua54 -> lua55 as one isolated commit" - status: pending + status: completed - id: raii-teardown content: "Step 3: SectionContext RAII teardown with completed flag" - status: pending + status: completed - id: protocol-types content: "Step 4: Request/Answer enums with strict validation" - status: pending + status: completed - id: shim-layer content: "Step 5: Lua shim prelude installed per VM (infer, handle:infer, execute)" - status: pending + status: completed - id: coroutine-chunks content: "Step 6: section chunks run via Thread::resume; hook/pcall/jump spikes resolved" - status: pending + status: completed - id: scheduler-core content: "Step 7: Chain + driver loop + Infer/Execute dispatch; nested execute end-to-end on current-thread runtime (DECISION GATE)" - status: pending + status: completed - id: walk-core content: "Step 8: walk translation - fall-through, off-walk, reply/var roll-forward, id counter" - status: pending + status: completed - id: walk-jumps content: "Step 9: walk translation - jumps, return semantics, depth cap, observation boundaries" - status: pending + status: completed - id: h1-live content: "Step 10: H1 live pass on the scheduler (BlockRunMode)" - status: pending + status: completed - id: fanout-join content: "Step 11: fanout as N chains with join, concurrency window, ordered results" - status: pending + status: completed - id: fanout-semantics content: "Step 12: fanout write-race, fatal-arm abort, cancellation while suspended" - status: pending + status: completed - id: flip content: "Step 13: flip run() to the scheduler, delete the old engine (walkers, old fanout, bridge callers)" - status: pending + status: completed - id: cleanup content: "Step 14: delete bridge_blocking, fix doc example, current-thread callers, module docs" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-28-4-cuda-llama-provisioning.md b/vibe/2026-08/2026-08-28-4-cuda-llama-provisioning.md index 53d79234a..2b804d0e3 100644 --- a/vibe/2026-08/2026-08-28-4-cuda-llama-provisioning.md +++ b/vibe/2026-08/2026-08-28-4-cuda-llama-provisioning.md @@ -4,37 +4,37 @@ overview: Fix the remaining digest-marker defect, extract the tool, transcriptio todos: - id: commit-runtime-rule content: Commit the authorized root AGENTS.md runtime-compilation rule as Step 0 - status: pending + status: completed - id: marker-write-fallback content: Make both digest-marker persistence paths best-effort - status: pending + status: completed - id: extract-tools content: Extract the tool contract and move web-search ownership behind it - status: pending + status: completed - id: extract-transcribe content: Extract the Whisper transcription engine into promptforge-transcribe - status: pending + status: completed - id: extract-desktop-shell content: Extract the desktop window and WebView bridge into promptforge-desktop-shell - status: pending + status: completed - id: cuda-build content: Produce an embedded machine-native CUDA llama-server bundle during Cargo build - status: pending + status: completed - id: cuda-stage content: Verify and atomically stage the embedded CUDA bundle at runtime - status: pending + status: completed - id: companion-config content: Add typed speculative and multimodal companion configuration - status: pending + status: completed - id: companion-runtime content: Provision companion artifacts and preserve their launch arguments across respawn - status: pending + status: completed - id: live-integration content: Prove and document CUDA, MTP, multimodal, and cache behavior - status: pending + status: completed - id: profile-migration content: Roll out E2B safely, then permanently remove the cached 31B model - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-29-1-crate-extraction-execution.md b/vibe/2026-08/2026-08-29-1-crate-extraction-execution.md index 8cbcc84fb..5664c67d1 100644 --- a/vibe/2026-08/2026-08-29-1-crate-extraction-execution.md +++ b/vibe/2026-08/2026-08-29-1-crate-extraction-execution.md @@ -4,31 +4,31 @@ overview: "Execute all nine crate extractions from the extraction menu in depend todos: - id: extract-protocol content: Extract promptforge-gateway-protocol (wire + upstream, re-type shutdown, http_util) - status: pending + status: completed - id: extract-client content: Extract promptforge-gateway-client (core client + model, picker stays in core) - status: pending + status: completed - id: workshop-facade content: Rewire workshop client as facade over shared wire crate - status: pending + status: completed - id: extract-store content: Extract promptforge-store (store.rs + store/ together) - status: pending + status: completed - id: extract-local content: Extract promptforge-gateway-local behind additive local feature - status: pending + status: completed - id: extract-lua content: Extract promptforge-lua plus core-support crate for untrusted/cancel/observe - status: pending + status: completed - id: extract-parser content: Extract promptforge-parser - status: pending + status: completed - id: ui-artifact content: Convert UI build to versioned artifact consumption - status: pending + status: completed - id: extract-search-service content: Extract gateway web-search service behind additive feature - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-31-2-rebuild-mdbook-from-includes.md b/vibe/2026-08/2026-08-31-2-rebuild-mdbook-from-includes.md index de7604c8f..4bf82771b 100644 --- a/vibe/2026-08/2026-08-31-2-rebuild-mdbook-from-includes.md +++ b/vibe/2026-08/2026-08-31-2-rebuild-mdbook-from-includes.md @@ -4,13 +4,13 @@ overview: Replace the stale hand-authored mdBook chapters with thin wrappers tha todos: - id: chapters content: Delete old chapter files and create thin include wrappers - status: pending + status: completed - id: summary content: Rewrite SUMMARY.md and update introduction.md - status: pending + status: completed - id: verify content: Verify mdbook build succeeds - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-08/2026-08-31-6-tauri-migration-for-workshop.md b/vibe/2026-08/2026-08-31-6-tauri-migration-for-workshop.md index 057062f23..0c605573f 100644 --- a/vibe/2026-08/2026-08-31-6-tauri-migration-for-workshop.md +++ b/vibe/2026-08/2026-08-31-6-tauri-migration-for-workshop.md @@ -4,25 +4,25 @@ overview: "Migrate the PromptForge Workshop desktop shell from raw tao/wry to Ta todos: - id: scaffold content: "Scaffold Tauri in promptforge-workshop: deps, build.rs, final tauri.conf.json, capabilities, icons (desktop-shell untouched)" - status: pending + status: completed - id: rust-core content: Rewrite main.rs on Tauri Builder and port navigation.rs (app boots under Tauri; no drops/mic yet) - status: pending + status: completed - id: rust-bridge content: "Port bridge.rs: Windows COM file drops plus per-platform mic permission handlers, with tests" - status: pending + status: completed - id: delete-shell content: Delete the promptforge-desktop-shell crate and its workspace.dependencies entry - status: pending + status: completed - id: frontend content: Switch window-chrome.ts and workshop-panel.ts to Tauri APIs; update UI test mocks; repackage dist - status: pending + status: completed - id: field-fixes content: "Field-report fixes: download resume in cache.rs, truncation warning, mic UX, tilde expansion, status bar recompute" - status: pending + status: completed - id: verify content: Full test/lint suite, docs sweep, cargo tauri build, manual smoke; then fold all checkpoints into one commit - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-01-1-build-simplification.md b/vibe/2026-09/2026-09-01-1-build-simplification.md index a39dc2a14..183b3a149 100644 --- a/vibe/2026-09/2026-09-01-1-build-simplification.md +++ b/vibe/2026-09/2026-09-01-1-build-simplification.md @@ -4,49 +4,49 @@ overview: Move the web UI build out of git and into a shared build helper. Build todos: - id: a-ui-build-crate content: Create crates/ui-build helper and switch both UI crates' build.rs to it - status: pending + status: completed - id: a-rust-embed-outdir content: Point both assets.rs at the build output folder; untrack both ui/dist trees and add them to .gitignore - status: pending + status: completed - id: a-delete-manifest content: Remove the manifest files and the packaged-crate fallback from both UI crates; document Node 22 + npm ci as a build requirement - status: pending + status: completed - id: d-gitattributes content: Extend .gitattributes and renormalize in a dedicated commit - status: pending + status: completed - id: b1-cli content: Convert promptforge-gateway-build into the crates/llama-cuda-build command-line tool with --source, --arch, --out, --no-smoke; put the CUDA runtime DLLs in the output zip - status: pending + status: completed - id: b2-workflow content: Add .github/workflows/llama-cuda-blackwell.yml (hosted build, self-hosted smoke test, publish release with checksums); retire cuda.yml - status: pending + status: completed - id: b3-runtime-select content: Add cuda-blackwell and upstream cuda rows to the asset table, add GPU-based backend selection with a config override, add the config/env/cache resolution order for llama-server, remove the embedded CUDA path and the llama.cpp submodule - status: pending + status: completed - id: c1-cargo-dist content: Set up cargo-dist for the gateway, Linux only, with a Node setup step and a post-build test job - status: pending + status: completed - id: c2-tauri-release content: Add release-workshop.yml for Windows, macOS (two builds), and Linux, with a post-build test job and checksums - status: pending + status: completed - id: c3-ci-clean-tree content: Add a clean-tree check to ci.yml so a build can never dirty the repository again - status: pending + status: completed - id: e-default-members content: Set workspace default-members to the gateway so plain cargo build works on macOS and Linux - status: pending + status: completed - id: e-setup-docs content: Add per-OS setup instructions (Ubuntu packages, macOS CLT/brew, Windows) to the README and crate docs - status: pending + status: completed - id: e-rust-toolchain content: Add rust-toolchain.toml pinned to the MSRV channel - status: pending + status: completed - id: e-readme-align content: Align README and AGENTS.md on when Node is required and what plain cargo build does - status: pending + status: completed - id: e-linux-workshop-ci content: Add a Linux workshop build job to ci.yml with the Tauri system packages - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-02-1-workshop-agent-window-clone.md b/vibe/2026-09/2026-09-02-1-workshop-agent-window-clone.md index d51f0f5aa..73fb8984a 100644 --- a/vibe/2026-09/2026-09-02-1-workshop-agent-window-clone.md +++ b/vibe/2026-09/2026-09-02-1-workshop-agent-window-clone.md @@ -4,58 +4,58 @@ overview: "Replace the workshop's plain-text agent session UI with a Cursor-styl todos: - id: tokens content: "Design token overhaul: sync style.css with Cursor's palette, spacing, typography, radius, shadows" - status: pending + status: completed - id: markdown-render content: Build markdown renderer (marked + shiki custom theme, full re-parse streaming, DOMPurify) - status: pending + status: completed - id: tool-cards content: Build collapsible ToolCallCard (details/summary, auto-expand, syntax-highlighted body) - status: pending + status: completed - id: feed-restyle content: Restyle message feed (single-column, streaming cursor, left accent on user messages) - status: pending + status: completed - id: feed-integrate content: Wire markdown renderer and ToolCallCard into AgentSessionView renderItem() - status: pending + status: completed - id: tiptap-base content: Build PromptInput (Tiptap Editor, StarterKit, Placeholder, submit/IME/auto-resize) - status: pending + status: completed - id: mention-chip content: Build MentionChip NodeView + mention extension registration - status: pending + status: completed - id: typeahead content: Build TypeaheadPopup (positioned, keyboard-navigable, suggestion lifecycle) - status: pending + status: completed - id: stt-rename content: Rename voice to stt across UI and wire routes (/voice -> /stt, incl. promptforge-stt crate) - status: pending + status: completed - id: tiptap-integrate content: Replace textarea in AgentSessionView with PromptInput, wire stt adapter - status: pending + status: completed - id: mode-chip content: Build ModeChip dropdown (UI-only, as const union, agent-mode-changed event) - status: pending + status: completed - id: model-picker content: Build ModelPickerTrigger dropdown (wired to ModelService) - status: pending + status: completed - id: token-ring content: Build TokenRing SVG (stroke-dasharray, stub percentage) - status: pending + status: completed - id: toolbar content: Build AgentToolbar composing ModeChip + ModelPickerTrigger + TokenRing - status: pending + status: completed - id: toolbar-integrate content: Mount toolbar above PromptInput in AgentSessionView, thread ModelService - status: pending + status: completed - id: zoom content: Add global Ctrl+/Ctrl-/Ctrl+0 zoom (shortcuts.ts + window-menu.ts, localStorage) - status: pending + status: completed - id: rec-led content: Replace REC text badge with a second status LED (red when recording, left of activity LED) - status: pending + status: completed - id: css-sweep content: Sweep all component CSS to use new design tokens, remove hardcoded values - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-02-6-product-user-guides.md b/vibe/2026-09/2026-09-02-6-product-user-guides.md index 9ad8e92ce..671741eb5 100644 --- a/vibe/2026-09/2026-09-02-6-product-user-guides.md +++ b/vibe/2026-09/2026-09-02-6-product-user-guides.md @@ -4,34 +4,34 @@ overview: Sweep the repo clean (stale files, tape remnants, publishing metadata) todos: - id: step-00 content: "Repo cleanup (3 commits): stale file deletion + link repair; tape-remnant excision (gateway-config, config UI, examples); publishing-metadata strip (publish=false, docs.rs, semver-checks)" - status: pending + status: completed - id: step-01 content: Edit design/what-promptforge-is.md down to general terms (product-level roles, no crate names or knobs - drift-proof), STE register for the edits - status: pending + status: completed - id: step-02 content: Delete the old guide system (per-crate guides, topic pages); mdbook skeleton with four parts; build-user-guide gutted to a compiling skeleton - status: pending + status: completed - id: step-03 content: "The generator tool: guided dokuman as an agent tool file (lens blocks, recon, manifest contract, per-file extract, tier, verify, evidence packet, single writer with checkpoints, per-chapter thoroughness gate, audit); proven on the language set" - status: pending + status: completed - id: step-04 content: Agent-programs documentation set - status: pending + status: completed - id: step-05 content: Gateway documentation set (configuration and operation lens) - status: pending + status: completed - id: step-06 content: Workshop documentation set (UI operation lens, incl. the TypeScript UI source) - status: pending + status: completed - id: step-07 content: "The assembler: build-user-guide becomes the book assembler (SUMMARY.md TOC, per-set exports, link check); the intro lens run generates the introduction" - status: pending + status: completed - id: step-08 content: "README storefront: prominent evergreen per-platform download links (Windows/macOS/Linux), existing images preserved in place, remove crates.io/install/MCP" - status: pending + status: completed - id: step-09 content: "Final verification: mdbook build, Pages deploy, links, workspace green" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-04-1-async-boot-and-progress.md b/vibe/2026-09/2026-09-04-1-async-boot-and-progress.md index d5003fc91..ca9806b36 100644 --- a/vibe/2026-09/2026-09-04-1-async-boot-and-progress.md +++ b/vibe/2026-09/2026-09-04-1-async-boot-and-progress.md @@ -7,19 +7,19 @@ todos: status: completed - id: core-restructure content: "Step 2 - Pieces 1-3 in one commit: Gateway::new instant assembly, command queue (FIFO, debounced, cancellable), boot LoadProfile command, serve_thread restructured" - status: pending + status: completed - id: drop-indicatif content: "Step 3 - Delete the Renderer thread and indicatif dep; command worker emits tracing log lines via LineRenderer" - status: pending + status: completed - id: whisper-bridge content: "Step 4 - WhisperLibrary::set_log_callback bridges ggml/whisper logging into tracing; whisper_cpp=warn works" - status: pending + status: completed - id: status-surfaces content: "Step 5 - Tray label reads the queue; /admin/status gains queue+endpoints; POST /admin/queue/cancel; config UI bottom status bar (LEDs <-> progress swap, cancel button)" - status: pending + status: completed - id: cleanup-and-browser content: "Step 6 - Verify hack removal against the cancellable queue; rename --open-settings to --browser everywhere" - status: pending + status: completed - id: shared-ui content: "Step 7 - Extract shared-ui (Cursor Dark tokens, modal, dropdown, toast, status bar, progress bar, button/input); gateway adopts Cursor Dark; workshop adopts shared toast" status: completed diff --git a/vibe/2026-09/2026-09-04-3-unlock-inference-during-switches.md b/vibe/2026-09/2026-09-04-3-unlock-inference-during-switches.md index 34e0abe0b..c6c843929 100644 --- a/vibe/2026-09/2026-09-04-3-unlock-inference-during-switches.md +++ b/vibe/2026-09/2026-09-04-3-unlock-inference-during-switches.md @@ -10,16 +10,16 @@ todos: status: completed - id: unlock-switch content: Step 3 - five-phase run_switch_with_config with switch held only for cut-over and commit; artifact/spawn split reusing the existing provisioner; LiveState.loading + GatewayError::ModelLoading 503 + status loading_models; downloading-models leaf and overlay stage; regression tests; scheduled Verify - status: in_progress + status: completed - id: bounded-waits content: Step 4 - timeout on worker.await in Gateway::serve (5 s, warn) with a parked-worker test; idle read timeout on artifact downloads so Cancel/Quit are honored within seconds; default tracing filter becomes info for gateway crates plus whisper_cpp=warn; log file at /logs/gateway.log with one-run rotation, no new dep - status: pending + status: completed - id: overlay-download-progress content: Step 5 - apply overlay renders Updated frames from */download paths as a progress bar with model name and percent under the active stage; tests - status: pending + status: completed - id: ui-hygiene content: Step 6 (final) - hoist isRecord, thread error.code for all refusals, onCancel-rejection and unmounted-Settings-revert tests, mark stale async-boot plan todos; final full-suite Verify - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-05-2-generic-realtime-stt.md b/vibe/2026-09/2026-09-05-2-generic-realtime-stt.md index 56f94d513..a2a861ed6 100644 --- a/vibe/2026-09/2026-09-05-2-generic-realtime-stt.md +++ b/vibe/2026-09/2026-09-05-2-generic-realtime-stt.md @@ -4,25 +4,25 @@ overview: Replace Workshop-specific speech transcription with a generic OpenAI R todos: - id: characterize content: Pin current batch and two-model realtime behavior with deterministic and native fixtures - status: pending + status: completed - id: backend-boundary content: Split the backend-neutral engine from the safe Whisper adapter and bound model workers - status: pending + status: completed - id: session-lifecycle content: Isolate per-item finalization and implement atomic cancellation-safe generation replacement - status: pending + status: completed - id: realtime-contract content: Implement the OpenAI Realtime transcription subset and hypothesis extension - status: pending + status: completed - id: workshop-adapter content: Convert Workshop to a payload-opaque relay with local capture and status ownership - status: pending + status: completed - id: verify-document content: Enforce architecture and debt budgets, complete acceptance, and document final boundaries - status: pending + status: completed - id: gateway-log-bookends content: Mark Gateway serving-log launch and terminal outcomes without changing logging infrastructure - status: pending + status: completed - id: ci-architecture-toolchain content: Run pinned architecture tools under the repository Cargo version inside stable CI status: completed @@ -49,7 +49,7 @@ todos: status: completed - id: realtime-live-hypotheses content: Bind precommit hypothesis item IDs to the active browser take - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-08-1-remove-unsupported-ratchets.md b/vibe/2026-09/2026-09-08-1-remove-unsupported-ratchets.md index 4cb9935b7..e5fdf4d91 100644 --- a/vibe/2026-09/2026-09-08-1-remove-unsupported-ratchets.md +++ b/vibe/2026-09/2026-09-08-1-remove-unsupported-ratchets.md @@ -4,16 +4,16 @@ overview: Prune repository policy, remove unsupported structural ratchets, prese todos: - id: remove-structural-ratchets content: Prune policy guidance and remove unsupported structural ratchets - status: pending + status: completed - id: establish-gateway-ownership content: Preserve logging and establish Gateway ownership - status: pending + status: completed - id: repair-profile-and-registry content: Preserve profile behavior and generation identity - status: pending + status: completed - id: revoke-publication-and-shutdown content: Revoke publication and bound supervisor shutdown - status: pending + status: completed isProject: false --- # Remove Unsupported Ratchets and Repair Runtime Defects diff --git a/vibe/2026-09/2026-09-09-2-async-stt-boot-reset.md b/vibe/2026-09/2026-09-09-2-async-stt-boot-reset.md index 0ecfec8ad..a59a0b213 100644 --- a/vibe/2026-09/2026-09-09-2-async-stt-boot-reset.md +++ b/vibe/2026-09/2026-09-09-2-async-stt-boot-reset.md @@ -4,19 +4,19 @@ overview: Restart the STT hot-swap removal from the clean base while preserving todos: - id: simplify-gateway-infrastructure content: Rename GatewayDiscoveryFile and replace the duplicated bounded command channel with one notified pending deque - status: pending + status: completed - id: one-time-speech content: Build the minimal one-time speech load while preserving existing status and retained ownership - status: pending + status: completed - id: bind-first-queue content: Load STT once through the existing post-readiness boot profile command and remove runtime reload paths - status: pending + status: completed - id: ui-restart content: Add the browser STT change predicate and successful-Apply restart toast - status: pending + status: completed - id: docs-qualification content: Reconcile documentation and complete full automated, package, and microphone qualification - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-10-1-unified-prompt-model.md b/vibe/2026-09/2026-09-10-1-unified-prompt-model.md index 03d12fe70..1a8a647ed 100644 --- a/vibe/2026-09/2026-09-10-1-unified-prompt-model.md +++ b/vibe/2026-09/2026-09-10-1-unified-prompt-model.md @@ -4,43 +4,43 @@ overview: "Implement a narrow `promptforge: 0` core: rename heading-based `execu todos: - id: define-unified-contracts content: Define only the active plain-message, message-builder, models.loop, and call-time model/tool selection contracts; retain every other contract as deferred description. - status: pending + status: completed - id: rename-section-call content: Rename heading-based `execute` to synchronous `call` as the first behavior change, preserving current execution semantics. - status: pending + status: completed - id: resolve-concurrency-primitive content: "Deferred: retain the designed `call_async`, task, `tasks.await_all`, and thin Lua fanout contracts without implementing or migrating them." status: deferred - id: replace-prose-semantics content: Replace positional prose inference with a pending Markdown buffer, thematic-break reset, and a read-only, lazy, memoized `prose` value for each following Lua fence. - status: pending + status: completed - id: add-message-list content: Ship optional pure-Lua `messages.new()` builders over the unchanged plain message-array contract. - status: pending + status: completed - id: add-context-projection content: Add only provider-neutral message projection and role and tool-call correlation validation required by active `models.loop`. - status: pending + status: completed - id: add-compactor-fail content: Add the minimum compactor surface `models.loop` requires (optional compactor parameter defaulting to `compactors.fail`, overflow invocation with reason, typed context exhaustion) and defer the replacement-handling framework. - status: pending + status: completed - id: add-models-loop content: Expose Rust-backed `models.loop(handle?, messages, compactor?)`, append complete assistant and tool history, return nil, and default omitted compaction policy to `compactors.fail`. - status: pending + status: completed - id: add-modules-execution content: "Deferred: retain inherited Lua, `require`, path-based `execute`, `execute_async`, `include`, `local_include`, and Markdown fragments as documented future contracts." status: deferred - id: generalize-user-input content: Lift blocking, fallback, and failing user input into a generic broker used by direct Lua `user_input()`, the model-visible input tool, and the Agent window. - status: pending + status: completed - id: restore-unified-sessions content: "Deferred: retain journaled operations, snapshots, incarnation-gated restore, and replay contracts." status: deferred - id: migrate-prompts-guides content: Migrate only prompts, fixtures, APIs, and guides directly affected by `execute` to `call`, `tool_call` to `tools.call`, `handle:infer` to `models.infer(handle, prompt)`, explicit lazy prose, `reply` removal, message builders, `models.infer`, and `models.loop`. - status: pending + status: completed - id: verify-complete-model content: Run focused parser, Lua, models.loop, projection, and migration verification for the active core only. - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-10-2-debt-fixes.md b/vibe/2026-09/2026-09-10-2-debt-fixes.md index d2cca2b2a..e31ee4260 100644 --- a/vibe/2026-09/2026-09-10-2-debt-fixes.md +++ b/vibe/2026-09/2026-09-10-2-debt-fixes.md @@ -4,22 +4,22 @@ overview: "Remove two accepted debts from the unified prompt model run: narrow t todos: - id: fix-input-tool content: Narrow the input tool description and schema in input.rs to its real behavior; update affected tests. - status: pending + status: completed - id: flip-gate content: "Flip the version gate to `promptforge: 0` only and clean up `SUPPORTED_MAJOR`." - status: pending + status: completed - id: migrate-prompts content: "Migrate every `promptforge: 1` string to `promptforge: 0` repo-wide, inline test strings included; historical `vibe/` records excluded." - status: pending + status: completed - id: gate-tests content: Invert version-acceptance tests into rejection tests and add the gate behavior test. - status: pending + status: completed - id: verify content: Run focused tests, then the full gate (build, fmt, clippy, workspace suite). - status: pending + status: completed - id: close-out content: Commit one commit per debt and annotate the debt removal plan's execution items. - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-11-1-rulebook-debt-tiers.md b/vibe/2026-09/2026-09-11-1-rulebook-debt-tiers.md index f7e0fd545..148ac49e4 100644 --- a/vibe/2026-09/2026-09-11-1-rulebook-debt-tiers.md +++ b/vibe/2026-09/2026-09-11-1-rulebook-debt-tiers.md @@ -4,31 +4,31 @@ overview: Fix all Tier 1 (correctness/operational) and Tier 2 (hygiene) rust-rul todos: - id: ws-b content: "B: CI doctest step + aggregate job + drop unused tracing-subscriber dev-dep" - status: pending + status: completed - id: ws-a1 content: "A: fix blocking/fallible Drop impls (RecoveryCandidate, GatewaySupervisor, SttEngine/Transcriber)" - status: pending + status: completed - id: ws-a2 content: "A: restore discarded error causes (SessionError::Inference, AudioError::InvalidBase64)" - status: pending + status: completed - id: ws-a3 content: "A: bound the session_agents supervisor event channel" - status: pending + status: completed - id: ws-d1 content: "D: add #[non_exhaustive] to 7 public items/variants" - status: pending + status: completed - id: ws-d2 content: "D: convert 43 #[allow] sites to #[expect] with reasons" - status: pending + status: completed - id: ws-e content: "E: convert ~5 in-process async tests to paused time" - status: pending + status: completed - id: ws-c1 content: "C1: FixtureError for the fixture API + typed sources in the take/finalization pipeline (~26 sites)" - status: pending + status: completed - id: ws-c2 content: "C2: anyhow for test-only code and build tools + folded typed errors in remaining production (~32 sites)" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-11-2-unify-toolchain.md b/vibe/2026-09/2026-09-11-2-unify-toolchain.md index dea7f338d..feaae6852 100644 --- a/vibe/2026-09/2026-09-11-2-unify-toolchain.md +++ b/vibe/2026-09/2026-09-11-2-unify-toolchain.md @@ -4,22 +4,22 @@ overview: Remove the MSRV/local-vs-CI toolchain split so local builds and CI use todos: - id: fix-ci-failures content: Fix audio.rs:317 parens, bridge.rs expect lints, decode.rs flaky test - status: pending + status: completed - id: unify-toolchain-files content: rust-toolchain.toml to stable, remove rust-version from workspace and all 34 crates - status: pending + status: completed - id: simplify-ci-yml content: Remove RUSTUP_TOOLCHAIN overrides and msrv job from ci.yml - status: pending + status: completed - id: simplify-other-workflows content: Remove MSRV env and validation step from stt-miri.yml, remove from release-workshop.yml - status: pending + status: completed - id: docs-and-cleanup content: Remove skew bullet from AGENTS.md, MSRV section from README.md, delete validation script and its test - status: pending + status: completed - id: verify content: Run full gate locally on stable, push, confirm CI green - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-11-3-vfs-foundation.md b/vibe/2026-09/2026-09-11-3-vfs-foundation.md index aa4fd317c..bd9230414 100644 --- a/vibe/2026-09/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09/2026-09-11-3-vfs-foundation.md @@ -4,37 +4,37 @@ overview: "Focused foundation layer: a Vfs/VfsAccess trait pair with a claims mo todos: - id: vfs-trait content: "Design the Vfs/VfsAccess trait pair and VfsRef handle: sync, bytes-based, Store-derived semantics plus stat/list/grep, acquire/release on the backend trait" - status: pending + status: completed - id: router content: "Build the mount router: builder-style mount installation, longest-prefix dispatch, lazy per-mount acquire, nestable, universal path canonicalization (Windows/macOS/Linux)" - status: pending + status: completed - id: mem-backend content: Implement the in-memory Vfs backend carrying former MemStore semantics - status: pending + status: completed - id: hostdir-backend content: "IN SCOPE (without it the Vfs is useless): real-filesystem backend in shared-vfs (std::fs is std - the zero-dependency rule holds) with `HostBackend::identity()` and `HostBackend::rooted(dir)`. Stage 1 thin: direct std::fs ops, lexical+canonicalize containment, failure-atomic writes (sibling temp + rename). Stage 2 hardening toward the Bashkit RealFs oracle (resolver trio, symlink policies, Windows long paths and device names) as the threat model demands" - status: pending + status: completed - id: policy content: "Implement the Policy layer: Op and Verdict (Deny/Ask carry reason strings), per-op check in Access before the claims check, AllowAll in shared-vfs, ModePolicy (Ask/Plan/Agent) in promptforge-vfs with a UI-flippable shared mode" - status: pending + status: completed - id: store-rewrite content: Rewrite Store as a public concrete facade (no trait) over Vfs mounted at /_promptforge/store; Lua store table unchanged; WriteScope registry deleted in favor of the claims model; the Harness seeds/extracts declared input/output keys through it - status: pending + status: completed - id: executor-api content: Change executor::run() to take VfsRef instead of StoreRef; RunContext builds the Store facade internally - status: pending + status: completed - id: store-yield content: Make Lua store operations leaf yields in the coroutine protocol (new Request/Answer variants, one dispatch arm, spawn_blocking over the sync Vfs); uniform for all backends - no inline fast path - status: pending + status: completed - id: bashkit-adapter content: "SPIKE (deliverable is evidence, not integration): implement the Bashkit FsBackend adapter over VfsRef as a path dependency against the local clone at bashkit/, compile-check, and smoke-test an ls/cat/grep script against mounted backends. Toolchain already verified compatible: workspace runs stable 1.98, Bashkit's 1.95 pin applies only inside its own repo. Success proves the trait subsumes Bashkit; a mapping failure here is the spike working as intended, cheaply" - status: pending + status: completed - id: claims content: "Implement the claims model: ExecId vending, Access RAII capability (acquire/spawn/borrow/move/drop), readers/writers claims tables keyed by interned canonical paths, canonicalize (stub-to-intern acceptable in v1), fatal determinism RunErrorKind, scheduler installation of the current access per chain step" - status: pending + status: completed - id: tests content: Port the promptforge-store test suite onto the rewritten Store; add router and path-canonicalization matrices - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-12-2-dependency-rules-vfs-hook.md b/vibe/2026-09/2026-09-12-2-dependency-rules-vfs-hook.md index 32eb09c5f..829867e76 100644 --- a/vibe/2026-09/2026-09-12-2-dependency-rules-vfs-hook.md +++ b/vibe/2026-09/2026-09-12-2-dependency-rules-vfs-hook.md @@ -4,25 +4,25 @@ overview: Land the AGENTS.md dependency-rule revisions, rename shared-protocol t todos: - id: land-rules content: Commit the AGENTS.md rule revisions together with the two design/ deletions - status: pending + status: completed - id: rename-crate content: Rename shared-protocol to gateway-protocol (crate, manifests, imports, AGENTS.md, lockfile) - status: pending + status: completed - id: extend-arch-test content: "Extend architecture.rs: Shared set + fifth rule, workshop prefix fix, fixture coverage" - status: pending + status: completed - id: fix-unix-clippy content: "Add #[expect(unnecessary_wraps)] to the cfg(unix) mode_of in shared-vfs host.rs" - status: pending + status: completed - id: fix-writerace content: "dispatch_store: drop the access clone before sending the answer" - status: pending + status: completed - id: vfs-hook content: "shared-vfs: op-observation sink on the handle plus mandatory acquire identity label" - status: pending + status: completed - id: verify content: "Verify: local gates plus PR #35 CI green across clippy, test, native-whisper" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-12-3-workshop-server-decomposition.md b/vibe/2026-09/2026-09-12-3-workshop-server-decomposition.md index 423a8e424..2a566aabe 100644 --- a/vibe/2026-09/2026-09-12-3-workshop-server-decomposition.md +++ b/vibe/2026-09/2026-09-12-3-workshop-server-decomposition.md @@ -4,43 +4,43 @@ overview: Full-stack decomposition of Workshop - server-side crate split (18.2k todos: - id: fix-ci content: "Fix failing CI: scheduler determinism-violation test broken by drain commit" - status: pending + status: completed - id: enforcement content: Add xtask with tidy-style architecture tests and a new-crate generator - status: pending + status: completed - id: spa-directory-split content: "SPA: split flat workshop/ into editor/, agent/, tree/, layout/, menu/ subdirectories" - status: pending + status: completed - id: spa-css-colocation content: "SPA: colocate CSS beside TypeScript, add .ws-* prefix, extract --ws-* design tokens" - status: pending + status: completed - id: registry content: Create workshop-registry crate; migrate status bus as proof of self-registration - status: pending + status: completed - id: tier0 content: Extract workshop-protocol and workshop-support (with generic retained-bus collapse) - status: pending + status: completed - id: services content: Extract workshop-gateway, workshop-status, workshop-menu - status: pending + status: completed - id: features content: Extract workshop-workspace and workshop-sessions; decompose AppState into registry handles - status: pending + status: completed - id: spa-lazy-panels content: "SPA: dynamic import registry for lazy panel loading; enable esbuild splitting" - status: pending + status: completed - id: spa-god-objects content: "SPA: break up window-menu.ts god object; add Part base class for panels; centralize zone/tree state" - status: pending + status: completed - id: view-decouple content: Decouple the SPA bundle behind a narrow asset interface; add headless test feature; content-hashed filenames - status: pending + status: completed - id: shell-cleanup content: Delete workshop shell's duplicated gateway supervisor in favor of shared-sidecar - status: pending + status: completed - id: rulebook content: Write the architecture invariants into workspace rules - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-12-4-workshop-debt-removal.md b/vibe/2026-09/2026-09-12-4-workshop-debt-removal.md index 2026c5401..5bb2ed8bc 100644 --- a/vibe/2026-09/2026-09-12-4-workshop-debt-removal.md +++ b/vibe/2026-09/2026-09-12-4-workshop-debt-removal.md @@ -4,19 +4,19 @@ overview: "Remove the seven accepted debts the workshop decomposition (f566cc1e. todos: - id: d1-docs-link content: "D1 (WSD-07): qualify the Observer intra-doc link in promptforge-core input.rs; docs command green" - status: pending + status: completed - id: d2-facade-enforcement content: "D2 (WSD-04, WSD-03): handles.rs facades; workshop-server opts into tidy; split tests/it/agents.rs; rule wording matches enforcement" - status: pending + status: completed - id: d3-registry-collections content: "D3 (WSD-01, WSD-02): registry redesigned to contribution collections keyed by kind and type; single wiring path; tasks register; boot-time require" - status: pending + status: completed - id: d4-drift-tests content: "D4 (WSD-05, WSD-06): build-implementer drift test; settle the lazy-CSS premise empirically" - status: pending + status: completed - id: d5-records content: "D5 (WSD-08): dated corrections to the decomposition plan, rule files, and registry docs" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-12-5-one-door-promptforge-api.md b/vibe/2026-09/2026-09-12-5-one-door-promptforge-api.md index 2636076ef..3fca1193b 100644 --- a/vibe/2026-09/2026-09-12-5-one-door-promptforge-api.md +++ b/vibe/2026-09/2026-09-12-5-one-door-promptforge-api.md @@ -4,31 +4,31 @@ overview: Remove the standalone .lua agent program path and the promptforge-agen todos: - id: remove-lua-agent-path content: "Remove .lua agent path in workshop-sessions: discovery to .md, delete launch_lua, local run error type, audit UserInputTool, fix tests and docs" - status: pending + status: completed - id: delete-agent-crate content: Delete crates/promptforge-agent and its Cargo/facade edges - status: pending + status: completed - id: fix-guides content: Rewrite agent guides and READMEs for .md directory agents - status: pending + status: completed - id: rename-core-support content: Rename promptforge-core-support to shared-promptforge-api; sink model vocabulary and the entire tool vocabulary into it - status: pending + status: completed - id: rename-core-to-api content: Rename promptforge-core to promptforge-api, retire the promptforge facade crate, make picker optional and vfs defaulted in the run API - status: pending + status: completed - id: delete-integration-tests content: Delete crates/product-integration-tests/ and all references to it - status: pending + status: completed - id: migrate-consumers content: Migrate workshop-sessions, workshop-gateway, workshop-protocol to promptforge-api + shared-promptforge-api; reimplement gateway progress - status: pending + status: completed - id: enforce-boundary content: Add product-boundary check to xtask tidy with test wrapper - status: pending + status: completed - id: verify content: Run full verification suite and metadata sweep - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-13-1-capabilities-global-naming.md b/vibe/2026-09/2026-09-13-1-capabilities-global-naming.md index 7f90ad8c6..1f4c4dbac 100644 --- a/vibe/2026-09/2026-09-13-1-capabilities-global-naming.md +++ b/vibe/2026-09/2026-09-13-1-capabilities-global-naming.md @@ -4,25 +4,25 @@ overview: "Plugins for PromptForge: globally named Plugins (arity encodes kind: todos: - id: global-names content: steps 1-3 - GlobalName grammar, ToolId re-base + built-in id migration, picker ToolId migration - status: pending + status: completed - id: interface content: step 4 - Environment/RunContext consolidation (pure refactor, the parity gate) - status: pending + status: completed - id: frontmatter content: steps 5-6 - plugins/tools/args/models keys, structured parse error locations - status: pending + status: completed - id: capabilities content: steps 7-9 - Plugin trait + RunServices, PluginRegistry, prepare/Requirements/per-run VFS - status: pending + status: completed - id: binding content: steps 10-15 - ModelBindings + trivial fill, catalog assembly, ToolBindings + slot filling, Lua surface consolidation, args/argv, prose migration - status: pending + status: completed - id: first-party content: steps 16-17 - the promptforge/web Plugin, Workshop wiring - status: pending + status: completed - id: docs content: step 18 - guide chapters + AGENTS.md Principles rule - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-14-1-debt-removal-capabilities-follow-ups.md b/vibe/2026-09/2026-09-14-1-debt-removal-capabilities-follow-ups.md index b40ddc60c..69c8cdd9c 100644 --- a/vibe/2026-09/2026-09-14-1-debt-removal-capabilities-follow-ups.md +++ b/vibe/2026-09/2026-09-14-1-debt-removal-capabilities-follow-ups.md @@ -4,16 +4,16 @@ overview: "Remove the three accepted debts from the Plugins/global-naming range todos: - id: retype-capability-accessor content: Re-type ToolId::plugin() to PluginId, delete the fill.rs string round-trip, migrate GlobalName-comparing consumers (PF-DEBT-01) - status: pending + status: completed - id: registry-twin-rejection content: Reject punctuation-normalized Plugin id twins at registry registration with RegistryErrorKind::NormalizationCollision (PF-DEBT-02) - status: pending + status: completed - id: remove-fallback-descriptor content: Delete the 8192 fallback descriptor; report catalog fetch failure as the chat launch error (PF-DEBT-03) - status: pending + status: completed - id: tool-errors-as-results content: Convert bound-tool dispatch failures into model-readable tool results in the model loop (PF-EXP-01) - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-14-2-provider-model-sheets.md b/vibe/2026-09/2026-09-14-2-provider-model-sheets.md index 11f08e0f5..54b4c283c 100644 --- a/vibe/2026-09/2026-09-14-2-provider-model-sheets.md +++ b/vibe/2026-09/2026-09-14-2-provider-model-sheets.md @@ -4,16 +4,16 @@ overview: "Phase 1 infrastructure for provider model sheets: two new shared crat todos: - id: settle-open-questions content: Settle remaining open questions (raw payload embedding, v1 scope confirmation, cadence) - status: pending + status: completed - id: shared-gateway-api content: Create shared-gateway-api with the sheet schema types (Sheet, ProviderSlice, SliceStatus, ModelEntry, Thinking, Pricing, Deprecation, Tier); hoist Capabilities, ModelInfo, ModelKind (extended with transcription/image/video), ThinkingMode from gateway-config/gateway-protocol with re-exports at the old paths - status: pending + status: completed - id: shared-cloud-providers content: Create shared-cloud-providers (lib + bin) with Provider descriptor (name, display_name, tier, key_env, base_url), provider registry, per-provider files, fetch/normalize behind an injected reqwest::Client, build_sheet/fetch_sheet - status: pending + status: completed - id: aggregation-workflow content: Build the GitHub workflow (manual + cron) that compiles the binary, runs it with secrets as env vars, and publishes models.json as a release artifact - DEFERRED 2026-09-14 to a separate aggregation repo, not this plan's scope - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-14-3-fix-previous-sheet-swallow.md b/vibe/2026-09/2026-09-14-3-fix-previous-sheet-swallow.md index 65b1e50c0..6c37fc79c 100644 --- a/vibe/2026-09/2026-09-14-3-fix-previous-sheet-swallow.md +++ b/vibe/2026-09/2026-09-14-3-fix-previous-sheet-swallow.md @@ -4,7 +4,7 @@ overview: "Remove DEBT-PMS-1: the sheet binary treats a failed previous-sheet do todos: - id: debt-pms-1 content: "DEBT-PMS-1: distinguish previous-sheet error kinds in fetch_sheet/previous_sheet; fatal exit on outage or corruption, tolerate unset URL and 404; unit + 3 integration tests; nextest/clippy/fmt green" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-14-4-provider-expansion-local-run.md b/vibe/2026-09/2026-09-14-4-provider-expansion-local-run.md index 45b050c7b..c250a3b06 100644 --- a/vibe/2026-09/2026-09-14-4-provider-expansion-local-run.md +++ b/vibe/2026-09/2026-09-14-4-provider-expansion-local-run.md @@ -4,31 +4,31 @@ overview: "Expand shared-cloud-providers from the ten Prime providers to the ful todos: - id: descriptor-keyless content: "Descriptor extension: Provider.key_env to Option, fetch_models takes Option<&str>, adapt ten Prime files, keyless fetch path" - status: pending + status: completed - id: concurrent-fanout content: build_sheet_with fetches providers concurrently via FuturesUnordered; BTreeMap keeps output deterministic; in-flight-counter test proves overlap - status: pending + status: completed - id: secrets-loader content: cloud-provider-secrets.env loading (profile-dir path, override semantics) in the binary + .gitignore guard + subprocess test - status: pending + status: completed - id: subprime-openai-dialect content: "OpenAI-dialect Subprime files: minimax, stepfun, groq (STT kind) on openai_shape" - status: pending + status: completed - id: subprime-rich content: "Rich Subprime files: mistral, cohere (token pagination), baidu (CNY pricing), soniox, leonardo" - status: pending + status: completed - id: keyless-files content: "Keyless files: nvidia (no released_at from placeholder) and openrouter (pricing, modalities, deprecation)" - status: pending + status: completed - id: subprime-heavy content: "Heavy files: bedrock (SigV4 + region), foundry (endpoint env), azure_speech (region env)" - status: pending + status: completed - id: registry-completeness content: Registry completeness test extended to all 23 providers with tiers - status: pending + status: completed - id: local-run content: Run the binary locally against the real cloud-provider-secrets.env and verify the emitted cloud-provider-models.json parses as a valid Sheet - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-14-5-models-aggregation-workflow.md b/vibe/2026-09/2026-09-14-5-models-aggregation-workflow.md index 10e3bcca5..7fb8634d5 100644 --- a/vibe/2026-09/2026-09-14-5-models-aggregation-workflow.md +++ b/vibe/2026-09/2026-09-14-5-models-aggregation-workflow.md @@ -4,19 +4,19 @@ overview: "Turn the placeholder models.yml in cppalliance/promptforge-cloud-prov todos: - id: make-public content: "Make the repository public so release assets download anonymously" - status: pending + status: completed - id: rewrite-workflow content: "Rewrite .github/workflows/models.yml: build, run, secret-scan gate, content-only change detection, conditional rolling-release publish" - status: pending + status: completed - id: readme content: "Rewrite README.md: badges, stable asset URL, what the sheet is, why PromptForge downloads it" - status: pending + status: completed - id: validate content: Validate the workflow YAML and inline scripts locally - status: pending + status: completed - id: verify-dispatch content: "Land the changes, then operator dispatches manually twice: first run publishes the initial sheet, second run skips publish; verify the asset URL downloads anonymously" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-14-6-model-taxonomy-cloud-ui.md b/vibe/2026-09/2026-09-14-6-model-taxonomy-cloud-ui.md index 2bda27b7c..54d90a971 100644 --- a/vibe/2026-09/2026-09-14-6-model-taxonomy-cloud-ui.md +++ b/vibe/2026-09/2026-09-14-6-model-taxonomy-cloud-ui.md @@ -4,34 +4,34 @@ overview: Bake model taxonomy into every provider file of shared-cloud-providers todos: - id: schema-extension content: "Schema extension: family, variant_of, variant, languages on ModelEntry; openai_base_url and env_vars (EnvVar/EnvRole) on ProviderSlice; openai_base_url and env_vars (EnvVarSpec) on the Provider descriptor; all required, no version bump; round-trip test; missing-field fixture fails to parse" - status: pending + status: completed - id: taxonomy-primitives content: "taxonomy.rs: fixed-width snapshot-suffix strippers, vendor/ prefix splitter, :sku splitter, collapse_variants post-pass; unit tests per primitive" - status: pending + status: completed - id: taxonomy-providers content: "All 23 provider files: fixture excerpts committed from the saved 2026-09-14 sheet; family_of + snapshot patterns, openai_base_url, env_vars declared; kind inference for openai/qwen/moonshot/mistral; deepgram dedup + languages; elevenlabs/soniox/azure_speech languages; table-driven taxonomy tests; every-entry-has-a-family assertion" - status: pending + status: completed - id: local-verify content: Run the binary locally against the real secrets file; verify families, Deepgram collapsed, OpenAI kinds, openai_base_url/env_vars on every slice; crate gates green - status: pending + status: completed - id: republish content: "Stage 1 close: push the crate, dispatch the aggregation workflow (release already deleted; re-delete first if the cron republished), confirm the rebuilt asset parses in the new shape" - status: pending + status: completed - id: gateway-sheet-cache content: "Gateway: disk-cached sheet at profile dir, background download at boot when older than a week, unparseable cache treated as absent, atomic replace, GET /admin/cloud-models loopback-walled, route tests" - status: pending + status: completed - id: ui-view content: "UI view: kind radio, tier-grouped provider select (greyed when empty for kind), family select with All default, full-height table with display_name primary and id beneath when different, +N snapshots disclosures; pure cascade helper + tests; subscribable sheet store polling until the sheet lands; Cloud tab registration" - status: pending + status: completed - id: add-model content: "Add-model flow: merge helper (endpoint from the slice's openai_base_url) + unit tests, confirm-details dialog, shadow/apply submit, disabled add with reason for providers without an OpenAI-compatible API" - status: pending + status: completed - id: secrets-dropdown content: Provider accessor over the sheet (services/providers.ts, no table) + tier-grouped provider dropdown in the Secrets add-variable row; multi-variable providers add all rows; keyless and present providers greyed - status: pending + status: completed - id: e2e content: "End-to-end: gateway integration test of fetch-serve-merge-apply; manual run against the republished URL and the real UI" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-15-1-cloud-sheet-debt-removal.md b/vibe/2026-09/2026-09-15-1-cloud-sheet-debt-removal.md index b52389e8a..3cdadc317 100644 --- a/vibe/2026-09/2026-09-15-1-cloud-sheet-debt-removal.md +++ b/vibe/2026-09/2026-09-15-1-cloud-sheet-debt-removal.md @@ -4,16 +4,16 @@ overview: "Remove the debt attributed to the model taxonomy and cloud models UI todos: - id: reader-cap-version content: "Gateway reader: capped download in download_once, shared schema-version constant, version gate on cache and download paths, module split for the ceiling; cap and version tests (DEBT-MTCU-3, DEBT-MTCU-4)" - status: pending + status: completed - id: refresh-delivery content: "Refresh delivery per the operator's choice (A: POST awaits and answers with the sheet or error; UI drops post-refresh poll); route and store tests (DEBT-MTCU-1)" - status: pending + status: completed - id: e2e-align content: "Align the e2e test to the UI merge: drop the profile push and the fixture profile; rerun (DEBT-MTCU-2 test half)" - status: pending + status: completed - id: schema-policy content: Document the additive-field serde(default) policy in shared-gateway-api; schema_version stays 1, no bump (DEBT-MTCU-4 discipline) - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-15-2-profiles-gate-local.md b/vibe/2026-09/2026-09-15-2-profiles-gate-local.md index 175aa8cde..8b53c31e1 100644 --- a/vibe/2026-09/2026-09-15-2-profiles-gate-local.md +++ b/vibe/2026-09/2026-09-15-2-profiles-gate-local.md @@ -4,46 +4,46 @@ overview: Redefine profiles so they select only local and STT models, serve ever todos: - id: config-semantics content: "gateway-config: validate local/STT-only members with remote-name error, never narrow models(), accept None selection; rewrite validation and schema tests" - status: pending + status: completed - id: boot-load content: "gateway: replace profile_switch.rs with boot-only boot_load.rs, drop drain/in_flight/switch mutex, LoadProfile boot-only and FIFO with ApplyConfig, runner assembles full remote routing" - status: pending + status: completed - id: config-version-0 content: config-version required value and messages to 0; first-run default; every literal under crates/, guide/src, and crate READMEs - status: pending + status: completed - id: remove-state-shadow content: gateway-config shadow.rs drops the state shadow and refuses active_profile in PUT bodies; gateway config_write/config_pending/config_apply lose state-shadow paths; config-pending envelope reads the state file - status: pending + status: completed - id: switch-route content: "gateway: switch-profile persists state (or deletes it for null) and answers JSON {profile, restart_required}; delete SSE shaping; unit tests" - status: pending + status: completed - id: stale-state-degrade content: "gateway-config: resolve_selection reports its source; stale state-file name boots with no profile and stale_state_selection(); clear_profile_state; runner warning; boot.rs IT" - status: pending + status: completed - id: workshop-none-profile content: workshop-protocol switch_profile name nullable; workshop-menu begin_switch Option; workshop-gateway switch_profile Option; Model menu No profile entry; tests - status: pending + status: completed - id: ui-select-route content: config UI selects through POST /admin/switch-profile (switcher, Set Active, banner, Active/Selected pills); tests - status: pending + status: completed - id: proxy-rule content: workshop-server forward_allowed becomes /admin/* except GET /admin/progress plus GET /v1/cache and digest-checked DELETE; rewrite admitted/refused test lists - status: pending + status: completed - id: apply-reload content: "gateway: census restart_required for profile/local_model/stt_model/stt; apply reloads remote routing merged with live locals; tests" - status: pending + status: completed - id: gateway-it content: "gateway IT: rewrite profiles.rs and boot.rs; restore wait_for_catalog in cloud_models.rs" - status: pending + status: completed - id: workshop-restart content: workshop-gateway JSON switch + GatewaySnapshot is_sidecar()/request_shutdown(); workshop-sessions restart ladder, LAN notice, 90 s bound; update IT mocks - status: pending + status: completed - id: config-ui content: "config UI: Profiles Available list local/STT only, drop client STT toast diff, rename overlay stage; tests" - status: pending + status: completed - id: docs-archdoc content: Guide chapters, READMEs, archdoc A5 rewrite, regenerate guide exports with build-user-guide; no version constants change - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-15-3-profiles-debt-removal.md b/vibe/2026-09/2026-09-15-3-profiles-debt-removal.md index 688635ed5..88d909a7e 100644 --- a/vibe/2026-09/2026-09-15-3-profiles-debt-removal.md +++ b/vibe/2026-09/2026-09-15-3-profiles-debt-removal.md @@ -4,16 +4,16 @@ overview: Remove the three debts the Profiles-gate-local plan (c17173ce..73b17e6 todos: - id: pgl-001-orphans-status content: "PGL-001: admin_orphans diffs against catalog_local_models/catalog_stt_models; admin_status configured closure reads catalog_local_models; fix orphans.rs docs and guide 11 line 35; add post-apply orphan and configured tests" - status: pending + status: completed - id: pgl-002-config-roundtrip content: "PGL-002: remove active_profile insertion from admin_config; drop boot_speech_tests strip; add GET-then-PUT round-trip test and config-store empty-diff UI test" - status: pending + status: completed - id: pgl-003-switch-in-flight content: "PGL-003: add switch_in_flight to WorkbenchSnapshot and snapshot(); thread through protocol.ts, workbench-service.ts, window-menu.ts; remove Known gap notes; update frames.rs, workbench-frames.mjs, and window-menu.mjs tests" - status: pending + status: completed - id: gate content: Run full regression list, regenerate guide exports with build-user-guide, confirm exit rg checks are empty - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-15-4-flat-sources.md b/vibe/2026-09/2026-09-15-4-flat-sources.md index a77a1691b..4e37b544f 100644 --- a/vibe/2026-09/2026-09-15-4-flat-sources.md +++ b/vibe/2026-09/2026-09-15-4-flat-sources.md @@ -4,13 +4,13 @@ overview: Add a flat-sources rule to promptforge/AGENTS.md (source subdirs need todos: - id: create-rule content: Add flat-sources bullet to Structural Rules in promptforge/AGENTS.md - status: pending + status: completed - id: flatten-crates content: Flatten all 19 non-compliant crates per the audit table and mechanics spec - status: pending + status: completed - id: verify content: Run the full workspace verification gate - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-15-5-workshop-menu-overhaul.md b/vibe/2026-09/2026-09-15-5-workshop-menu-overhaul.md index b862b02f1..80e5654e5 100644 --- a/vibe/2026-09/2026-09-15-5-workshop-menu-overhaul.md +++ b/vibe/2026-09/2026-09-15-5-workshop-menu-overhaul.md @@ -4,31 +4,31 @@ overview: Replace the workshop's 5-menu bar with Cursor's 8-menu bar (File, Edit todos: - id: services-registries content: DOM-free registries and stateful services under src/services/ (command, menu, context-key, keybinding parser/resolver/registry, action, quick-access, recent-files, text-control) with jsdom-free tests; re-export shims at the old ui/menu paths. See Technical Design. - status: pending + status: completed - id: menubar-widgets content: "ui/menu/menubar.ts + menu.ts: buttons generated from MenubarMainMenu, recursive flyouts, registry-derived shortcut labels, when/precondition/toggled rendering, title-bar left/center/right restructure. See Technical Design." - status: pending + status: completed - id: keybinding-dispatcher content: "ui/layout/keybinding-dispatcher.ts: capture-phase dispatch over the resolver, multi-stroke chords with pending status, claimed chords swallowed. See Technical Design." - status: pending + status: completed - id: editor-contribution content: "Editor: eager contribution with lazy run; all CodeMirror-backed commands from the catalog; EditorSettingsService with compartments; TextControl adapter; ':' quick-access provider; recent-files recording. See Functional Specification." - status: pending + status: completed - id: files-contribution content: "Files/workspace: openFile, openFolder, addRootFolder, saveAs, saveAll, revert; Open Recent provider rows; '' quick-access provider. See Functional Specification." - status: pending + status: completed - id: shell-contribution content: "Shell: edit commands over TextControlService; chrome, layout, status, agent, gateway contributions; menubar and stub tables; workbench.contributions.ts; remove window-menu.ts; main.ts cleanup; macOS traffic-light chrome (overlay title-bar style in the shell's macOS config, hidden custom controls and left inset in window-chrome). See Technical Design." - status: pending + status: completed - id: quick-input content: "ui/quickinput/: QuickInputService widget, '>' palette, '?' help, placeholder providers, the four quick-access actions. See Technical Design." - status: pending + status: completed - id: command-center content: "ui/chrome/command-center.ts: title-bar pill over MenuId.CommandCenter with WindowTitle. See Technical Design." - status: pending + status: completed - id: closing-sweep content: Remove shims, enable noUncheckedIndexedAccess, update ui/AGENTS.md and the panel-registry doc comment, run all gates. See Testing Plan. - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-16-1-turso-workspace-files.md b/vibe/2026-09/2026-09-16-1-turso-workspace-files.md index 27be1fef5..634468267 100644 --- a/vibe/2026-09/2026-09-16-1-turso-workspace-files.md +++ b/vibe/2026-09/2026-09-16-1-turso-workspace-files.md @@ -4,34 +4,34 @@ overview: "Make the Workshop's workspace a document: a single Turso database fil todos: - id: boundary content: "Land the shell boundary first: new workshop-server-api crate with the nine-item re-export surface, retarget the shell, enforce via the build-xtask product-boundary matrix; rename shared-sidecar to shared-gateway-discovery" - status: pending + status: completed - id: spike content: "Dependency spike: add turso 0.7.2, cargo deny, Windows build (no MSRV pin anymore - stable toolchain)" - status: pending + status: completed - id: workspace-file content: "Build workspace_file module in workshop-workspace: Turso-backed open/create/save-as/duplicate of the single anchor file, schema v1, meta validation" - status: pending + status: completed - id: workspace-integration content: Persist grant/revoke to the open workspace file; bulk-replace grants on open - status: pending + status: completed - id: routes content: Add /workspace/file/* HTTP endpoints in workshop-workspace handlers, registered via the Registry - status: pending + status: completed - id: pointer content: Last-workspace pointer in state_dir; boot reopens it before serving - status: pending + status: completed - id: spa-actions content: Replace the three File-menu stubs with real SPA actions (dialogs + fetch); recents integration - status: pending + status: completed - id: window-state content: "Shell: drop tauri-plugin-window-state, restore/save geometry through the server" - status: pending + status: completed - id: tests-docs content: Tests (module, routes, UI actions) and docs (READMEs, guide, module comments) - status: pending + status: completed - id: verify content: "CI gates plus manual smoke: save, quit, relaunch, open, duplicate; inspect with tursodb" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-16-2-workshop-ui-state.md b/vibe/2026-09/2026-09-16-2-workshop-ui-state.md index 74ec2318b..51dd0072e 100644 --- a/vibe/2026-09/2026-09-16-2-workshop-ui-state.md +++ b/vibe/2026-09/2026-09-16-2-workshop-ui-state.md @@ -4,22 +4,22 @@ overview: "Replace the Workshop SPA's browser localStorage, which is wiped on ev todos: - id: workspace-bucket content: "workshop-workspace: allow-listed kv keys layout/tree/closed_editors, generic put command through the actor, ui_state read at open, GET /workspace/file/state and PUT /workspace/file/state/{key}, tests" - status: pending + status: completed - id: user-bucket content: "New workshop-user-state crate: ui-state.json behind a mutex with atomic writes and tolerant reads, GET /user/state and PUT /user/state/{key}, registration in app.rs, tests and boot test" - status: pending + status: completed - id: spa-adapter content: "SPA: ui-storage.ts adapter (get/set per bucket, boot preload with timeout and defaults, suppress-while-applying guard); main.ts preloads before building stores" - status: pending + status: completed - id: spa-user-stores content: "SPA: editor-settings-service, zoom, recent-files-store, commands-history (as registry service) read initial from the adapter and write through it; drop Storage; tests" - status: pending + status: completed - id: spa-workspace-stores content: "SPA: layout-persistence, tree-state-service, closed editors read initial from the adapter and write through it; shared applyLayoutOrDefault; workshop-panel fetches listings for restored expanded folders; Open re-applies, Save As writes; localStorage grep test" - status: pending + status: completed - id: docs-gates content: Docs (crate READMEs, guide 'What persists', AGENTS.md rule made absolute, archdoc line), full gates, manual smoke - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-16-3-workspace-debt-removal.md b/vibe/2026-09/2026-09-16-3-workspace-debt-removal.md index 2fcfc7c96..92c492533 100644 --- a/vibe/2026-09/2026-09-16-3-workspace-debt-removal.md +++ b/vibe/2026-09/2026-09-16-3-workspace-debt-removal.md @@ -4,22 +4,22 @@ overview: "Remove the six accepted debts introduced by the 30 commits on top of todos: - id: item-a-wal-safety content: "Item A: same-path guard in Workspace::open_file, production close_backing(), shutdown BackgroundTaskAdapter registration, TWF-001 unit tests, TWF-002 integration test" - status: pending + status: completed - id: item-b-grant-order content: "Item B: per-grant position and added_at kept in memory; replace_all preserves, grant assigns, grant_rows orders by position; TWF-004 tests" - status: pending + status: completed - id: item-c-open-recent content: "Item C: workbench.action.openWorkspace optional path arg; .pfwork entries in open-recent.ts and quick access route to it in a 1_workspaces group; TWF-003 tests" - status: pending + status: completed - id: item-d-roots-load content: "Item D: TreeStateService.roots() memoized load shared by tree panel and WindowTitle; remove rootsGeneration; invalidateRoots first in applyOpenedWorkspaceState; retarget restore test; boot single-fetch and switch no-stale-render assertions" - status: pending + status: completed - id: item-e-docs content: "Item E: AGENTS.md two homes; 02-workbench.md zoom sentence; sidecar-while-open note and Open Recent wording in workspace-files.md and workshop-workspace README; grep assertions; mdbook build" - status: pending + status: completed - id: exit-gates content: "Exit: full gate list plus the five manual reproductions" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-17-1-promptforge-family-reorg.md b/vibe/2026-09/2026-09-17-1-promptforge-family-reorg.md index 36a0c6306..da8ac79e9 100644 --- a/vibe/2026-09/2026-09-17-1-promptforge-family-reorg.md +++ b/vibe/2026-09/2026-09-17-1-promptforge-family-reorg.md @@ -4,7 +4,7 @@ overview: Rename promptforge-api to promptforge-api-runtime, rename shared-promp todos: - id: single-commit content: "One commit: move, rename, re-export, privacy rule, docs, verification" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-17-2-gateway-family-reorg.md b/vibe/2026-09/2026-09-17-2-gateway-family-reorg.md index 176d77a06..9c384d081 100644 --- a/vibe/2026-09/2026-09-17-2-gateway-family-reorg.md +++ b/vibe/2026-09/2026-09-17-2-gateway-family-reorg.md @@ -4,7 +4,7 @@ overview: Apply the promptforge container shape to the gateway family: rename sh todos: - id: single-commit content: "One commit: structural-check updates, moves, renames, sweeps, path fixes, docs, verification" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-17-3-gateway-app-decomp.md b/vibe/2026-09/2026-09-17-3-gateway-app-decomp.md index b216bd081..1eb96ef9e 100644 --- a/vibe/2026-09/2026-09-17-3-gateway-app-decomp.md +++ b/vibe/2026-09/2026-09-17-3-gateway-app-decomp.md @@ -4,25 +4,25 @@ overview: "Split crates/gateway/app/src/lib.rs (4,199 lines: route table, handle todos: - id: extract-relay content: Extract relay.rs, speech.rs, models.rs with their test siblings from lib.rs; test siblings import shared fixtures from test_support.rs instead of copying them - status: pending + status: completed - id: extract-admin content: Create admin.rs + admin/ directory (profiles, status, progress, config, queue) with test siblings; test siblings import shared fixtures from test_support.rs - status: pending + status: completed - id: absorb-auth-boot-tray content: Move auth helpers into auth.rs, boot/tray/loopback test modules to kebab siblings; siblings import shared fixtures from test_support.rs - status: pending + status: completed - id: rewire-router content: Update build_router mounts to module paths; shrink lib.rs to docs, state, route table, tiny glue handlers; state the handler-placement rule in the crate docs - status: pending + status: completed - id: consolidate-fixtures content: Dedup the remaining fixture definitions crate-wide (shutdown.rs, handoff.rs, orphans.rs, commands.rs, config_apply.rs, boot_load-tests.rs); move auth.rs's parameterized state(trust_loopback) into test_support.rs as the configurable entry point behind a default state() wrapper - status: pending + status: completed - id: verify content: Run the verification battery; test counts before and after must match; one commit - status: pending + status: completed - id: function-shape-idioms content: Apply the six function-shape idioms from the field study (snapshot extractor, auth extractor, Json rejection, cancellation policy, error extensions, agent error fields) as a separate commit after the decomposition lands - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-17-4-ci-repair-state-accessors.md b/vibe/2026-09/2026-09-17-4-ci-repair-state-accessors.md index 0a7693b57..ede8a692f 100644 --- a/vibe/2026-09/2026-09-17-4-ci-repair-state-accessors.md +++ b/vibe/2026-09/2026-09-17-4-ci-repair-state-accessors.md @@ -4,25 +4,25 @@ overview: Fix the three CI failures from the family reorg (stale crate names in todos: - id: ci-package-name content: "ci.yml: rename shared-gateway-discovery to gateway-api-discovery (both jobs), add shell: bash to both ownership-race steps" - status: in_progress + status: completed - id: ci-shared-vfs content: Remove workspace-hack from shared-vfs, exclude it in hakari.toml, verify hakari - status: pending + status: completed - id: ci-ui-fixtures content: Fix gateway-stt fixture paths in four workshop-server/ui test files - status: pending + status: completed - id: ci-verify content: Part 1 verification battery, commit CI repair - status: pending + status: completed - id: snapshot-core content: Add focused AppState accessors (routing, config, profile_name, model_allowlist, is_loading, local_children) in lib.rs - status: pending + status: completed - id: snapshot-sweep content: Convert single-field read sites to accessors; keep scoped guards for atomic multi-field reads and writers - status: pending + status: completed - id: snapshot-verify content: Part 2 verification battery, commit - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-17-5-workshop-family-reorg.md b/vibe/2026-09/2026-09-17-5-workshop-family-reorg.md index 1c895dba6..7d1b5692a 100644 --- a/vibe/2026-09/2026-09-17-5-workshop-family-reorg.md +++ b/vibe/2026-09/2026-09-17-5-workshop-family-reorg.md @@ -4,22 +4,22 @@ overview: Move the twelve workshop-* crates into a manifestless, fully private c todos: - id: harness content: build-xtask product.rs gains crates/workshop/ fully-private container rule + fixtures; sync architecture test matrix - status: in_progress + status: completed - id: moves content: git mv the twelve crates into crates/workshop/ with short dir names - status: pending + status: completed - id: manifests content: Root Cargo.toml members/exclude/workspace.dependencies - status: pending + status: completed - id: paths content: "Path literals: gitignore, package.json + lockfile, chat_gate.rs, gateway build.rs icon, sidecar tool, 6 workflow files, README, document.md" - status: pending + status: completed - id: docs content: AGENTS.md root rule + moved docs sweep - status: pending + status: completed - id: verify content: Cargo.lock regen, full verification battery, one commit - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-17-6-minimal-run-window.md b/vibe/2026-09/2026-09-17-6-minimal-run-window.md index b64849bf7..6c1019116 100644 --- a/vibe/2026-09/2026-09-17-6-minimal-run-window.md +++ b/vibe/2026-09/2026-09-17-6-minimal-run-window.md @@ -4,28 +4,28 @@ overview: "A bare-minimum Run window for the Workshop SPA: a Run menu item opens todos: - id: reexports content: Add contract type re-exports to promptforge-api-runtime parser.rs - status: pending + status: completed - id: route content: POST /prompts/contract route + hand-written contract DTO in workshop-workspace (handlers-prompts.rs) + promptforge-api-runtime dep edge - status: pending + status: completed - id: route-tests content: "Server route tests: 200 full contract, 422 with line number, implicit prose args" - status: pending + status: completed - id: panel content: "SPA ui/run/ concern: panel, states, rows, run-api.ts, RunTab shimmer title, shared-ui shimmer.css, panel-registry/zones wiring" - status: pending + status: completed - id: menu content: workbench.action.newRunWindow menu action with active-editor pre-fill - status: pending + status: completed - id: dnd content: Tree drag-out (application/x-workshop-path) + OS drop-target hook in workspace-drops.ts - status: pending + status: completed - id: spa-tests content: "SPA tests: empty open, pre-filled open, tree drag, parse error row" - status: pending + status: completed - id: zones-fix content: "Zone stability: size memory + group resurrection with placeholder panel in zones.ts, persisted across relaunch" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-18-1-single-rustls-backend.md b/vibe/2026-09/2026-09-18-1-single-rustls-backend.md index d27dd8c85..a9e51d278 100644 --- a/vibe/2026-09/2026-09-18-1-single-rustls-backend.md +++ b/vibe/2026-09/2026-09-18-1-single-rustls-backend.md @@ -4,37 +4,37 @@ overview: Move the PromptForge workspace from reqwest 0.12 (ring) to reqwest 0.1 todos: - id: manifest content: Change root Cargo.toml reqwest pin to 0.13 with json, query, rustls, stream and the invariant comment - status: pending + status: completed - id: compile content: cargo check --workspace --all-features and fix any reqwest 0.13 API fallout - status: pending + status: completed - id: comments content: Update stale 0.12 / rustls-tls / webpki-roots comments in gateway-local, workshop shell manifest, deny.toml - status: pending + status: completed - id: hakari content: cargo hakari generate, manage-deps, verify; confirm ring left workspace-hack - status: pending + status: completed - id: invariant content: "cargo tree checks: no ring in gateway, workshop, promptforge-api-runtime; single reqwest version" - status: pending + status: completed - id: ci content: Add the ring guard step to the supply-chain CI job, failing on cargo tree error as well as on a ring hit - status: pending + status: completed - id: docs content: "Update vibe/dependency-surface.md: resolve Finding 1, refresh measured counts and header commit" - status: pending + status: completed - id: gates content: Run clippy, headless gateway check, nextest, deny, build-xtask, fmt for the rustls slice; commit - status: pending + status: completed - id: toml-bump content: "toml slice: move the workspace toml pin from 0.8 to 1; fix any API fallout in gateway-config, gateway app, workshop-support, build-xtask" - status: pending + status: completed - id: toml-verify content: "toml slice: check shadow.rs round-trip and rendered-TOML fixtures; cargo tree confirms toml 0.8, toml_edit 0.19, winnow 0.5 are gone; hakari regenerate" - status: pending + status: completed - id: toml-gates content: "toml slice: rerun the gates, update dependency-surface.md, commit" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-18-2-remove-tool-picker.md b/vibe/2026-09/2026-09-18-2-remove-tool-picker.md index 2ff1a225b..65bde4dc6 100644 --- a/vibe/2026-09/2026-09-18-2-remove-tool-picker.md +++ b/vibe/2026-09/2026-09-18-2-remove-tool-picker.md @@ -7,16 +7,16 @@ todos: status: completed - id: consumers content: Remove the picker, fuzzy slots, conflicts, the Plugin lint, and the models.bind resolver from model-client, lua, parser, runtime, and workshop-workspace - status: pending + status: completed - id: crate content: Delete the crate; prune root manifest members and the candle/tokenizers/hf-hub/safetensors/half pins; regenerate hakari; fix nextest filters; remove the CI model cache - status: pending + status: completed - id: ui-docs content: Remove fuzzy tool rendering from the Workshop SPA; update guide sources, crate READMEs, and dependency-surface.md with fresh measurements - status: pending + status: completed - id: verify content: Full gate list, cargo tree invariants, and reference sweep - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-18-3-empty-zone-groups.md b/vibe/2026-09/2026-09-18-3-empty-zone-groups.md index bd1fc6ebe..63d1ec96c 100644 --- a/vibe/2026-09/2026-09-18-3-empty-zone-groups.md +++ b/vibe/2026-09/2026-09-18-3-empty-zone-groups.md @@ -4,10 +4,10 @@ overview: "Replace the Workshop's placeholder panel with dockview's native empty todos: - id: upgrade-dockview content: Upgrade dockview and dockview-core to 8.3.1 in the SPA lockfile and prove the existing suite still passes - status: pending + status: completed - id: empty-zone-groups content: Rebuild zones with dock.addGroup on the mutation boundary, remove the placeholder panel type and its artifacts, bump the layout schema, rewrite the zone tests, refresh the workbench guide - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-18-4-sans-io-engine-harness.md b/vibe/2026-09/2026-09-18-4-sans-io-engine-harness.md index bf43b8e3d..07b0a4e95 100644 --- a/vibe/2026-09/2026-09-18-4-sans-io-engine-harness.md +++ b/vibe/2026-09/2026-09-18-4-sans-io-engine-harness.md @@ -4,31 +4,31 @@ overview: "Turn the PromptForge executor into a sans-IO Engine (a deterministic todos: - id: structural-guards content: "Steps 1-3: Harness family row in the product matrix; Engine manifest guard, retired-symbol scan, and Harness clippy-ban check as fixture-tested functions" - status: pending + status: completed - id: harness-scaffolding content: "Steps 4-5: harness-api door with its type surface and set_gateway; skeleton crates under crates/harness with the tagged spawn wrapper" - status: pending + status: completed - id: harness-log content: "Steps 6-7: Turso run log schema, append path, and read path" - status: pending + status: completed - id: lua-loop content: "Steps 8-14: scheduler and protocol file splits; structured error values; Chat and tool_call arms; models.loop in Lua and the Rust loop deleted; Checkpoint 1" - status: pending + status: completed - id: tasks-fanout content: "Steps 15-21: hierarchical ids; task arena, spawn, chain-end rules, waits, timeouts; fanout in Lua and the join machinery deleted; Checkpoint 2" - status: pending + status: completed - id: model-tasks content: "Steps 22-24: model task origin and built-ins; notices and await_tasks; Checkpoint 2b" - status: pending + status: completed - id: run-api-purity content: "Steps 25-40: vocabulary types, sync CancelHandle, events as values, effects behind an internal table, Run::step/resume, drivers and determinism tests, RunContext inputs, bindings by id, bridge and interim driver, transport and Plugin moves, dependency-free manifests, guards live; Checkpoint 3" - status: pending + status: completed - id: harness-runner content: "Steps 41-44: performer traits and the effect loop; performers; run preparation; Checkpoint 4" - status: pending + status: completed - id: harness-sessions content: "Steps 45-51: session pieces moved; lifecycle and effective_interrupt; input registry; Session and Harness runtime; Workshop on harness-api and workshop-sessions deleted; docs and Papergate note; Checkpoint 5" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-19-1-chatbox-extraction.md b/vibe/2026-09/2026-09-19-1-chatbox-extraction.md index 6151a775d..a92d4f057 100644 --- a/vibe/2026-09/2026-09-19-1-chatbox-extraction.md +++ b/vibe/2026-09/2026-09-19-1-chatbox-extraction.md @@ -4,19 +4,19 @@ overview: In the promptforge2 checkout (c:/Users/Vinnie/cursor/promptforge2, bra todos: - id: step-1 content: "Step 1: relocate - git mv crates/workshop/server/ui to crates/workshop/ui (build_sibling, build.rs, drift test, build.mjs depth, shared-ui link + npm install, parent-walk tests, gitignore, gitattributes, workflows, workshop-spa, docs, doc comments; dist compare between moves), then git mv src/ui to src/parts (main.ts, panel-registry.ts, test stdin paths, AGENTS.md); one commit, npm test + cargo build + build-ui drift green" - status: pending + status: completed - id: step-2 content: "Step 2: chat box foundation - git mv composer files into parts/chatbox/, types.ts, @tiptap/suggestion, mention node attrs kind/icon/preview/tone/data with renderHTML/parseHTML, chip-view.ts renderChip with data-kind, chat-box-view.ts renderDraft; gate typecheck + node --test chat-box/typeahead-popup/mention-chip" - status: pending + status: completed - id: step-3 content: "Step 3: dictation ownership - SpeechCaptureService owner tokens/busy/owner/onOwnerChange; setupStt press/state/onState, ownership filter, blocked precedence; stt-stream two-instance test; eight-line interim view adaptation; gate typecheck + node --test speech-capture/stt-stream/agent-stt/agent-stt-boot" - status: pending + status: completed - id: step-4 content: "Step 4: ChatBox class (defaulted props, update, controls slot, buttons, strip, data attributes, textControls) and the agent-session-view rewire in one commit; migrate chat-box, agent-session-view, agent-stt, agent-stt-boot tests; two-view data-mic test; gate typecheck + those node --test files + lazy-panel-sizing" - status: pending + status: completed - id: step-5 content: "Step 5: seams (mentionSource/signal/debounce, commandSource, onPasteFiles, serialize/restore v1, insertMention), typeahead description/groups/loading/Tab, test/chatbox-boundary.mjs; then the deferred full verification - npm test, npm run build, cargo build, chat_gate nextest, build-ui drift, cargo fmt, clippy, clean tree; operator smoke pass reported" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-20-1-chatbox-debt-removal.md b/vibe/2026-09/2026-09-20-1-chatbox-debt-removal.md index 7c27dd6be..4ca9dad24 100644 --- a/vibe/2026-09/2026-09-20-1-chatbox-debt-removal.md +++ b/vibe/2026-09/2026-09-20-1-chatbox-debt-removal.md @@ -4,10 +4,10 @@ overview: Remove the two debts the ChatBox extraction (37be5c6d..bbff0144 in pro todos: - id: step-1 content: "Step 1 (CBX-001): renderChip removable option, strip and static renderer pass false, reserved docstrings, restored-strip assertion" - status: pending + status: completed - id: step-2 content: "Step 2 (CBX-002): delete 11 dead crates/workshop/*/ui/package-lock.json globs in 4 workflow files; fix test/mention-chip.mjs header" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-20-1-harness-debt-removal.md b/vibe/2026-09/2026-09-20-1-harness-debt-removal.md index 00f0d534b..29f65a389 100644 --- a/vibe/2026-09/2026-09-20-1-harness-debt-removal.md +++ b/vibe/2026-09/2026-09-20-1-harness-debt-removal.md @@ -4,10 +4,10 @@ overview: "Remove the two debts the Debt Collector accepted from the sans-io Eng todos: - id: settle-on-run-end content: Settle every live task with AbandonReason::RunTerminated before the scheduler tears the run down, with one cancel-mid-task regression test - status: pending + status: completed - id: typed-failure-boundary content: Carry FailureKind plus message on the Harness error channel, match on the kind in workshop-server, re-pin the status test - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-20-2-gateway-api-types-progress.md b/vibe/2026-09/2026-09-20-2-gateway-api-types-progress.md index 2ee388994..477531202 100644 --- a/vibe/2026-09/2026-09-20-2-gateway-api-types-progress.md +++ b/vibe/2026-09/2026-09-20-2-gateway-api-types-progress.md @@ -4,22 +4,22 @@ overview: Rename gateway-api to gateway-api-types; collapse progress to a busy f todos: - id: types-crate content: Rename gateway-api to gateway-api-types, add Progress, repoint dependents, update PUBLIC_GATEWAY, delete architecture.rs and its CI step - status: pending + status: completed - id: shared-ui-barberpole content: shared-ui status bar barberpole (144px, left of LEDs), setBusy(bool), tokens - status: pending + status: completed - id: gateway-progress-rewrite content: Rewrite shared-progress to ProgressHub::begin/Activity::set_text over watch; migrate every gateway producer; SSE, /admin/status, tray, config-ui; delete render.rs - status: pending + status: completed - id: workshop-busy content: workshop-gateway decodes Progress and pushes busy/idle; StatusBarUpdate.busy; push_busy; remove hub, renderer, ProgressMeter use; SPA setBusy - status: pending + status: completed - id: move-and-docs content: Move shared-progress to crates/gateway/progress as gateway-progress; AGENTS.md, README, Harness Invariants, crate AGENTS.md trims - status: pending + status: completed - id: mandatory-marker content: tidy.rs selects workshop-*/harness-* by name and requires the marker; mark and split Harness web crates; scope the 500-line sentence - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-20-2-rust-rulebook-sweep.md b/vibe/2026-09/2026-09-20-2-rust-rulebook-sweep.md index 785bd7fb1..168401eec 100644 --- a/vibe/2026-09/2026-09-20-2-rust-rulebook-sweep.md +++ b/vibe/2026-09/2026-09-20-2-rust-rulebook-sweep.md @@ -4,43 +4,43 @@ overview: Apply the Rust rulebook across the promptforge workspace on master, on todos: - id: step-1 content: "Step 1 (ci-baseline): gate the child's store write in cancel_ends_a_parked_task_idempotently; 20 consecutive passes" - status: pending + status: completed - id: step-2 content: "Step 2 (async-fs): spawn_blocking for workshop-workspace fs work and harness-sessions Runtime::launch; move UI_STATE_VALUE_CAP re-export" - status: pending + status: completed - id: step-3 content: "Step 3 (emitter-flags): DebugMode and OutputTrust replace Emitter bool params" - status: pending + status: completed - id: step-4 content: "Step 4 (error-model): strip {source} from 15 Display sites and rewire renderers; newtype 5 leaked foreign error types with manual From; 13 message fixes" - status: pending + status: completed - id: step-5 content: "Step 5 (error-model): #[non_exhaustive] on 10 error enums, ~17 wire enums, events! macro, response/event structs" - status: pending + status: completed - id: step-6 content: "Step 6 (error-model): attach causes at ~62 non-lua .map_err(|_|) sites; report sites left" - status: pending + status: completed - id: step-7 content: "Step 7 (layout-lint-hygiene): retire 11 mod.rs files by pure moves, form per three-file rule" - status: pending + status: completed - id: step-8 content: "Step 8 (layout-lint-hygiene): allow->expect with reason, 3 test-only re-exports, 2 doc unwraps, vm.rs fences, get_all rename, must_use, doubled cfg" - status: pending + status: completed - id: step-9 content: "Step 9 (layout-lint-hygiene): hollow parser lib.rs into modules; shared-loopback if over 500; rename types.rs and wire/shared.rs" - status: pending + status: completed - id: step-10 content: "Step 10 (paused-time): 5 sentinels to pending(), start_paused on ~14 in-process tests, Workspace clock injection for wait_past" - status: pending + status: completed - id: step-11 content: "Step 11 (docs-prose): gateway family summaries to third person and //! lines" - status: pending + status: completed - id: step-12 content: "Step 12 (docs-prose): promptforge and Harness families summaries and //! lines" - status: pending + status: completed - id: step-13 content: "Step 13 (docs-prose): workshop, shared, build summaries and //! lines; FULL verification block" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-20-4-debt-removal.md b/vibe/2026-09/2026-09-20-4-debt-removal.md index 58bf716d0..1b89e7170 100644 --- a/vibe/2026-09/2026-09-20-4-debt-removal.md +++ b/vibe/2026-09/2026-09-20-4-debt-removal.md @@ -4,25 +4,25 @@ overview: Remove six accepted debts added across the 30 commits in upstream/mast todos: - id: buildxtask-enumeration content: Unify the build-xtask crate enumeration on one recursive walk and make the marker check report unreadable or nameless manifests instead of skipping - status: pending + status: completed - id: error-chain-guards content: Add the dedup predicate and empty-text guard to both gateway error_chain renderers with a regression test each - status: pending + status: completed - id: event-exhaustiveness content: Add the every-Event-variant and every-TaskOrigin drive tests and correct the docs that assert an unconditional guarantee - status: pending + status: completed - id: walled-tier-inversion content: Split admin/walled/handoff.rs at its cfg seam, rehome the five non-route items to homes named for what they are, and repoint the import sites - status: pending + status: completed - id: walled-tier-ratchet content: Add a build-xtask rule forbidding admin::walled paths outside the tier, allowlisting the three assembly files and skipping comments - status: pending + status: completed - id: error-source-gateway content: Create the shared error-source crate with accessors and collapse the gateway family's newtypes onto it - status: pending + status: completed - id: error-source-harness-workshop content: Collapse the Harness and workshop newtypes onto the shared types, record the new component, and run the wide gates - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-22-1-replayed-tool-calls.md b/vibe/2026-09/2026-09-22-1-replayed-tool-calls.md index a5397a8f6..5399851fd 100644 --- a/vibe/2026-09/2026-09-22-1-replayed-tool-calls.md +++ b/vibe/2026-09/2026-09-22-1-replayed-tool-calls.md @@ -4,22 +4,22 @@ overview: Make the Engine's projection render replayed assistant tool_calls in t todos: - id: audit-consumers content: Audit projected-tool_calls consumers and goldens/fixtures; verify the arguments-always-object invariant at every construction path - status: pending + status: completed - id: fix-projection content: Render OpenAI wire shape in wire_message (projection.rs); update the function doc, the module doc, and the ToolCallRecord/Message docs - status: pending + status: completed - id: update-tests content: Update the two existing projection tests and add a wire-shape pin whose assertion re-decodes the arguments string - status: pending + status: completed - id: e2e-openai-shape content: Add an api-runtime end-to-end test whose ScriptedGateway validates inbound tool_calls against the OpenAI schema - status: pending + status: completed - id: gateway-diagnostics content: Log the bounded, escaped upstream error code/type in OpenAiUpstream::post (not the raw body) - status: pending + status: completed - id: verify content: Run promptforge-lua, promptforge-model-client, promptforge-api-runtime, gateway-protocol, and the gateway app suite; then the papergate or local reproduction - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-22-2-emit-infer-reply.md b/vibe/2026-09/2026-09-22-2-emit-infer-reply.md index ed52ac18a..21cb6b1d2 100644 --- a/vibe/2026-09/2026-09-22-2-emit-infer-reply.md +++ b/vibe/2026-09/2026-09-22-2-emit-infer-reply.md @@ -4,25 +4,25 @@ overview: Report tool-less `models.infer` turns as an `assistant_reply` carrying todos: - id: api-types-variant content: Add ReplyOrigin and the AssistantReply origin field, give Emitter::assistant_reply an origin parameter, and cover the round-trip in promptforge-api-types - status: pending + status: completed - id: hoist-metrics content: Hoist call_metrics to execute/support.rs as pub(crate) and re-point chat.rs - status: pending + status: completed - id: emit-infer-reply content: Capture model/metrics/thinking before the result move, then emit thinking and an infer-origin assistant_reply from accept_infer_completion; update its doc comment - status: pending + status: completed - id: observer-forwarding content: Add origin to on_assistant_reply on the test_support observer (dropping on_infer_reply) and forward it - status: pending + status: completed - id: session-parity content: "Give infer-origin replies session parity: one AssistantReply arm covers both origins in settle_current_turn and reply_stamp in session.rs" - status: pending + status: completed - id: tests content: Add api-runtime (execute/tests) and sessions (session-tests.rs + tests/it/session.rs) tests, including the reply-index sequence and a chat-origin assertion - status: pending + status: completed - id: verify content: Run crate test suites and a local tool-less infer reproduction - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-22-3-replay-json-fidelity.md b/vibe/2026-09/2026-09-22-3-replay-json-fidelity.md index b741acfa3..8aefe502d 100644 --- a/vibe/2026-09/2026-09-22-3-replay-json-fidelity.md +++ b/vibe/2026-09/2026-09-22-3-replay-json-fidelity.md @@ -4,28 +4,28 @@ overview: Make every payload the run log stores round-trip bit-for-bit (exact fl todos: - id: float-roundtrip content: Enable serde_json float_roundtrip on the workspace dependency; regenerate workspace-hack with cargo hakari generate; hakari verify passes - status: pending + status: completed - id: round-timings content: Round ClientTiming fields to microseconds in model-client read.rs (duration_ms + mean_itl_ms) with a bit-exact round-trip test - status: pending + status: completed - id: log-fidelity-test content: "Add harness-log tests/it/fidelity.rs covering awkward floats, key order, nested Value, real AssistantReply metrics; add the round-trip bullet to harness-log ## Invariants" - status: pending + status: completed - id: sort-key-extract content: Extract SortKey and sort_key helper from collection.rs into a shared pub(crate) module; collection_members reuses it - status: pending + status: completed - id: sorted-pairs content: Add install_deterministic_iteration replacing pairs/next with sorted iteration (honor __pairs, skip cleared keys); call after harden in SectionVm::new; tests - status: pending + status: completed - id: pairs-audit content: Audit Rust-side table.pairs() walks that produce ordered Vecs; sort or record each site - status: pending + status: completed - id: agents-rule content: Add the one-line round-trip rule to AGENTS.md Engineering; add the sorted pairs sentence to the user guide - status: pending + status: completed - id: verify content: fmt, clippy, hakari verify, build-xtask, docs, full nextest + doctests, 20x loop of the end_to_end test, mdbook build - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-22-4-api-runtime-debt.md b/vibe/2026-09/2026-09-22-4-api-runtime-debt.md index 64269eb42..7f8e5d188 100644 --- a/vibe/2026-09/2026-09-22-4-api-runtime-debt.md +++ b/vibe/2026-09/2026-09-22-4-api-runtime-debt.md @@ -4,37 +4,37 @@ overview: "Reduce technical debt in promptforge-api-runtime without changing beh todos: - id: docs content: "Docs accuracy: README badges/version, lib.md, stale lua/untrusted docs, execute.rs module map, test_support voice, legacy comments, crate AGENTS.md, Run step/resume contract docs" - status: pending + status: completed - id: facade content: "Root facade: make execute private, root-only re-exports, rewrite dependent imports and doctests, resolve unreachable_pub and private_intra_doc_links" - status: pending + status: completed - id: modelseams content: "Model seams: drop Applied/SseScanner/StreamAccumulator hidden re-exports (keep ToolSchemaError and the transport seams)" - status: pending + status: completed - id: demotions content: "Demotions: apply each demotion candidate that meets its condition, with the accessor it is paired with, and leave the rest public" - status: pending + status: completed - id: testsupport content: "Trim test_support pub surface to the listed names (recording keeps Observer and Observation), delete RecordingObserver, keep TokioDriver/mock/hooks cfg(test)" - status: pending + status: completed - id: movetests content: "Move args_surface, lazy_prose, offline exec_flow cases to tests/suite and declare them in tests/suite/main.rs" - status: pending + status: completed - id: splittests content: "Split tests/scheduler.rs and tests.rs along identified seams (tests/scheduler/ in standard layout; plain modules under tests/)" - status: pending + status: completed - id: answerinline content: "Hoist answer_inline into core Scheduler and replace the 16 hand-written sites across 11 modules" - status: pending + status: completed - id: testhost content: "Remove cfg(test) test_host from RunContext/RunState and the RunContext test builders; build RunHost in tests and pass it to TokioDriver" - status: pending + status: completed - id: substtests content: "Move subst.rs test block to subst-tests.rs" - status: pending + status: completed - id: verify content: "Record baseline test count with --all-features; run the crate inner loop after each work item; before finishing run clippy, nextest + doctests, cargo doc -D warnings, fmt, build-xtask and confirm the count is not lower" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-23-1-promptforge-api-firewall.md b/vibe/2026-09/2026-09-23-1-promptforge-api-firewall.md index 21e0bd8be..99601e64e 100644 --- a/vibe/2026-09/2026-09-23-1-promptforge-api-firewall.md +++ b/vibe/2026-09/2026-09-23-1-promptforge-api-firewall.md @@ -4,43 +4,43 @@ overview: Replace promptforge-api-runtime with a pure re-export facade, package todos: - id: step-1 content: "Step 1: record the baseline, rename crates/promptforge/ to crates/promptforge-internal/, update build-xtask (ENGINE_CONTAINER, container exception key, family() exact match)" - status: pending + status: completed - id: step-2 content: "Step 2: surface audit (non-test vs test-only uses), create the facade package promptforge with auto-trait tests, add TRANSITIONAL_CONTAINER_EXCEPTIONS" - status: pending + status: completed - id: step-3 content: "Step 3: migrate every outside crate to the facade; rewrite outside tests that need unjustified items" - status: pending + status: completed - id: step-4 content: "Step 4: move the types and runtime crates into the container, move the suite and bench to the facade, delete the transitional exception" - status: pending + status: completed - id: step-5 content: "Step 5: drop the types-to-vfs edge, fold shared-vfs into promptforge-vfs, keep its no-dependencies guarantee" - status: pending + status: completed - id: step-6 content: "Step 6: facade shape check" - status: pending + status: completed - id: step-7 content: "Step 7: cargo xtask api (rustdoc JSON, closure, link targets, doc text, listing) with fixtures" - status: pending + status: completed - id: step-8 content: "Step 8: move engine-only operations into detail functions, including ToolId::from_validated" - status: pending + status: completed - id: step-9 content: "Step 9: clear all 78 doc(hidden), document the transport codec, gate test helpers, turn on the ban" - status: pending + status: completed - id: step-10 content: "Step 10: surface docs name only facade paths (intra-doc links, doc text, doc examples)" - status: pending + status: completed - id: step-11 content: "Step 11: minimality pass (non-test uses only), bless public-api.txt, add the CI job" - status: pending + status: completed - id: step-12 content: "Step 12: facade lib.md and topic docs, facade docs gate" - status: pending + status: completed - id: step-13 content: "Step 13: update AGENTS.md, sweep old names, run the exit gates" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-24-1-debt-removal-api-firewall.md b/vibe/2026-09/2026-09-24-1-debt-removal-api-firewall.md index ae8d93431..c900bbcd4 100644 --- a/vibe/2026-09/2026-09-24-1-debt-removal-api-firewall.md +++ b/vibe/2026-09/2026-09-24-1-debt-removal-api-firewall.md @@ -4,16 +4,16 @@ overview: "Remove the four debts the promptforge API firewall commits (75245481. todos: - id: step-1 content: "Step 1 (chunk-names): repoint TASKS/FANOUT/MESSAGES chunk names to crates/promptforge-internal/lua/src/; content-match tests in coro-tests.rs and messages-tests.rs" - status: pending + status: completed - id: step-2 content: "Step 2 (test-support): harness-plugins suite on a store-only Run::step/resume loop; Engine drive_tokio example on Engine paths; delete facade test-support feature, module, and page; single-build xtask api; remove facade_shape cfg allowance; sweep ci.yml and AGENTS.md" - status: pending + status: completed - id: step-3 content: "Step 3 (doctest-cycle): doctest-only facade dev-dependency exception in AGENTS.md line 35; amend the six crate statements" - status: pending + status: completed - id: step-4 content: "Step 4 (listing-fidelity): non_exhaustive, kind, and private-field markers in api/render.rs; pinned-nightly fixtures; re-bless public-api.txt; exit gates" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-24-2-issues-69-59-70.md b/vibe/2026-09/2026-09-24-2-issues-69-59-70.md index 8c97579db..be3a06304 100644 --- a/vibe/2026-09/2026-09-24-2-issues-69-59-70.md +++ b/vibe/2026-09/2026-09-24-2-issues-69-59-70.md @@ -4,13 +4,13 @@ overview: "Fix three promptforge issues, one commit each on vibe2. Issue 69: a s todos: - id: step-1 content: "Step 1 (models-use-options, issue 69): before the run, add the close-line convention to .cursor/rules/commit.mdc in the workspace; ModelBinding::with_invocation and re-bless public-api.txt; models.use options table decoded and validated in lua/src/models.rs, applied in resolve_model_binding, handle over the adjusted binding; Lua-crate and Engine tests; 06-models.md and 03-chat-rounds.md edits and guide regeneration; draft PR text; commit with `close #69`" - status: pending + status: completed - id: step-2 content: "Step 2 (tray-tint-gating, issue 59): cfg(any(windows, linux, test)) on grayed, error_tint, and tint in crates/gateway/app/src/tray/logic.rs with a doc note naming the callers; gateway tests, clippy, and headless check; commit with `close #59`; the user runs the Mac clippy check" - status: pending + status: completed - id: step-3 content: "Step 3 (thinking-and-stream-deadlines, issue 70): no-thinking fails only on Always in fill.rs; per-receive tokio deadline in crates/harness/models/src/transport.rs keeping the ClientTimeout marker; RunLimits and guide docs; prepare and timing tests; full exit gates; commit with `close #70`" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-24-2-workshop-crates-cleanup.md b/vibe/2026-09/2026-09-24-2-workshop-crates-cleanup.md index 486841803..10354484e 100644 --- a/vibe/2026-09/2026-09-24-2-workshop-crates-cleanup.md +++ b/vibe/2026-09/2026-09-24-2-workshop-crates-cleanup.md @@ -4,55 +4,55 @@ overview: Cleanup of crates/workshop in the promptforge repository (branch maste todos: - id: baseline content: "Baseline: record test results on master before any change" - status: pending + status: completed - id: bind content: "Bind: refuse non-loopback addresses in reuse_bind, with a test" - status: pending + status: completed - id: dead-code content: "Dead code: delete WorkshopObserver and StatusBus helpers, drop unused server re-exports, fix gateway description and a stale comment" - status: pending + status: completed - id: shell-rename content: "Shell rename: gateway icon copies, shell/ to desktop/, tier and constant renames, prose, UI names (desk, view, placeholder, entry bundle), vocabulary in AGENTS.md" - status: pending + status: completed - id: config-ui content: "config-ui: views/ to pages/, page identifiers, shell to desk" - status: pending + status: completed - id: flaky-tests content: "Flaky tests: symlink tests fail under CI, replace fixed sleeps" - status: pending + status: completed - id: save-timeout content: "Save timeout: reproducing test, JSON 408 body on every deadline route, unknown-token state in the editor, audit of UI error consumers" - status: pending + status: completed - id: security-tests content: "Security tests: realtime relay refusals, jail edge cases" - status: pending + status: completed - id: wire-fixture content: "Wire fixture: shared /ws frame fixture for Rust and TypeScript, SelectModelFrame" - status: pending + status: completed - id: registry-docs content: "Registry docs: keep subsystem-named traits, reword claims, record runtime links in the registry crate docs" - status: pending + status: completed - id: code-docs content: "Code-level docs: fix code-comment drift; make the AGENTS.md import pointer explicit" - status: pending + status: completed - id: prompts-route content: "Prompts route: move /prompts/contract into workshop-server, drop workspace Engine dependency" - status: pending + status: completed - id: helpers content: "Helpers: move render_message, JSON bucket validator, mock server helper, UI reconnect backoff into shared homes" - status: pending + status: completed - id: layout content: "Layout: directories for hyphenated groups of three or more, distinct ui_state names" - status: pending + status: completed - id: renames content: "Renames: remove server aliases, move /ws socket to a workshop_socket module, rename status relay and gateway SwitchOutcome to SwitchProfileBody, handles named structs, UI tokens into services" - status: pending + status: completed - id: split content: "Split: supervisor.rs, socket.rs framing, compose, heartbeat and progress run loops" - status: pending + status: completed - id: docs content: "Docs removal: delete workshop guide chapters, export, and READMEs; drop them from the guide build, doc tool, and docs-claims test" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-25-1-vibe2-debt-removal.md b/vibe/2026-09/2026-09-25-1-vibe2-debt-removal.md index e1c497aab..941319784 100644 --- a/vibe/2026-09/2026-09-25-1-vibe2-debt-removal.md +++ b/vibe/2026-09/2026-09-25-1-vibe2-debt-removal.md @@ -4,19 +4,19 @@ overview: "Remove the four debts that the 39 vibe2 commits since upstream/master todos: - id: debt-04-linux-clippy content: "DEBT-04: gate use super::*; in workspace/tests/jail.rs to Windows; verify workshop-workspace clippy and jail tests" - status: pending + status: completed - id: debt-01-heartbeat content: "DEBT-01: restore real-time quiet window in startup_convergence.rs; prove with 5 runs and heartbeat mutation" - status: pending + status: completed - id: debt-02-overwrite content: "DEBT-02: add writeCurrent helper in editor-panel.ts for save() and overwrite(); add cases (a) and (b) to editor-save-timeout.mjs; mutation proof" - status: pending + status: completed - id: debt-03-docs content: "DEBT-03: intra-doc-link module inventories in workshop-server agents.rs and lib.rs; add private-items rustdoc step to check-workshop in ci.yml; link mutation proof" - status: pending + status: completed - id: exit-checks content: Run workshop clippy, nextest, UI npm test, and the new docs step - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-25-2-multi-product-docs-site.md b/vibe/2026-09/2026-09-25-2-multi-product-docs-site.md index b43432f2d..735b3c199 100644 --- a/vibe/2026-09/2026-09-25-2-multi-product-docs-site.md +++ b/vibe/2026-09/2026-09-25-2-multi-product-docs-site.md @@ -4,19 +4,19 @@ overview: "Replace the single-mdBook Pages deploy with one assembled site: an HT todos: - id: stubs content: "Add scaffolding: guide/books/*/book.toml, guide/landing (index.html with the drafted intro and 4-row two-column product table with image slots, style.css, img/), guide/chrome (banner.html, back-link.js), guide/src/workshop/01-stub.md, harness-api lib.md stub + include_str" - status: pending + status: completed - id: user-guide-books content: "build-user-guide: SETS -> BOOKS table (sole list of books), default mode = checks + exports, new `stage ` mode rendering per-book SUMMARY and index into staging; update tests" - status: pending + status: completed - id: xtask-site content: "build-xtask: add `site` subcommand (site.rs) - stage books, mdbook build each staged folder, isolated default-feature cargo doc for promptforge and harness-api with banner via CARGO_ENCODED_RUSTDOCFLAGS, file-targeted redirects, copy landing; absolute paths throughout; `--books-only` flag skips rustdoc" - status: pending + status: completed - id: workflow-housekeeping content: "Create site.yml (full build + deploy on push/dispatch; path-filtered PR build with --books-only, own cancel-in-progress concurrency group, not required); facade documentation= URLs; tools/document.md, CONTRIBUTING.md, crates/README.md, .gitignore edits; move guide.yml and retired guide files to cabinet/_trash" - status: pending + status: completed - id: verify content: Run tests + clippy, cargo xtask site twice, click-through over file://, confirm Harness crates.js isolation and clean git status - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-25-2-workshop-fixes.md b/vibe/2026-09/2026-09-25-2-workshop-fixes.md index 71817c425..9d2c8f928 100644 --- a/vibe/2026-09/2026-09-25-2-workshop-fixes.md +++ b/vibe/2026-09/2026-09-25-2-workshop-fixes.md @@ -4,49 +4,49 @@ overview: Fix the user-facing bugs, the Windows CI gap, and the weakened tests i todos: - id: baseline content: Record the baseline of every exit command before any change - status: pending + status: completed - id: save-as content: Fix Save As onto an existing file and its timeout, and the reconcile-mismatch dialog wording, with UI tests (Overwrite timeout already fixed in 49d97995) - status: pending + status: completed - id: decode content: Remove the second percent-decode on workspace GET paths; %41 round-trip test; traversal still refused - status: pending + status: completed - id: windows-ci content: Run workshop-workspace tests with --all-features in the Windows check-workshop CI job - status: pending + status: completed - id: binary content: Delete the workshop-server binary, workbench.toml fallback, open dependency; drop the open_browser field (old configs still load); fix guide sentence - status: pending + status: completed - id: quit content: Stop the gateway supervisor before posting /shutdown on quit, with ordering and no-relaunch tests - status: pending + status: completed - id: menu-memory content: Make menu memory writes latest-wins with a sequence number - status: pending + status: completed - id: roots-signal content: Add a grant-set change signal to WorkspaceRoots so revokes reach running agent sessions - status: pending + status: completed - id: boot-order content: Validate loopback before composing state and bind before reopening the last workspace - status: pending + status: completed - id: workspace-fixes content: List linked folders as directories; reproduce and fix the grant-during-open race; fix the UNC test helper - status: pending + status: completed - id: switch-resync content: Resync the UI from GET /workspace/file/current after a workspace-switch 408 - status: pending + status: completed - id: timing-tests content: Repair the chat quiet test; replace remaining real-clock windows (heartbeat convergence already repaired in 63ddccb5) - status: pending + status: completed - id: doc-sweep content: Rewrite the two .cursor/rules files and sweep every listed stale doc claim across crates, UI, and repo root - status: pending + status: completed - id: crate-map content: Add crates/workshop/AGENTS.md crate map and the new Vocabulary entries - status: pending + status: completed - id: exit content: Run the exit commands and retired-string checks; record results - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-25-3-workshop-structure.md b/vibe/2026-09/2026-09-25-3-workshop-structure.md index 9cbad7189..7e1035964 100644 --- a/vibe/2026-09/2026-09-25-3-workshop-structure.md +++ b/vibe/2026-09/2026-09-25-3-workshop-structure.md @@ -4,67 +4,67 @@ overview: The structure plan that follows the landed workshop fixes plan, merged todos: - id: step-1 content: "Step 1: Record the baselines (no code)" - status: pending + status: completed - id: step-2 content: "Step 2: Reword the stale UI comments and clarify the layout rule" - status: pending + status: completed - id: step-3 content: "Step 3: Enforce the desktop dependency boundary in the build check" - status: pending + status: completed - id: step-4 content: "Step 4: Trim the gateway public surface and delete the cache API" - status: pending + status: completed - id: step-5 content: "Step 5: Rename the gateway modules and share the heartbeat stop-and-rebind logic" - status: pending + status: completed - id: step-6 content: "Step 6: Deduplicate the gateway binding and client, and close the gateway test gaps" - status: pending + status: completed - id: step-7 content: "Step 7: Tidy the small crates: register tests, private handles, menu directory, one chat filter" - status: pending + status: completed - id: step-8 content: "Step 8: Apply the layout rule to the workspace and rename the Save Workspace As route" - status: pending + status: completed - id: step-9 content: "Step 9: Replace the workspace response type, trim its surface, and close its test gaps" - status: pending + status: completed - id: step-10 content: "Step 10: Share the envelope response and validate state buckets once" - status: pending + status: completed - id: step-11 content: "Step 11: Finish the /ws split: websocket module, shared SocketState, own route and state, renames" - status: pending + status: completed - id: step-12 content: "Step 12: Split the socket loops and move the /ws integration tests" - status: pending + status: completed - id: step-13 content: "Step 13: Move the agent frames into the server and drop the protocol crate Engine dependency" - status: pending + status: completed - id: step-14 content: "Step 14: Trim the server public surface and share its require check" - status: pending + status: completed - id: step-15 content: "Step 15: Restructure the server integration tests and close the server test gaps" - status: pending + status: completed - id: step-16 content: "Step 16: Tidy the desktop gateway module and its tests" - status: pending + status: completed - id: step-17 content: "Step 17: Merge launch-and-wait and survive a stale discovery file" - status: pending + status: completed - id: step-18 content: "Step 18: Settle the UI service tokens, barrels, backoff, and contribution name" - status: pending + status: completed - id: step-19 content: "Step 19: Move the tree panel into workspace and settle the workspace folders" - status: pending + status: completed - id: step-20 content: "Step 20: Guard every inbound frame in the UI" - status: pending + status: completed - id: step-21 content: "Step 21: Update the docs and run the exit gates" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-26-2-harness-facade.md b/vibe/2026-09/2026-09-26-2-harness-facade.md index 4864fb5f2..53df03c38 100644 --- a/vibe/2026-09/2026-09-26-2-harness-facade.md +++ b/vibe/2026-09/2026-09-26-2-harness-facade.md @@ -4,28 +4,28 @@ overview: Turn crates/harness/ into a pure `harness` facade shaped like crates/p todos: - id: log-wrappers content: "harness-log owns DatabaseSource/JsonSource; drop shared-error-source from harness-log; fix shared-error-source docs" - status: pending + status: completed - id: move-rename content: "git mv crates/harness/* to crates/harness-internal/ and crates/harness-api to crates/harness; update root Cargo.toml members/exclude/workspace deps; rename package to harness" - status: pending + status: completed - id: facade-rewrite content: "Make the facade match crates/promptforge - trimmed manifest, lib.rs of re-exports only (no Invariants block), prose into lib.md/cancel.md, delete harness.rs/session.rs/cancel.rs/README.md/clippy.toml, tests/it to tests/suite" - status: pending + status: completed - id: consumers content: "workshop-server manifest and harness_api:: to harness:: imports; internal crates' Invariants blocks" - status: pending + status: completed - id: xtask-names content: "build-xtask product.rs family/PUBLIC_HARNESS/container exception, harness_clippy_bans on the container only, facade_shape FACADE_DIRS, site.rs, fixture tests" - status: pending + status: completed - id: surface content: "Facade pub mod log re-exports; gate Harness::log behind test-support with a self dev-dependency (fallback: unit tests)" - status: pending + status: completed - id: guard-rules content: "Extend test_support_leak and doc_hidden to the Harness facade and container; in product.rs ban Harness to gateway and to shared, and generalize the facade-only rule to mirror PUBLIC_PROMPTFORGE; tests" - status: pending + status: completed - id: docs-gate content: "AGENTS.md, READMEs, archdoc, promptforge-types cancel.rs doc; full gate plus site build" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-26-2-local-tools-coroutine.md b/vibe/2026-09/2026-09-26-2-local-tools-coroutine.md index d851678ef..0361c8d54 100644 --- a/vibe/2026-09/2026-09-26-2-local-tools-coroutine.md +++ b/vibe/2026-09/2026-09-26-2-local-tools-coroutine.md @@ -4,22 +4,22 @@ overview: Run `tools.add_local` handlers inside the block coroutine by having th todos: - id: lua-storage content: Move handler storage into a per-VM Lua registry table; LocalTools keeps (alias, schema); delete LocalTools::call and SectionVm::call_local_tool; fix poison test - status: pending + status: completed - id: protocol content: Add ToolCallOutcome::Local (render as (true, nil, handler, args)), Request::LocalToolDone with LocalToolOutcome, and the local_tool_done parse arm applying scalar_return - status: pending + status: completed - id: prelude content: Add dispatch_tool/run_local_tool to __impl_coro.lua with swap_jump capture passed from install_shim_prelude; route tools_call and tools_call_as_model through it - status: pending + status: completed - id: scheduler content: Add frame local_calls stack; local branch pushes and answers Local; replace answer_local_tool with dispatch_local_tool_done; route in dispatch.rs with blocked None - status: pending + status: completed - id: tests content: Add Engine tests (store in handler via models.loop and tools.call, jump refused and restored, bad return, nested local, handler calling a bound tool on success and on failure inside a model-issued call), shim protocol walk test, lua parse/render tests - status: pending + status: completed - id: docs-verify content: Update module docs and 07-tools.md plus aggregate guide; run nextest, doctests, clippy, fmt, docs, site gates - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-26-3-harness-dependency-rule.md b/vibe/2026-09/2026-09-26-3-harness-dependency-rule.md index 72c903f5f..9e8b01504 100644 --- a/vibe/2026-09/2026-09-26-3-harness-dependency-rule.md +++ b/vibe/2026-09/2026-09-26-3-harness-dependency-rule.md @@ -4,10 +4,10 @@ overview: "Close the one debt item the Harness facade refactor (0f43e1b3..3e2a95 todos: - id: hf1-rule content: "DEBT-HF-1: add WORKSPACE_HACK constant and (Harness, Build) / (Harness, Unaffiliated except workspace-hack) arms in crates/build-xtask/src/product.rs; rewrite the Harness module-doc bullet" - status: pending + status: completed - id: hf1-tests content: "DEBT-HF-1: add build-ui build-dependency and unaffiliated-crate fixtures to product-harness-tests.rs; run cargo test -p build-xtask, clippy, fmt" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-26-4-store-into-vfs.md b/vibe/2026-09/2026-09-26-4-store-into-vfs.md index 83a9b0090..e5b25ba02 100644 --- a/vibe/2026-09/2026-09-26-4-store-into-vfs.md +++ b/vibe/2026-09/2026-09-26-4-store-into-vfs.md @@ -4,46 +4,46 @@ overview: Fold the Store facade into the VFS with the smallest public API that k todos: - id: empty-anchor content: Step 1 - Refuse empty anchors in Access::str_replace and the default VfsAccess::str_replace, with tests; both str_replace doc comments say so - status: pending + status: completed - id: vfs-semantics content: Step 2 - Every Access has a root and paths without a leading slash join onto it; Access::remove returns Result; Access::glob returns files, or directories for a trailing slash, and validates the raw pattern; update the path, remove and glob doc comments and the vfs.md path rules and Access entries; regenerate public-api.txt - status: pending + status: completed - id: vfs-error content: Step 3 - Restructure VfsError into 13 struct variants (Backend carries a message; derives unchanged), move PathReason into the VFS with Wildcard and IntoDescendant and give every VFS InvalidPath site a reason, migrate all construction sites; rewrite the VfsError doc comments, the vfs.md VfsError section and every example that matches a tuple variant; point the facade's PathReason re-export at the VFS - status: pending + status: completed - id: engine-joins content: Steps 4 and 5 - Rename when_any/when_all to join_any/join everywhere, Harness tests included (Step 4); the root chain keeps one identity through H1 and the walk, record each task's ExecId, join on every delivery (join_any, join, model task notices) and at chain end, and simplify the AppendGate tests (Step 5) - status: pending + status: completed - id: hb-claims content: Step 5 - Replace liveness claims with happens-before (FastTrack epochs per region) - scopes, vector clocks, region claims, fork at spawn instead of retire, detail::access_join, identity refcount, scope lookup for mounted handles; rewrite the claims doc comments and the vfs.md "Identities and claims" section; drop the drop-before-answer wording from effect.md, run-effect.rs and the Harness performers; one commit with the joins - status: pending + status: completed - id: scheduler-limit content: Step 6 - RunLimits::max_concurrency replaces max_fanout_concurrency; hierarchical per-chain limits; slot released while parked on a join; resumptions before fresh starts; blocked 'queued'; task_started at admission; tasks.concurrency(n?); fanout spawns every arm up front; update the RunLimits, fanout and shim doc comments, lib.md and event.md - status: pending + status: completed - id: store-root content: Step 7 - VfsRefBuilder::store(root, backend), Default for VfsRef, overlay inheritance, and the internal store view (rooted, strict, confined to its mount, relative error paths, same identity), with the store tests ported to the VFS; STORE_MOUNT, empty() and the /_promptforge wording go in Step 10 - status: pending + status: completed - id: engine-store content: Step 8 - perform_store_op over the store view returning VfsError; Conflict ends the run, shared library load included; other failures raise Lua error kind store with the rendered message; a re-raised store error keeps Store; Run::new requires a declared store; RunContext::new uses VfsRef::default(); delete Environment::base_vfs, run_vfs and vfs_explicit; drop the store crate from the engine and StoreError and StoreErrorKind from the facade; migrate the tests; update the Engine, vfs.md, effect.md and lib.md docs - status: pending + status: completed - id: lua-binding content: Step 8 - The Lua store table calls Access through the store view (no Store::new); store_error_message writes model-quality text with recovery hints; ErrorKind::Store and an integer error field for err.count; record shared library load conflicts; depend on promptforge-vfs directly; the StoreOutcome derives land in Step 9 - status: pending + status: completed - id: harness content: Step 8 - StorePerformer returns VfsError; Services::vfs is the run's complete filesystem passed straight to the context's VFS; sessions and web pass VfsRef::default(); plugins tests stop naming the store path; prepare and RunServices docs describe the complete filesystem (the wait renames are in Step 4 and the drop-rule docs in Step 5) - status: pending + status: completed - id: records-events content: Step 9 - StoreOutcome derives Clone, Eq, Serialize, Deserialize; drop StoreAnswerRecord (AnswerRecord::Store holds StoreOutcome); add StoreExistsSucceeded/StoreExistsFailed from both dispatch and the direct closure; document them in the event doc comments, event.md and effect.md - status: pending + status: completed - id: retire-store content: Step 10 - Move crates/promptforge-internal/store (with its README and AGENTS.md) to cabinet/_trash, remove it from the workspace, delete STORE_MOUNT and empty(), and remove the /_promptforge wording and the store crate from the VFS docs, the internal README and root AGENTS.md - status: pending + status: completed - id: guide-docs content: Step 11 - Revise guide chapters 01, 05, 09, 14, 15, 16, 17 and 18 per Step 11's inventory, with chapter 09's rewritten sections and chapter 15's new material written through the doc pass, then regenerate guide/promptforge-language-guide.md with build-user-guide - status: pending + status: completed - id: living-docs content: Step 12 - Update vibe/archdoc.md and vibe/papergate-harness-migration.md through the doc pass, give vfs.md, effect.md, event.md and lib.md a fresh audit each, and run the facade API check and the stale-term sweep - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-27-1-fork-shared-ui-look.md b/vibe/2026-09/2026-09-27-1-fork-shared-ui-look.md index 6315770d8..c90da6ad6 100644 --- a/vibe/2026-09/2026-09-27-1-fork-shared-ui-look.md +++ b/vibe/2026-09/2026-09-27-1-fork-shared-ui-look.md @@ -4,25 +4,25 @@ overview: Make crates/workshop one npm workspace (one install, one lockfile, one todos: - id: npm-workspace content: "Create the crates/workshop npm workspace: root package.json with shared tooling, git mv the lockfile, widen the .gitignore node_modules line, build-ui esbuild lookup and watch changes, CI and docs" - status: pending + status: completed - id: copy-switch content: Copy shared-ui's tracked files to crates/workshop/look as @workshop/look, add .gitattributes lines, add it to the workspace, switch workshop/ui imports, update docs - status: pending + status: completed - id: drop-aliases content: Repoint look's 12 alias uses to Cursor tokens and delete the Gateway alias block from look/tokens.css - status: pending + status: completed - id: token-tiers content: Move workshop/ui/src/tokens/base.css and semantic.css into look as sizes.css and semantic.css; update main.ts import order - status: pending + status: completed - id: icons content: Move parts/shared/icons.ts into look with lucide as a look dependency; confirm one hoisted lucide copy - status: pending + status: completed - id: tests-guards content: Move the four component tests into look/test, add boundary test and workshop no-shared-ui guard; CI runs tests across the workspace - status: pending + status: completed - id: agents-md content: Rewrite look/AGENTS.md and document the workspace in crates/workshop/AGENTS.md - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-27-1-store-vfs-debt.md b/vibe/2026-09/2026-09-27-1-store-vfs-debt.md index 9460f6fd8..a90f67684 100644 --- a/vibe/2026-09/2026-09-27-1-store-vfs-debt.md +++ b/vibe/2026-09/2026-09-27-1-store-vfs-debt.md @@ -4,31 +4,31 @@ overview: "Remove the 12 debts the Debt Collector retained from the store-into-V todos: - id: scope-through-acquire content: "D1-10: add AcquireContext, change Vfs::acquire to take it, rewire VfsRef/Router/spawn/store_view, delete SCOPES/remember_scope/forwarded_scope, update every implementer and doc, regenerate public-api.txt; directly-wrapped VfsRef test" - status: pending + status: completed - id: end-run-scope content: "C-33: closed flag on Scope, detail::end_scope called at Done and on early drop, closed scopes refuse with PermissionDenied, drop-timing doc pass; held-access post-run read test and after-Done refusal test" - status: pending + status: completed - id: fix-pruning content: "D1-1, D1-2: ordered pruning only with one live scope, remove empty regions, reset the prune threshold from the surviving count; extend two_live_scopes_writing_one_path_conflict past 4096 claims, empty-region test" - status: pending + status: completed - id: claim-write-two-phase content: "D1-5, D1-6: claim_write checks everything before recording, adds ancestors[P].created and paths[ancestor].write checks; rename/copy combined claim; both-order sibling tests and refused-claim-records-nothing tests" - status: pending + status: completed - id: call-chain-admission content: "D1-3: a call chain parked on a join releases the nearest holding task's slots and retakes them on wake; stall guard fires when nothing queued can be admitted; call-in-arm fanout test at tasks.concurrency(1)" - status: pending + status: completed - id: captured-store-functions content: "D1-4, D1-31: store function values follow the load phase so captured locals go through Effect::Store after load; captured-local Determinism test and Effect::Store test" - status: pending + status: completed - id: store-probe content: "D1-7: the probe stats the store root through the one-mount router (NotFound on root is success); refusing-backend test expects Done with RunErrorKind::Store" - status: pending + status: completed - id: store-error-reporting content: "D1-8, D1-9: StoreScoped trims only the leading slash; vfs.md line 582 names both InvalidRange producers; /my/store path test" - status: pending + status: completed - id: exit-checks content: Full local check run (tests, clippy, fmt, rustdoc -D warnings, xtask api --check, guide build) and stale-term sweep - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-27-2-look-fork-debts.md b/vibe/2026-09/2026-09-27-2-look-fork-debts.md index c681e1c56..b8cce1bc1 100644 --- a/vibe/2026-09/2026-09-27-2-look-fork-debts.md +++ b/vibe/2026-09/2026-09-27-2-look-fork-debts.md @@ -4,10 +4,10 @@ overview: "Remove the two debts that PR #81 (f4f92899..6a0ad504) added: build-ui todos: - id: w1-install-root content: "W1 (D1-1, D1-3): add install_root in crates/build-ui/src/lib.rs, bound find_esbuild and workspace_members at it, add the three failing-first tests and update the existing ones in lib-tests.rs, verify with build-ui tests and both UI cargo builds" - status: pending + status: completed - id: w2-look-theming content: "W2 (D1-2): add --accent-bright to look/tokens.css and use it in look/controls.css line 48; reword look/AGENTS.md line 7, AGENTS.md line 89, workshop-spa.mdc line 20; verify with the git grep checks, workspace npm suite, and a visual hover check" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-27-3-platform-extraction.md b/vibe/2026-09/2026-09-27-3-platform-extraction.md index 05b34ab47..2ba294452 100644 --- a/vibe/2026-09/2026-09-27-3-platform-extraction.md +++ b/vibe/2026-09/2026-09-27-3-platform-extraction.md @@ -4,16 +4,16 @@ overview: "Extract the Workshop SPA's generic browser-side mechanics into a new todos: - id: platform-move content: "Create crates/workshop/platform (@workshop/platform) and move the mechanics: git mv the listed modules, split Result/ok/err into result.ts, rewrite ~350 imports and 134 test paths, move the status bar contract, add platform/AGENTS.md and the approved boundary test, align docs and rules" - status: pending + status: completed - id: open-registry content: "Open the panel registry: move it to platform with string ids and entry fields, features register their own panel types, layout policy from the composition root, layout schema v5, synthetic-panel behavior test" - status: pending + status: completed - id: tab-menu content: "One generic tab with a registry-driven tab menu: MenuId.EditorTitleContext, per-tab overlay, VS Code-aligned activeEditor, confirm-then-close for the X button and the Close and Close Others commands, tab-menu stub rows" - status: pending + status: completed - id: status-indicators content: "Status bar indicator slots: platform contract, generic status bar, contributed activity and recording LEDs wired at the composition root, setRecording removed from the status bar contract, recording verified end to end" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-28-1-platform-extraction-debts.md b/vibe/2026-09/2026-09-28-1-platform-extraction-debts.md index 3ede3cf62..c1c362fc1 100644 --- a/vibe/2026-09/2026-09-28-1-platform-extraction-debts.md +++ b/vibe/2026-09/2026-09-28-1-platform-extraction-debts.md @@ -4,13 +4,13 @@ overview: "Remove the debt the platform extraction run added: move Close and Clo todos: - id: close-ownership content: "D1-1: move the generic close path and both close commands into parts/layout (panel-close.ts, registered from layout.contribution.ts), add WorkshopPart.isDirty, delete the editor copies, re-point tests, add the layout-only close test" - status: pending + status: completed - id: tab-affordances content: "D1-2 and D1-3: in panel-tab.ts, guard the X's pointerdown and drop it from the Tab order, and move focus to the neighbouring tab after a keyboard close" - status: pending + status: completed - id: activity-reset content: "D1-4: ActivityIndicator.reset cancels the pulse timer, clears the lit set and repaints, with a regression test in activity-led.mjs" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-28-2-capability-harness-redesign.md b/vibe/2026-09/2026-09-28-2-capability-harness-redesign.md index 3609b5016..5447ca139 100644 --- a/vibe/2026-09/2026-09-28-2-capability-harness-redesign.md +++ b/vibe/2026-09/2026-09-28-2-capability-harness-redesign.md @@ -4,49 +4,49 @@ overview: Turn PromptForge Plugins into self-describing units, installable by na todos: - id: prelude content: Carry Plugin preludes from Contribution.prelude through Environment::preludes to section setup; install each in a restricted environment with sealed exports, checking load failures and name collisions when a section VM installs them - status: pending + status: completed - id: fullid content: Resolve a script tools.call that names the full id of a tool in the run's catalog, with no global, no scoping, and no advertising - status: pending + status: completed - id: origin content: Add ToolCallOrigin (execution, section, and caller, script or model) to Effect::ToolCall and EffectRecord::ToolCall - status: pending + status: completed - id: broker-needs content: Add InputBroker and InputError to harness-plugins and RunServices.input; make Services.input an optional broker, bridged to the old input performer until Engine removal; add Service, Plugin::needs, the activation check, Requirements.missing_services, and Activation.service_gaps - status: pending + status: completed - id: userinput-cap content: Build promptforge/user-input in harness-plugins; register it in every first-party registry; map its script-side result to user_message in Workshop; move chat.md to input.ask() and declare it required - status: pending + status: completed - id: park-tests content: Port the Engine and runner tests that use user_input() only as a parking point to fixture tools called by full id (first commit of the Engine removal change) - status: pending + status: completed - id: migrate-ask content: Move every prompt, test body, and guide chapter that uses user_input() as user input to input.ask() with a declaration, while user_input() still works through the adapter (second commit) - status: pending + status: completed - id: engine-removal content: Delete the user-input facility from the Lua, engine, types, runner, sessions, Workshop, and facade crates, with the adapter, the tests of the removed types, and the regenerated public API (third commit) - status: pending + status: completed - id: guard-nonce content: Derive the per-run guard nonce with SHA-256 instead of SplitMix64 and pin the derivation with a known-answer test - status: pending + status: completed - id: factories content: (Held until its open questions are settled) Zero-argument factories with manifest and config JSON; gateway as Host service - status: pending + status: completed - id: answer-records content: (Held until its open questions are settled) Project large tool and store answers to path, length, hash, version - status: pending + status: completed - id: open-slot content: (Held until its open questions are settled) Design and build the open slot for runtime-only tools - status: pending + status: completed - id: menu content: (Held until its open questions are settled) Workshop Plugin menu and saved agent definitions - status: pending + status: completed - id: dll-abi content: (Held until its open questions are settled) Revise the Plugin DLL plan to the one-verb manifest ABI - status: pending + status: completed - id: report content: (Held until its open questions are settled) Correct or supersede the 2026-09-19 survey report - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-28-3-capability-harness-debts.md b/vibe/2026-09/2026-09-28-3-capability-harness-debts.md index af420a2c8..ab30bda8c 100644 --- a/vibe/2026-09/2026-09-28-3-capability-harness-debts.md +++ b/vibe/2026-09/2026-09-28-3-capability-harness-debts.md @@ -4,16 +4,16 @@ overview: "Debt collector result for the Plugin Harness redesign run (15 commits todos: - id: l1-proxy content: Move the three read-only proxy builders in promptforge-lua (prelude.rs, argv.rs, sys.rs) onto one crate-private builder in proxy.rs, pinning every refusal text and getmetatable label first (L-1) - status: pending + status: completed - id: d1-18-merge content: Make Requirements::merge drop missing_required entries that missing_services already names, with a failing-first runner regression test and a requirements unit test (D1-18) - status: pending + status: completed - id: doc-fixes content: Correct the sessions AGENTS.md broker sentence, the prelude environment doc, and the guide's Letting the model ask sentence, then regenerate the combined guide (D1-26, D1-27, D1-32) - status: pending + status: completed - id: d1-2-engine-docs-gate content: Add a CI step to the docs job that builds promptforge-engine's private-item docs with warnings denied (D1-2) - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-28-4-internal-crates-critical-fixes.md b/vibe/2026-09/2026-09-28-4-internal-crates-critical-fixes.md index 46546f4de..182acb909 100644 --- a/vibe/2026-09/2026-09-28-4-internal-crates-critical-fixes.md +++ b/vibe/2026-09/2026-09-28-4-internal-crates-critical-fixes.md @@ -4,34 +4,34 @@ overview: "Fix the three critical defects found in the promptforge-internal revi todos: - id: lua-cancel content: "Lua cancellation: cancel_requested capture; re-raise in pcall, xpcall, local handler call, compactor call; tests" - status: pending + status: completed - id: chain-section-label content: "Engine: record entered section name on Chain; section_name reads it; walk-end and between-sections notice tests" - status: pending + status: completed - id: host-symlinks content: "VFS real-filesystem backend: no-follow resolution for remove, exists, stat, mkdir, rename; Windows directory links; symlink tests" - status: pending + status: completed - id: scheduler-cleanup content: "Engine: delete dead call stack; saturating slot release; const nz_*; concurrent-call test" - status: pending + status: completed - id: lua-guard-lock content: "Lua: __metatable on _G guard metatables in argv.rs and prose.rs; guard test (resolved by a separate change, not a step of this plan; see the Decision Record)" status: completed - id: dead-items content: Remove NormalizedTurn non_exhaustive, unused StreamDelta and metrics re-exports; rename promptforge-api- temp prefixes - status: pending + status: completed - id: docs-sweep content: Engine AGENTS/README/lib.md/Cargo description; types, lua, model-client, container README; new vfs README; root AGENTS.md line 55 - status: pending + status: completed - id: comments-sweep content: Fix stale comments and remove history/audit-tag comments across engine, vfs, model-client, types, parser - status: pending + status: completed - id: dir-folds content: Fold engine run-* into execute/run/ and recording-* into test_support/recording/ - status: pending + status: completed - id: verify content: AGENTS.md gates, cargo deny, xtask api --check unchanged, facade docs build - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-28-5-internal-crates-fix-batch.md b/vibe/2026-09/2026-09-28-5-internal-crates-fix-batch.md index 678eecfc5..551f3381e 100644 --- a/vibe/2026-09/2026-09-28-5-internal-crates-fix-batch.md +++ b/vibe/2026-09/2026-09-28-5-internal-crates-fix-batch.md @@ -4,67 +4,67 @@ overview: Fix the cancellation, real-filesystem backend, claims-ledger, and vali todos: - id: types content: "Types: Cancelled deregisters on drop; fold ModelCatalog::from_validated; event test gaps" - status: pending + status: completed - id: model-client content: "Model client: validating from_result/from_parts with their Engine test and facade doctest callers; linear SSE scan; null error field; #[non_exhaustive] ClientError with wildcard arms; re-bless listing" - status: pending + status: completed - id: lua-cancel content: "Lua cancellation: skip the author xpcall handler under cancel; refuse __gc metatables and __mode on _G; tests" - status: pending + status: completed - id: lua-cleanup content: "Lua cleanup: pin var.k = nil; delete the unreachable Rust placeholder behind models.infer (models.infer itself stays); correct the run_store_op doc; crate docs" - status: pending + status: completed - id: vfs-host content: "VFS real-filesystem backend: refuse dangling symlinks in contain; README sentence; tests" - status: pending + status: completed - id: vfs-claims content: "VFS claims: close the three verified gaps (nested subtrees, subtree vs parent listing, created ancestors vs listings and globs); both-order tests" - status: pending + status: completed - id: dead-api content: Remove grep (VFS crate, facade re-exports, vfs.md, tests) and Environment::max_depth (engine, lib.md); re-bless listing - status: pending + status: completed - id: parser-frontmatter content: "Parser frontmatter: duplicate-Plugin refusal; optional-Plugin slot refusal; guide text; regenerate the combined guide; tests" - status: pending + status: completed - id: parser-structure content: "Parser structure: span doc correction; column-0 frontmatter closer; container headings; structural error locations; tests" - status: pending + status: completed - id: engine-surface content: "Engine surface: narrow the unused public surface with the bench and doctest paths; bench-only test-support docs; Requirements::notice and prelude-collision tests" - status: pending + status: completed - id: engine-scheduler content: "Engine scheduler: entered placeholder; drive.rs panic; shared Chain constructor; rename fanout.rs to heading_address.rs; tests" - status: pending + status: completed - id: housekeeping content: "Housekeeping: fold build-xtask product-* and api/listing-*; vfs manifest test catches the missing dependency table forms; audit-tag and history comment sweep" - status: pending + status: completed - id: split-small content: "Split to the ceiling and add the marker: promptforge-types, promptforge-model-client, promptforge-parser" - status: pending + status: completed - id: split-lua-tests content: "Split promptforge-lua src/tests.rs to the ceiling" - status: pending + status: completed - id: split-lua-rest content: "Split the rest of promptforge-lua to the ceiling and add the marker" - status: pending + status: completed - id: split-engine-source content: "Split the promptforge-engine source files and forward-tests.rs to the ceiling" - status: pending + status: completed - id: split-engine-exec-flow content: "Split promptforge-engine suite/exec_flow.rs to the ceiling" - status: pending + status: completed - id: split-engine-scheduler-tests content: "Split the promptforge-engine scheduler/ test files to the ceiling" - status: pending + status: completed - id: split-engine-other-tests content: "Split the remaining promptforge-engine test files to the ceiling and add the marker" - status: pending + status: completed - id: split-vfs-handle content: "Split promptforge-vfs src/handle.rs to the ceiling" - status: pending + status: completed - id: split-vfs-rest content: "Split the rest of promptforge-vfs to the ceiling, add the marker, and pass the full exit gates" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-09/2026-09-30-1-engine-harness-host-terminology.md b/vibe/2026-09/2026-09-30-1-engine-harness-host-terminology.md index feed73a34..40c7d2498 100644 --- a/vibe/2026-09/2026-09-30-1-engine-harness-host-terminology.md +++ b/vibe/2026-09/2026-09-30-1-engine-harness-host-terminology.md @@ -4,37 +4,37 @@ overview: "One PR in three commits. Commit 1 gives the root AGENTS.md one Defini todos: - id: preflight content: "Preflight: clean tree, new branch, and a trivial shell command to learn whether cargo, git and node can run on this machine" - status: pending + status: completed - id: c1-agents content: "Commit 1: write the Definitions and Using-the-terms sections into the root AGENTS.md, trim Roles, and fix lines 3, 37, 54, 56 and 79" - status: pending + status: completed - id: c1-rulebooks content: "Commit 1: apply the rulebook removals and rewordings in section 1.2, including .cursor/rules/workshop-architecture.mdc" - status: pending + status: completed - id: c1-sweep-code content: "Commit 1: run the seven code-area workers (Appendix A1 to A7) in parallel with the brief in section 1.3" - status: pending + status: completed - id: c1-sweep-docs content: "Commit 1: run the guide worker (A8) and the three vibe workers (A9 to A11) in parallel, including the rewrites in section 1.4 and the heading renames in 1.5" - status: pending + status: completed - id: c1-verify content: "Commit 1: regenerate the guide exports, run checks V1, V2, V4 and V5 until clean, then the full gate list" - status: pending + status: completed - id: c1-commit content: "Commit 1: commit with a message written per the workspace commit rule" - status: pending + status: completed - id: c2-caps content: "Commit 2: run the capitalization workers over the same eleven areas (section 2)" - status: pending + status: completed - id: c2-verify content: "Commit 2: regenerate the guide exports, run checks V1, V2, V3 and V6, then the gates, and commit" - status: pending + status: completed - id: c3-guard content: "Commit 3: extend crates/workshop/ui/test/docs-claims.mjs per section 3, add the pointer bullet to AGENTS.md, run npm test in crates/workshop, and commit" - status: pending + status: completed - id: followup content: "Write the follow-up rename plan from Appendix C as a separate plan; do not execute it in this PR" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-01-1-remove-task-event-reads.md b/vibe/2026-10-01-1-remove-task-event-reads.md index c597c0c03..5b0dbdde6 100644 --- a/vibe/2026-10-01-1-remove-task-event-reads.md +++ b/vibe/2026-10-01-1-remove-task-event-reads.md @@ -4,13 +4,13 @@ overview: "Delete the task history read completely, with no trace outside vibe/: todos: - id: step-1-task-history-code content: "Step 1: remove the Engine, Lua protocol and Harness runner code, tests and prose, hand-edit the facade pages, bless public-api.txt" - status: pending + status: completed - id: step-2-run-log-read-path content: "Step 2: remove the harness-log per-task read path and fix its tests" - status: pending + status: completed - id: step-3-guide-chapter content: "Step 3: delete guide chapter 16, renumber 17 and 18, fix every link and per-site sentence (including crates/harness/src/vfs.md), regenerate exports, run the residual searches" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-01-2-store-to-vfs-inline.md b/vibe/2026-10-01-2-store-to-vfs-inline.md index 1c1b2222a..6fae0d924 100644 --- a/vibe/2026-10-01-2-store-to-vfs-inline.md +++ b/vibe/2026-10-01-2-store-to-vfs-inline.md @@ -4,10 +4,10 @@ overview: "Rename the Engine's Store effect vocabulary to Vfs forms, then have t todos: - id: step-1-rename-store-to-vfs content: "Step 1: rename the Store effect vocabulary to Vfs across the Engine, Lua types, facade, Harness, tests and docs; bless public-api.txt; commit 1 (pure rename)" - status: pending + status: completed - id: step-2-answer-vfs-inline content: "Step 2: answer Effect::Vfs inline in the Harness effect loop; delete StorePerformer, VfsStore and Performers.store; fix tests and docs, including the vibe/archdoc.md performer sentence; commit 2" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-02-1-gateway-client.md b/vibe/2026-10-02-1-gateway-client.md index a069ee1a7..08d9cab14 100644 --- a/vibe/2026-10-02-1-gateway-client.md +++ b/vibe/2026-10-02-1-gateway-client.md @@ -4,22 +4,22 @@ overview: "Create a root crate, harness-gateway-client, parallel to harness, hol todos: - id: neutral-api content: Add public read accessors on ToolSchema and CompletionOptions, and Completion builders for reasoning and metadata diagnostics, in model-client; bless public-api.txt - status: pending + status: completed - id: engine-suites content: Replace the engine's axum ScriptedGateway and MockGatewayClient with an in-process ScriptedChat performer and neutral ScriptedReply, keeping delta behavior; convert suites, bench, and drivers; drop axum/reqwest/bytes dev-deps - status: pending + status: completed - id: create-crate content: Create crates/harness-gateway-client, move the wire code (request, read, stream, classify, OpenAI body and metadata parsing) out of model-client, and repoint harness-internal/models to it; then remove promptforge::transport from the facade - status: pending + status: completed - id: move-client content: "Move GatewayClient, catalog, config, and failure from harness-internal/models into the new crate, rewriting harness_models:: doc examples; move the performer adapter, its delta tests, and end_to_end into harness-sessions; delete harness-internal/models" - status: pending + status: completed - id: rules-workspace content: Widen the harness facade allow-list in build-xtask product rules, update fixtures and the AGENTS.md rule line, remove models from root Cargo.toml members and swap its workspace dependency, and run hakari - status: pending + status: completed - id: docs-records content: Document the new public items in rustdoc; fix stale doc references (facade model.md and event.md, model-client docs); add the crates/README.md entry; no cicerone runs - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-02-2-host-services.md b/vibe/2026-10-02-2-host-services.md index 94e195c12..003307c24 100644 --- a/vibe/2026-10-02-2-host-services.md +++ b/vibe/2026-10-02-2-host-services.md @@ -4,22 +4,22 @@ overview: First trim every AGENTS.md to the rules no test or check enforces, cut todos: - id: agents-trim content: "First: trim every AGENTS.md by the keep rule. Delete 10 crate files and cut 21 to their listed rules. Cut root to Definitions, Principles, Engineering, Verification, the flat-directory and dependency-cycle rules, and the SPA token and storage rules, removing the cicerone facade-page rule. Fix the build-xtask comments that point at removed AGENTS.md text" - status: pending + status: completed - id: service-ids content: In harness-plugins, replace the closed Service enum with const ServiceKey and ServiceId (id literal plus provider type) and HostServices; provides checks id and type, so activation treats a wrong-typed provider as missing and reports it by id; the input broker becomes the named service promptforge/input-broker - status: pending + status: completed - id: host-registry content: Harness::new takes the Host's PluginRegistry and HostServices; first_party_registry stops registering UserInput and keeps only a temporary built-in web until step 5; publish harness::plugin with its hand-written page; update every Harness::new call; Workshop registers UserInput - status: pending + status: completed - id: gateway-search content: Copy the Gateway web-search HTTP client into harness-gateway-client as GatewaySearch over GatewayEndpoint and SecretString, keeping every message, deadline, and header unchanged; the internal web-search crate stays live until step 5; GatewayEndpoint errors stop echoing the URL - status: pending + status: completed - id: harness-web content: Create root crates/harness-web merging web, webfetch, and the search tool; search calls the promptforge/search-provider service and renders typed results; fetch runs on the promptforge/tokio-runtime service with abort-on-drop; delete the three harness-internal web crates and the Harness's built-in web; in the same step update root Cargo.toml members and dependencies, run hakari, rewrite the harness inventory test, add harness-web to PUBLIC_HARNESS with fixtures, and have Workshop register Web with a Gateway-backed SearchProvider adapter and its tokio runtime handle - status: pending + status: completed - id: workshop-wiring content: Update crates/README.md, vibe/archdoc.md, guide chapters 5, 12, and 13 with the regenerated export, and the crate READMEs; run the full exit criteria - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-02-3-debt-removal.md b/vibe/2026-10-02-3-debt-removal.md index e4599d681..d4c7059a5 100644 --- a/vibe/2026-10-02-3-debt-removal.md +++ b/vibe/2026-10-02-3-debt-removal.md @@ -4,16 +4,16 @@ overview: "Remove the debt that changes 2 to 5 (upstream/master b845508c to 1f05 todos: - id: failure-contract content: "D1-3, D1-12: catalog regression test first; Gateway text to detail in catalog.rs; align CompletionError and classify_http_failure docs with the shipped messages" - status: pending + status: completed - id: guide-order content: "D1-4: fix the search row order sentence in guide chapter 13 and regenerate the export" - status: pending + status: completed - id: layout-prose content: "D1-6: fix archdoc lines 10 and 23, replace the root AGENTS.md import pointer in the new-crate template and surviving Invariants blocks, name the platform boundary guard" - status: pending + status: completed - id: exit-checks content: Run the exit checks once after all three items - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-02-4-inference-broker.md b/vibe/2026-10-02-4-inference-broker.md index fdd697ef5..10ea637a9 100644 --- a/vibe/2026-10-02-4-inference-broker.md +++ b/vibe/2026-10-02-4-inference-broker.md @@ -4,19 +4,19 @@ overview: Replace the Harness's own Gateway bindings with a required Host-suppli todos: - id: broker-seam content: "Harness: InferenceBroker and OnDelta replace ChatPerformer; runner maps stream to on_delta; Harness::new takes the broker; current_model resolves via models(); delete GatewayBinding, CatalogBinding, set_gateway, set_catalog, generations, GatewayUnusable, performer.rs; drop harness-sessions' harness-gateway-client dependency; tests and doc tours on scripted or offline brokers" - status: pending + status: completed - id: gateway-client content: "harness-gateway-client: rename GatewayClient to GatewayChat; add GatewayBroker with the moved SSE performer tests; GatewaySearch implements harness_web::SearchProvider with wire types private and Workshop's mapping tests moved in; depend on harness and harness-web" - status: pending + status: completed - id: workshop-broker content: "Workshop: gateway-following broker passed to Harness::new; search delegator without conversion; bindings.rs pushes only HostSnapshot; launch refusal with today's GatewayUnusable text; Workshop tests for Unavailable, gateway replacement, and no cancel on replacement" - status: pending + status: completed - id: docs-tooling content: RUSTDOC_SITES gains harness-gateway-client and harness-web with test; harness lib.md reference entries; archdoc, crates/README.md, crate READMEs, cicerone harness plan inventory; regenerate workspace-hack - status: pending + status: completed - id: exit-criteria content: Run the leftover rg searches and the exit criteria once at the end, plus the manual Workshop chat, web, and profile-switch checks - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-03-1-harness-per-run.md b/vibe/2026-10-03-1-harness-per-run.md index 8555d321d..f811f5770 100644 --- a/vibe/2026-10-03-1-harness-per-run.md +++ b/vibe/2026-10-03-1-harness-per-run.md @@ -4,28 +4,28 @@ overview: "Finish moving I/O out of the Harness in one plan: each run gets its o todos: - id: engine-rounds content: "Engine: RoundId and Round replace stream on Effect::Chat; round ids on thinking, reply, and tool-call events; EffectRecord::Chat gains round and drops model; Completion::with_model and CompletionOptions::with_model; StreamDelta stays until the streaming step because the runner's OnDelta uses it; the runner passes on_delta for rounds whose origin is Chat; bless public-api.txt" - status: pending + status: completed - id: per-run-harness content: "harness-runner, landed under harness-sessions: per-run Harness with control() and run(RunRequest) returning RunReport or HarnessError; RunControl stop_round (spares operator questions) and cancel; the effect loop in its final single-future form (FuturesUnordered, Abortable, catch_unwind) with spawn.rs, the answer channel, and Answering deleted; prompt as source text and prepare_run's file read gone; HostSnapshot and current_model move to the runner and OutputError lands there for RunReport; input broker via HostServices, insert_input_broker deleted; Host-supplied Timer; drive_run sink removed; InferenceBroker::chat takes Round beside a transitional Option the Harness always passes as None; harness-sessions rebuilt on it and still cancels and relaunches on Stop; never-block rule in the performer trait docs; spawn Invariants rewritten; thread-parking test; checkpoint: Workshop's suite passes with only the broker signature changed" - status: pending + status: completed - id: workshop-agents content: "New tier-2 crate workshop-agents with conversations, transcript stamping reply from the event's round, input broker and waits, failure reports, recorder tee, discovery, chat.md, TokioTimer, adapted from harness-sessions; rewire workshop-server launch, socket, status, bindings, per-run broker (hands WorkshopBroker its delta sender for Chat rounds), Stop to stop_round, close to cancel; facade swaps to the per-run exports and its doc pages are rewritten; delete the workshop-architecture.mdc Tiers bullet and \"Repository policy binds plans.\" from root AGENTS.md, and update the tidy.rs FEATURES comment and workshop-server Invariants; harness-sessions stays compiling and unreferenced" - status: pending + status: completed - id: delete-sessions content: "Delete harness-sessions, its workspace member, and its session-only tests; RunMeta session_id becomes name and agent goes; the run-log session_id column becomes name and Workshop's per-run recorder writes the agent column itself" - status: pending + status: completed - id: broker-streaming content: "InferenceBroker::chat drops its transitional OnDelta, and OnDelta leaves the runner and facade; StreamDelta leaves the Engine (bless public-api.txt) for harness-gateway-client with GatewayBroker::chat_streaming and routed-model labels; Workshop per-run broker streams through it with round ids; WorkshopBroker substitutes the current pick per round and warns when the pick's context window is smaller than the launch binding's" - status: pending + status: completed - id: drop-tokio content: "Delete cancel.rs, harness::cancel and cancel.md, and the tokio and tokio-util deps of harness and every harness-internal crate; harness_bans becomes a dependency ban and the tokio clippy.toml entries go; the runner and plugins Invariants name the ban" - status: pending + status: completed - id: docs content: Update archdoc, crates/README.md, crate READMEs, cicerone harness plan, Engine doc pages, and guide chapters; regenerate guide exports (the facade pages change in the Workshop and tokio steps so their doctests pass at every commit) - status: pending + status: completed - id: exit-criteria content: Run the leftover rg searches, every exit criterion including the headless workshop-server suite, and the manual Workshop checks (stream, model swap labels, web) - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-03-2-debt-removal-6-9.md b/vibe/2026-10-03-2-debt-removal-6-9.md index e4b002fbe..a6f33a94e 100644 --- a/vibe/2026-10-03-2-debt-removal-6-9.md +++ b/vibe/2026-10-03-2-debt-removal-6-9.md @@ -4,22 +4,22 @@ overview: "Remove the debt the debt collector kept for commits 1f0554a2..6db7beb todos: - id: d1-1-stale-docs content: "D1-1: fix the stale harness::cancel sentence in promptforge-types cancel.rs and delete the StreamDelta item from the cicerone promptforge plan; verify with repo-wide rg and cicerone surveys" - status: pending + status: completed - id: d1-2-prepare-fields content: "D1-2: remove unread Prepared fields (seed, started_at, parse_events) and PrepareError events; fix docs and result_large_err expectation; rewrite runner tests to read the recorder" - status: pending + status: completed - id: d1-3-resend-unresolved content: "D1-3: delete WaitRegistry::resend_unresolved, its doctest and test; rewrite the WaitFrame and WaitRegistry docs to name the socket's resend" - status: pending + status: completed - id: d1-34-tokio-doc content: "D1-34: document that GatewayBroker needs a tokio runtime in its docs and the harness-gateway-client README" - status: pending + status: completed - id: d1-29-layout-race content: "D1-29: failing test first, then treat an unstamped layout with an empty runs table as fresh in layout::is_stale" - status: pending + status: completed - id: exit-checks content: Run fmt, both clippy runs, nextest for touched crates and workshop-server (default and headless), doctests, workspace docs with -D warnings, and docs-claims - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-03-3-agent-context-cleanup.md b/vibe/2026-10-03-3-agent-context-cleanup.md index 9ad0cd324..25fd4fdf5 100644 --- a/vibe/2026-10-03-3-agent-context-cleanup.md +++ b/vibe/2026-10-03-3-agent-context-cleanup.md @@ -4,43 +4,43 @@ overview: Hide stale docs from the agent, one audited item at a time. todos: - id: "1" content: Remove all rustdoc markdown includes (promptforge, harness, engine) and archive them, plus tools/cicerone, in promptforge-design/saved-docs/ - status: pending + status: completed - id: "2" content: Move or delete every remaining doctest, remove the doctest steps, and add a tidy check that rejects new doctests - status: pending + status: completed - id: "3" content: Rewrite every README.md as a broad, stale-proof statement of responsibility - status: pending + status: completed - id: "4" content: Replace the docs landing page with one table that xtask site generates - status: pending + status: completed - id: "5" content: Remove all crates.io metadata from every Cargo.toml - status: pending + status: completed - id: "6" content: Move the three guides' chapters to promptforge-docs; promptforge CI clones it and builds the whole site - status: pending + status: completed - id: "7" content: Remove stale enforcement code and config - status: pending + status: completed - id: "8" content: One strict workspace lint table; no copies - status: pending + status: completed - id: "9" content: Root clippy.toml bans on installing process-global state - status: pending + status: completed - id: "10" content: Ban serde_json preserve_order in deny.toml - status: pending + status: completed - id: "11" content: Gate settings - docs-link-free clippy, CARGO_BUILD_WARNINGS, hakari verify, git hooks on - status: pending + status: completed - id: "12" content: Shrink AGENTS.md - delete what checks now enforce, fix the wrong fact, reword CSS - status: pending + status: completed - id: "13" content: Make the 500-line limit absolute, compiler-enforced, and encapsulation-safe - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-04-1-tool-context.md b/vibe/2026-10-04-1-tool-context.md index 261ab0ce1..10c2ca981 100644 --- a/vibe/2026-10-04-1-tool-context.md +++ b/vibe/2026-10-04-1-tool-context.md @@ -4,28 +4,28 @@ overview: "Change `Tool::call` to `call(&self, cx: ToolContext<'_>, args)`. The todos: - id: engine-effect content: "Add access: Arc to Effect::ToolCall and update the effect.rs docs; fix run/tests.rs; bless public-api.txt" - status: pending + status: completed - id: engine-fork-join content: In prepare_tool_call fork the call's identity from the chain with access_spawn and keep its ExecId in ToolCallContinuation; in apply_answer join it into the chain on every ToolCall answer, including Dropped - status: pending + status: completed - id: harness-context content: "Add ToolContext<'a> (borrowed access and origin) to tool.rs, change Tool::call, rewrite the trait docs, and re-export ToolContext, ToolCallOrigin, ToolCaller, and Access from the harness facade's plugin and vfs modules" - status: pending + status: completed - id: harness-performer content: Pass the effect's access and origin through ToolPerformer, ActivatedTools (which owns them inside its future and lends them to the tool), and effect_loop perform - status: pending + status: completed - id: remove-services-vfs content: Remove the vfs field from RunServices and the vfs parameter from RunServices::new and with_host; update prepare.rs, the struct and prepare docs, the activation fixture and its two tests, and about 17 test call sites - status: pending + status: completed - id: mechanical content: Update the 7 Tool implementations, about 40 test call sites (with a small owned test fixture per crate that lends a ToolContext), and 7 test ToolPerformer implementations - status: pending + status: completed - id: tests content: "Add the four Engine tests (tool reads the chain's write, chain reads the tool's write, a task's tool call runs under the task, a Dropped call is still joined) and the end-to-end Harness test" - status: pending + status: completed - id: verify content: Run the nextest, clippy, fmt, doc, xtask api, and build-xtask gates - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-04-2-plugin-rename.md b/vibe/2026-10-04-2-plugin-rename.md index 93c490f34..31bb5f1f2 100644 --- a/vibe/2026-10-04-2-plugin-rename.md +++ b/vibe/2026-10-04-2-plugin-rename.md @@ -4,13 +4,13 @@ overview: Rename the activation unit from "capability" to Plugin, a capitalized todos: - id: step-1 content: "Step 1: rename the code to Plugin (crate move, name map, message texts, frontmatter key, Workshop JSON, service-id check, public-api.txt, Cargo.lock)" - status: pending + status: completed - id: step-2 content: "Step 2: extend docs-claims.mjs, define Plugin in the root AGENTS.md, and sweep the repository's prose (rulebooks remove-first)" - status: pending + status: completed - id: step-3 content: "Step 3: sweep the contents of every vibe/ plan and the promptforge-docs guide" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-05-1-debt-removal.md b/vibe/2026-10-05-1-debt-removal.md index 8c56175d1..703e2e816 100644 --- a/vibe/2026-10-05-1-debt-removal.md +++ b/vibe/2026-10-05-1-debt-removal.md @@ -4,16 +4,16 @@ overview: Debt Collector review of the 13 promptforge commits on master over ups todos: - id: end-tool-identity content: "D1-2/3/4: add a VFS primitive that ends a tool call's identity (refuse, join, drop snapshot), call it from apply_answer and abort_effect, with regression tests" - status: pending + status: completed - id: redact-gateway-config content: "D1-16: hand-written redacting Debug on workshop_support::GatewayConfig plus a regression test" - status: pending + status: completed - id: fix-web-tests-comment content: "D1-13: reword crates/harness-web/src/web-tests.rs lines 29-30 to drop the VFS claim" - status: pending + status: completed - id: run-gates content: Run the full root AGENTS.md gates (nextest, clippy, fmt, docs, xtask api --check, build-xtask) - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-05-1-whisper-cuda-backend-debt.md b/vibe/2026-10-05-1-whisper-cuda-backend-debt.md index f7161ee20..25cec651b 100644 --- a/vibe/2026-10-05-1-whisper-cuda-backend-debt.md +++ b/vibe/2026-10-05-1-whisper-cuda-backend-debt.md @@ -9,7 +9,7 @@ overview: >- todos: - id: export-vibe-plan content: Run /export-vibe-plan before any other step - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-06-1-stt-two-model-upgrade.md b/vibe/2026-10-06-1-stt-two-model-upgrade.md index deb1a8e43..7309a034f 100644 --- a/vibe/2026-10-06-1-stt-two-model-upgrade.md +++ b/vibe/2026-10-06-1-stt-two-model-upgrade.md @@ -4,40 +4,40 @@ overview: Upgrade PromptForge's real-time speech-to-text so words lock on a norm todos: - id: safe-fixes content: "Safe fixes: stale 700 ms comment, revision bump only on emitted change, single MAX_PROMPT_TOKENS, reconnect jitter (with tests)" - status: pending + status: completed - id: ffi-layout content: Verify whisper-ffi raw.rs struct layout against whisper.h at b4938; add a layout assertion test if feasible - status: pending + status: completed - id: characterization content: Characterization tests pinning agreed shrink, exact-equality block, final overwrite, queue-full take failure, whole-range applySnapshot - status: pending + status: completed - id: speech-sandbox content: Speech-sandbox with UPWR, UPSR, partial latency, commit lag, agreed-shrink, interim decode time; record baseline - status: pending + status: completed - id: params-guards content: Per-role whisper params (fallback off, audio_ctx, max_tokens, threads) and post-decode hallucination guards - status: pending + status: completed - id: commit-rule content: Normalized evidence-based monotonic commit rule with outlier hold in agreement.rs and window.rs - status: pending + status: completed - id: aligned-rewrite content: Aligned rewrite of final text with anchored, capped fast suffix in state.rs reusing the bounded alignment - status: pending + status: completed - id: timestamps content: Interim token timestamps, window start at last agreed word, force-commit on trim, derived audio_ctx - status: pending + status: completed - id: wire-ui content: Opt-in wire fields, live plain-client deltas, UI revision guard, diff patch, tentative styling, hold-back, undo, a11y, socket-loss keep - status: pending + status: completed - id: endpointing content: Hangover, pre-roll, punctuation-aware short close, pure three-rule endpoint function - status: pending + status: completed - id: interim-prompt content: Seed interim prompt with finalized tail; keep only if speech-sandbox shows less flicker - status: pending + status: completed - id: degrade content: Keep accepted interim text on full final queue with retry; skip overloaded interim ticks - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-07-1-plugin-api-minimal.md b/vibe/2026-10-07-1-plugin-api-minimal.md index 681f206c7..9af16b4c3 100644 --- a/vibe/2026-10-07-1-plugin-api-minimal.md +++ b/vibe/2026-10-07-1-plugin-api-minimal.md @@ -4,25 +4,25 @@ overview: Introduce the Plugin contract crate `promptforge-plugin`, a Host-built todos: - id: step-1-grammar content: "Step 1: one-segment PluginId and two-or-more-segment ToolId, the 53-file literal sweep, registry punctuation on one-segment names, regenerate public-api.txt, gates" - status: pending + status: completed - id: step-2-remove content: "Step 2: remove Plugin conflicts, optional Plugins, and the PluginDecl map form (ref, optional, config); Frontmatter::plugins becomes Vec; Workshop drops optional; regenerate public-api.txt, gates" - status: pending + status: completed - id: step-3-engine content: "Step 3: ToolCallOrigin move, survives_stop wired through the old performer and the effect loop, event tool ids, Workshop ask recognition by the event's tool, Requirements.unavailable/missing_tools, named preludes, regenerate public-api.txt, gates" - status: pending + status: completed - id: step-4-contract content: "Step 4: add promptforge-plugin, move service.rs, harness-plugins re-export bridge, structural-check updates, Cargo.lock and hakari, gates" - status: pending + status: completed - id: step-5-host content: "Step 5: HostContext and HostRunContext, plugin-web and plugin-user-input, Workshop install wiring, delete harness-plugins and performers-tools.rs, fixture Plugins in Harness tests, Family::Plugin rules, Cargo.lock and hakari, gates" - status: pending + status: completed - id: step-6-offering content: "Step 6: the offering: ToolSet.offered, tools.offered(), tools.add and tools.call taking records, chat.md opts in, regenerate public-api.txt, gates" - status: pending + status: completed - id: step-7-docs content: "Step 7: rewrite the root AGENTS.md Plugin definition and the crate docs and Invariants that still describe activation, two-segment ids, or a declared-only catalog, gates" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-07-1-silero-speech-detection.md b/vibe/2026-10-07-1-silero-speech-detection.md index d4e2bd787..c821b4cdc 100644 --- a/vibe/2026-10-07-1-silero-speech-detection.md +++ b/vibe/2026-10-07-1-silero-speech-detection.md @@ -4,34 +4,34 @@ overview: Replace PromptForge's loudness-based speech detection with whisper.cpp todos: - id: loudness-baseline content: Record the loudness baseline on dictation-01 and jfk with the gateway-level capture at the starting commit - status: pending + status: completed - id: abi-verify content: Verify b4938 VAD ABI and choose the Silero ggml model version; record exports and pin - status: pending + status: completed - id: ffi-bindings content: Bind whisper.cpp standalone VAD API with layout assertions and a safe VadContext; native jfk tests - status: pending + status: completed - id: cost-measure content: Measure per-chunk Silero CPU cost; decide inline versus session thread - status: pending + status: completed - id: detector-trait content: SpeechDetector trait, EnergyDetector, scripted detector; segmenter on 512-sample frames, behavior preserved - status: pending + status: completed - id: silero-artifact content: Pin and provision the Silero model; SileroDetector with loudness fallback and reporting - status: pending + status: completed - id: interim-gating content: Skip interim decodes without new speech; end windows at speech end plus tail; echo trim and silent spans from detector - status: pending + status: completed - id: tune-tail content: Add the short-burst close; tune speech tail, thresholds, and short-burst limit on dictation-01 and jfk via gateway-level capture; record research - status: pending + status: completed - id: mic-meter content: Status bar mic meter (scrolling loudness history, owner tooltip, click to reveal) between the barberpole and the record LED - status: pending + status: completed - id: full-verify content: Full verification and success-criteria check against the recorded loudness run - status: pending + status: completed - id: research-refine content: Fold targeted Silero, silence-detection, and hallucination research into this plan before execution status: completed diff --git a/vibe/2026-10-07-2-plugin-api-debt-removal.md b/vibe/2026-10-07-2-plugin-api-debt-removal.md index 9e45f87b0..19087793a 100644 --- a/vibe/2026-10-07-2-plugin-api-debt-removal.md +++ b/vibe/2026-10-07-2-plugin-api-debt-removal.md @@ -4,16 +4,16 @@ overview: "Remove the debt the plugin-api branch (3e917d209..e7c1c7733) added, p todos: - id: package-const-ctor content: "D1-1: make Package #[non_exhaustive] with const fn new/prelude/needs, migrate the 20 literal sites and docs, add the package test" - status: pending + status: completed - id: toolview-dead-methods content: "D1-2 and C-1: delete ToolView::binding and ToolView::offered_binding and the two test assertions" - status: pending + status: completed - id: remove-wire-name content: "D1-22: remove ToolDescriptor::wire_name, InvalidWireName, validate_identifier, TestTool::wire_name, the host-run wire step; update all ToolDescriptor::new calls and tests; regenerate public-api.txt" - status: pending + status: completed - id: exit-gates content: Run every Exit criteria command in the Testing Plan - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-07-2-stt-debt-removal.md b/vibe/2026-10-07-2-stt-debt-removal.md index 12474f2da..8d11e1c78 100644 --- a/vibe/2026-10-07-2-stt-debt-removal.md +++ b/vibe/2026-10-07-2-stt-debt-removal.md @@ -4,31 +4,31 @@ overview: "Remove the five debts kept after the Debt Collector's analysis and ch todos: - id: d1-5-overload-path content: "D1-5 and D1-32: remove the orphaned result_queue_overload path and the whole test-only result queue" - status: pending + status: completed - id: remove-fallback content: "Remove the loudness fallback: Silero is required; provisioning or load failure leaves speech unavailable, a take that cannot open or run Silero fails with an error event; delete FallbackDetector, FallbackReport, Activity::hub, and the native fallback recorder (supersedes D1-14, D1-18, D1-29)" - status: pending + status: completed - id: d1-23-test-constants content: "D1-23: tests use SPEECH_TAIL_SAMPLES, HANGOVER_SAMPLES, and SENTENCE_END_SILENCE_SAMPLES instead of literals" - status: pending + status: completed - id: d1-10-overload-log content: "D1-10: debug log for interim ticks skipped under worker overload" - status: pending + status: completed - id: d1-2-released-guard content: "D1-2: add SPEECH_TAIL_SAMPLES <= HANGOVER_SAMPLES assertion and released-range boundary tests; clip by close kind only if the stride test fails" - status: pending + status: completed - id: d1-1-phrase-veto content: "D1-1: add Bye./you echo tests, trial-remove SILENCE_HALLUCINATIONS, verify with noise test and scored captures" - status: pending + status: completed - id: d1-4-ranges-request content: "D1-4: Workshop stops requesting and decoding the hypothesis ranges extension; update wire fixture tests" - status: pending + status: completed - id: d1-3-native-fixture content: "D1-3: re-capture jfk native replay as jfk-native-recaptured, point NATIVE_FIXTURE at it, document no hand edits" - status: pending + status: completed - id: exit-checks content: "Exit: full build, fmt, lint, docs, full suite, Miri, native noise test, and scored captures" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-08-1-mcp-client-plugin.md b/vibe/2026-10-08-1-mcp-client-plugin.md index 321adc097..232a06cf2 100644 --- a/vibe/2026-10-08-1-mcp-client-plugin.md +++ b/vibe/2026-10-08-1-mcp-client-plugin.md @@ -4,22 +4,22 @@ overview: Add a `plugin-mcp` crate that connects PromptForge to remote MCP serve todos: - id: runtime-name content: "Step 1: name the tokio runtime service in promptforge-plugin, make web's key private, and build Workshop's key from the name" - status: pending + status: completed - id: contract-ready content: "Step 2: add defaulted Plugin::ready and the Harness readiness wait before each run's snapshot, end a refusal whose run was cancelled as cancelled, with tests" - status: pending + status: completed - id: plugin-mcp-crate content: "Step 3: add crates/plugin-mcp on rmcp's Streamable HTTP client with PACKAGE as its only public item, refusing local entries as not supported yet, plus entry, connect, server, and result modules and fixture tests" - status: pending + status: completed - id: workshop-install content: "Step 4: add [agents] mcp to workshop.toml, read the named mcp.json, and install one MCP Plugin per server" - status: pending + status: completed - id: remove-reserved-mcp content: "Step 5: remove the reserved Lua mcp request and its parse, dispatch, and tests" - status: pending + status: completed - id: verify-live content: "After Step 5: the operator's live check: github and wg21-papers work in chat, and pinecone-search and wg21-wiki are installed as unavailable with the not-supported reason" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-08-2-copy-cursor-everywhere.md b/vibe/2026-10-08-2-copy-cursor-everywhere.md index 696f20275..a5272a2e6 100644 --- a/vibe/2026-10-08-2-copy-cursor-everywhere.md +++ b/vibe/2026-10-08-2-copy-cursor-everywhere.md @@ -4,34 +4,34 @@ overview: Make every Workshop surface that has a Cursor counterpart look and beh todos: - id: step-1 content: "Step 1: IDE tokens, text tiers, inline codicons, shimmer, controls, progress, focus, scrollbars, and two dialog skins in look and ui" - status: pending + status: completed - id: step-2 content: "Step 2: reasoning timing, generating and reconnecting flags, tool ids, cancel, error titles, and the pure transcript model" - status: pending + status: completed - id: step-3 content: "Step 3: keyed transcript view, rows, markdown, feed scroll, turn footer, context menu, toast token, and the composer error popup" - status: pending + status: completed - id: step-4 content: "Step 4: composer, menus, agent keybindings, zone attribute, pane header, chat tabs, rename, empty state, and the lucide removal from ui" - status: pending + status: completed - id: step-5 content: "Step 5: title bar, menu skin and structure, zoom keys, About, and the Windows quit accelerator" - status: pending + status: completed - id: step-6 content: "Step 6: dockview palette, tabs, dirty dot, watermark, drop zones, default sizes, and the file tree" - status: pending + status: completed - id: step-7 content: "Step 7: editor theme, Cursor Dark highlight style, find widget, whitespace modes, and the Shiki theme" - status: pending + status: completed - id: step-8 content: "Step 8: quick input, toasts, status bar, command failure toasts, and workbench inputs" - status: pending + status: completed - id: step-9 content: "Step 9: run panel as Cursor Settings rows, with form-modal dialogs" - status: pending + status: completed - id: step-10 content: "Step 10: gateway config app on Cursor's settings look, then the branch-wide gate" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-08-2-stt-report-fixes.md b/vibe/2026-10-08-2-stt-report-fixes.md index fab892b24..7d4df8d33 100644 --- a/vibe/2026-10-08-2-stt-report-fixes.md +++ b/vibe/2026-10-08-2-stt-report-fixes.md @@ -4,25 +4,25 @@ overview: "Fix the promptforge gateway STT defects that the talktron evaluation todos: - id: s6-silent-exits content: "S6: skip budget-starved captions; route the three silent returns in realtime/route.rs through end_session (log, error event, 1011 close); tests" - status: pending + status: completed - id: s1-overlap-guard content: "S1: forward-only punctuation snap, AlignmentFailure reason + metrics logged, sparse-successor density guard with PendingForced.text_start; adversarial tests" - status: pending + status: completed - id: s3-pcm-errors content: "S3: count each append once in pcm.rs try_append; BufferTooLong carries retained_ms and requested_ms; ClientError constructors take impl Into; transcription_failed includes cause; replace finalization panic with TakeFailure::FinalTailEvicted" - status: pending + status: completed - id: s2-biasing content: "S2: prompt_terms helper; realtime guidance = config vocabulary + session prompt terms; batch honors prompt; update config doc and README; tests" - status: pending + status: completed - id: s5-model-name content: "S5: ModelNames::select maps realtime-transcribe to the final model; test" - status: pending + status: completed - id: s4-long-speech-test content: "S4: native realtime_long_speech test (no 5-word deletion run), fixture zip + hash, stt-miri.yml download and step" - status: pending + status: completed - id: verify-gates content: Run the full workspace gates (clippy with warnings denied, fmt, rustdoc, nextest) on the final code commit - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-08-3-debt-removal-mcp-cursor.md b/vibe/2026-10-08-3-debt-removal-mcp-cursor.md index cee40bcd0..2aba3bb29 100644 --- a/vibe/2026-10-08-3-debt-removal-mcp-cursor.md +++ b/vibe/2026-10-08-3-debt-removal-mcp-cursor.md @@ -4,13 +4,13 @@ overview: "Remove the debt the Debt Collector kept from promptforge's 19 target todos: - id: d1-2-remove-search content: "D1-2: update agent-session-view.mjs expectations first (confirm failure), then delete the Search with Google menu item and searchWithGoogle from transcript-view.ts; grep confirms no remaining references under crates" - status: pending + status: completed - id: d1-9-stopped-flag content: "D1-9: add the failing late-reasoning-after-Stop case to agent-session-service.mjs, then add the private stopped flag (set in cancelTurn; cleared in respond, onInputRequired, acknowledge) and open the late thought already ended in foldDelta; check whether a late text delta reproduces the same stuck row" - status: pending + status: completed - id: d1-1-runtime-key-guard content: "D1-1 option A: add the server test installing a valid remote MCP entry (closed loopback URL) through host_context and asserting the refusal names the handshake failure, not promptforge/tokio-runtime; run the fault-injection check on the server's RUNTIME key and revert it" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-08-3-stt-fix-debt.md b/vibe/2026-10-08-3-stt-fix-debt.md index f1882df99..85e8825d1 100644 --- a/vibe/2026-10-08-3-stt-fix-debt.md +++ b/vibe/2026-10-08-3-stt-fix-debt.md @@ -4,13 +4,13 @@ overview: "Remove the two debts that the STT report-fix work added: the sparse-s todos: - id: d1-1-successor-cut content: "D1-1: cut the sparse successor at the projected token with no punctuation snap, log the cut position, add the punctuated-successor regression test (fails first)" - status: in_progress + status: completed - id: d1-2-readme content: "D1-2: fix README lines 33, 43, 58 and 80 so CI scope and forced-window history match the code" - status: pending + status: completed - id: exit-checks content: Run STT crate tests, fmt, clippy with warnings denied, and the native long-speech test if fixtures are available - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-08-4-mcp-plugin-debt-removal.md b/vibe/2026-10-08-4-mcp-plugin-debt-removal.md index 7f0312657..05e995bd2 100644 --- a/vibe/2026-10-08-4-mcp-plugin-debt-removal.md +++ b/vibe/2026-10-08-4-mcp-plugin-debt-removal.md @@ -4,19 +4,19 @@ overview: "Removal plan from the Debt Collector run on promptforge2 commits 20aa todos: - id: w1-invariants-list content: "D1-1: edit the Tier bullet in crates/workshop/server/src/lib.rs to name plugin-mcp and promptforge-plugin; run docs-claims (npm test in crates/workshop/ui) and cargo test -p build-xtask" - status: pending + status: completed - id: w2-serde-verify content: "D1-2: verify serde is absent from plugin-mcp's Cargo.toml and Cargo.lock entry (already removed on HEAD by a1201da73); change nothing if so" - status: pending + status: completed - id: w3-runtime-leg-test content: "D1-3 option A: add the remote-entry behavior test and header clause in crates/workshop/server/src/agents/tests.rs" - status: pending + status: completed - id: w3-fault-injection content: "D1-3: inject the wrong reader literal in plugin-mcp lib.rs line 91, confirm the new test and plugin-mcp's integration tests fail, then restore the file and confirm git diff is clean of it" - status: pending + status: completed - id: exit-gates content: "Run exit checks from AGENTS.md Verification: formatter, Workshop clippy with warnings denied, Workshop nextest trio, cargo test -p build-xtask" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-09-1-finish-search-removal.md b/vibe/2026-10-09-1-finish-search-removal.md index fc774eba5..b3ee05330 100644 --- a/vibe/2026-10-09-1-finish-search-removal.md +++ b/vibe/2026-10-09-1-finish-search-removal.md @@ -4,10 +4,10 @@ overview: "Finish what the debt-removal run missed and close out the four follow todos: - id: f1-finish-removal-text content: "F1: reword the split-line header comment in transcript-view.ts and edit lines 558 and 1051 of the closed plan record vibe/2026-10-08-2-copy-cursor-everywhere.md; verify with a multi-line search and the workshop UI typecheck" - status: pending + status: completed - id: f2-config-panel-context-menu content: "F2: add panel and standalone contextmenu tests to config-ui mode.test.mjs (npm run build first, panel test must fail), then install a capture-phase contextmenu preventDefault on root.ownerDocument in boot()'s panel branch in main.ts; run config-ui npm test and typecheck" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-09-2-messages-userdata-refactor.md b/vibe/2026-10-09-2-messages-userdata-refactor.md index 163f17119..9604f961e 100644 --- a/vibe/2026-10-09-2-messages-userdata-refactor.md +++ b/vibe/2026-10-09-2-messages-userdata-refactor.md @@ -4,22 +4,22 @@ overview: Replace the pure-Lua messages.new() builder with a Rust-backed list th todos: - id: step-1-sandbox-pairs content: "Step 1: the sandbox pairs in iteration.rs honors a userdata __pairs, with tests" - status: pending + status: completed - id: step-2-list-core content: "Step 2: MessageList core in messages.rs (builders, replace, system-first, extra fields dropped, __len, __newindex, commit) plus parse_record, with tests" - status: pending + status: completed - id: step-3-record-views content: "Step 3: read-only RecordView, MessageRecord::to_json, list __index (integral floats too) and __pairs, with tests" - status: pending + status: completed - id: step-4-cutover content: "Step 4: messages.new() returns the Rust list; shim append and handle rule; parse_chat takes the list; engine imports swap MessageRecord for MessageList; test migrations, argument and author-shape tests; full test commands" - status: pending + status: completed - id: step-5-chat-record content: "Step 5: after and keep on Effect::Chat and EffectRecord::Chat, commit after the precheck, listing re-blessed, rebuild and refused-round tests; full gates" - status: pending + status: completed - id: docs-follow-up content: "Follow-up outside the run: rewrite promptforge-docs chapters 11 and 17 and rebuild the books" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-09-3-models-loop-in-rust.md b/vibe/2026-10-09-3-models-loop-in-rust.md index 76aacd853..8fb527807 100644 --- a/vibe/2026-10-09-3-models-loop-in-rust.md +++ b/vibe/2026-10-09-3-models-loop-in-rust.md @@ -4,25 +4,25 @@ overview: First replace models.loop's and models.infer's optional leading handle todos: - id: handle-methods content: Phase 0 - models.loop(messages, compactor?) and models.infer(prompt) run on the section's current model, and model handles gain h:loop(messages, compactor?) and h:infer(prompt), which run on the handle's model, through two field getters over one registry table; models::is_handle backs the new is_model_handle capture; the shim splits each function into an entry per form over a shared body, checks the list before the first yield, and drops the drain guard and the guessing rule; every handle-form call site, test, and Rust doc comment migrates; the methodless-handle bullet is deleted from lua/AGENTS.md and the lua crate's Invariants list; handles_reject_colon_methods is rewritten; new argument tests in models_loop-arguments.rs, shims.rs, and errors.rs - status: pending + status: completed - id: loop-docs content: Phase 0 docs - rewrite the handle-first passages of promptforge-docs chapters 05, 10, 11, and 17 for h:loop and h:infer, together with the messages refactor's open chapter 11 and 17 follow-up, and rebuild the books, as a vibe step committed in promptforge-docs - status: pending + status: completed - id: characterization-tests content: Add the gap tests from the behavior inventory to execute/tests/models_loop_contract.rs (a flat peer registered in execute/tests.rs) and engine/src/lua/tests/models_loop_contract.rs, driven through models.loop and h:loop from Lua; they pass on the Phase 0 shim before any Phase 1 code exists - status: pending + status: completed - id: loop-machine content: Phase 1 port - models_loop.rs adapter, models_loop-machine.rs typed machine (Machine, Phase, Input, Then over the existing ChatResult, ToolCallEvent, ToolCallRecord, MessageList, and Raised), models_loop-tests.rs VM-free and adapter tests; loop_begin passed as the shim chunk's last argument in place of compactors, max_tool_iterations, and is_message_list, returning a per-call step closure that owns the call's machine; one trampoline behind both loop entries replaces run_loop, append_record, drain_task_notices, EMPTY_MODEL_REPLY, NOT_A_LIST, and compact; normalized and the parse's list text shared; quota.rs rewritten; shim, coro.rs, quota.rs, and bench comments updated - status: pending + status: completed - id: drain-fold content: Phase 2, after Phase 1 - prepare_chat drains first and pushes notices onto the list before it reads the records and commits, the empty-list refusal moves from parse_chat to after that push, and Request::DrainTaskNotices and Answer::DrainTaskNotices go with their arms and tests; MessageList::push becomes pub so the Engine can call it - status: pending + status: completed - id: chat-handle content: Phase 3, after Phase 2 - the chat answer resumes as the scheduler's own ChatResult as an opaque userdata that the step takes, and chat_result_table and the adapter's table reader go with their table-shape tests - status: pending + status: completed - id: verify content: AGENTS.md verification commands after each landed change, the facade surface check with no diff, and a clean docs build - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-09-4-models-loop-debt-removal.md b/vibe/2026-10-09-4-models-loop-debt-removal.md index 51f23712b..e731bcb93 100644 --- a/vibe/2026-10-09-4-models-loop-debt-removal.md +++ b/vibe/2026-10-09-4-models-loop-debt-removal.md @@ -4,16 +4,16 @@ overview: "Remove the two debts the Debt Collector kept from the models-loop-in- todos: - id: c1-capture-error content: "C-1 and D1-20: write the rebound-error regression test (fails as Interrupted), then capture error chunk-wide in __impl_coro.lua; delete the unread infer return key, the stale H1 comment, and the dead models and tools guards" - status: pending + status: completed - id: d1-1-chatresult-fields content: "D1-1, C-2, C-3, D1-15: remove ChatResult.model and .metrics, their four initializers in scheduler/chat.rs, and test literal lines; fix the stale loop-shim wording and the empty-reply doc line in the touched docs; keep one shared copy of the call() and round() test fixtures" - status: pending + status: completed - id: d1-2-messages-doc content: "D1-2: correct the messages.rs module doc that says the round reads only author-built records with no second validation" - status: pending + status: completed - id: exit-checks content: Run the AGENTS.md Verification list; facade surface check shows no diff - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-09-5-author-api-reshape.md b/vibe/2026-10-09-5-author-api-reshape.md index c069d5f7e..817f8ed12 100644 --- a/vibe/2026-10-09-5-author-api-reshape.md +++ b/vibe/2026-10-09-5-author-api-reshape.md @@ -4,19 +4,19 @@ overview: "Reshape the prompt-author API in four changes: role labels stop being todos: - id: p1-role-globals content: "Role-globals change: delete the model half of install_captured_bindings, stop treating model labels as globals in the parser, drop them from the prelude collision list, migrate the 2 role-global calls and 3 other bare role-label reads in 3 test files to models.get" - status: pending + status: completed - id: p2-tools-by-id content: "Tools change: delete tools: slots, slot fill, and tool alias globals; the offering holds every catalog tool and resolves ids; ToolSet swaps bindings for declared; tool objects; tools.offer, always_offer, offer_local, required, extras, get, call and calls by id; consumers including the Workshop run panel, error texts, facade listing, tests" - status: pending + status: completed - id: p3-model-provider content: "Provider change: optional provider on the gateway's Capabilities and its validation, client catalog decode, ModelDescriptor, ModelBinding, handle.provider, facade listing, tests" - status: pending + status: completed - id: p4-plugins-table content: "Plugins change: plugins global with required, extras, get over one shared read-only object per plugin; tool objects gain plugin; RESERVED_NAMES; section and H1 install; tests" - status: pending + status: completed - id: gates content: Run the repository verification commands after each change; the facade listing changes only in the tools change and the provider change - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-10-1-author-api-debt-removal.md b/vibe/2026-10-10-1-author-api-debt-removal.md index 8ebec10b3..e77750178 100644 --- a/vibe/2026-10-10-1-author-api-debt-removal.md +++ b/vibe/2026-10-10-1-author-api-debt-removal.md @@ -4,16 +4,16 @@ overview: Remove the one debt the author API reshape introduced (a false "not of todos: - id: d1-1-wire-lookup content: "D1-1: add ToolSet::wire_binding, use it in model-call dispatch and the scope gate's global_exists, fix the field doc, with a failing-first canonical-id test in tool_loop.rs" - status: pending + status: completed - id: d1-2-local-gate content: "D1-2: run node --test crates/workshop/ui/test/docs-claims.mjs from the root AGENTS.md Verification list and .githooks/pre-push, proven by fault injection" - status: pending + status: completed - id: d1-12-renames content: "D1-12: rename ToolsAddEntry, collect_tools_add_entries, add_local_params_schema, tool_alias, and ToolCallCounts::aliases (with its alias parameters) to names that match the offer API" - status: pending + status: completed - id: d1-17-harness-doc content: "D1-17: correct the ToolPerformer::call doc in performers.rs so alias is the wire name, or the canonical id when the run could not offer the tool" - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-10-2-prompt-namespace-reads.md b/vibe/2026-10-10-2-prompt-namespace-reads.md index 00c333b07..99f778a54 100644 --- a/vibe/2026-10-10-2-prompt-namespace-reads.md +++ b/vibe/2026-10-10-2-prompt-namespace-reads.md @@ -4,19 +4,19 @@ overview: Add a `prompt` Lua namespace with `prompt.section(heading)` and `promp todos: - id: parser-markdown content: Add Section.markdown and its accessor; set it in build_sections from Heading.content with trim_leading_blank_lines (now pub(crate)) and trim_end; synthetic_section defaults it to empty; one parser test in tests/headings.rs (blank line above its ---) - status: pending + status: completed - id: scope-lookup content: Add find_visible in heading_address.rs as the one heading lookup (Ok(None) when the visible set has no such heading) and make resolve_sibling a thin adapter over it with unchanged messages; one unit test in heading_address-tests.rs; lands alone, after parser-markdown in the same checkout (or beside it in a separate worktree) - status: pending + status: completed - id: prompt-change content: "One atomic change in three parts. Table: SectionText, read_visible replacing list_items_from_visible (plus the stale home_without doc paragraph), install_prompt_table replacing install_list_global (Send + Sync read callback), prompt for list_from_section in RESERVED_NAMES, the read callback through setup_section_vm and both SectionContext constructors, the seven test stubs. Removal: every remaining list_from_section use (calls, test names, doc comments, the four fanout fixtures, the collection.rs error). Tests: prompt_reads.rs replacing list_from_section.rs, the two nil-rule edits, the three deletions" - status: pending + status: completed - id: docs content: "Update the language spec: index entries, the whole H1 availability sentence (call and jump work there too), the deferred markdown note, and the list_from_section API entry replaced by prompt.section and prompt.list entries with Implementation status lines" - status: pending + status: completed - id: verify content: nextest (both sets), clippy with warnings denied (both sets), fmt, docs and facade docs builds, xtask api --check, build-xtask tests, the docs-claims test, and a list_from_section search over crates/ and the spec that finds nothing - status: pending + status: completed isProject: false --- diff --git a/vibe/2026-10-10-3-prompt-reads-debt-removal.md b/vibe/2026-10-10-3-prompt-reads-debt-removal.md index d81dc089a..59287a1ce 100644 --- a/vibe/2026-10-10-3-prompt-reads-debt-removal.md +++ b/vibe/2026-10-10-3-prompt-reads-debt-removal.md @@ -222,7 +222,7 @@ isProject: false -### Step 3: Mark closed plans' todos completed, going forward and across vibe/ history +### Step 3: Mark closed plans' todos completed, going forward and across vibe/ history [completed] - Component: Plan history - Placement: third. It depends on no other component. It follows the code components so the 133-file data edit stays out of the `[WIP] Plan:` seed commit that Step 1 amends, and it precedes the last step so that step's exit gate runs once after every work item lands. From 1178c76940c9493524c16c5b378ffc8f9f5dd5f4 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 16:07:41 -0700 Subject: [PATCH 08/34] Rename fanout-arm test and fix reserved-alias docs A fanout-arm test's name said both lookups reject a section outside the worker's visible set, but only the call raises and the listing reads nil. The new name states each outcome. Two stale doc comments said the parser refuses a frontmatter tool alias with a reserved name. One now says only that a Plugin prelude global may not take a reserved name, and the other drops the case from the frontmatter error list. No runtime behavior changes. - `call_raises_and_list_reads_nil_for_non_visible_sections_in_a_fanout_arm` replaces `call_and_list_inside_a_fanout_arm_reject_non_visible_sections`. The test body and its doc comment are unchanged. - `RESERVED_NAMES` module doc keeps the definition of the reserved set and now limits the rule to Plugin prelude globals. The sentence about frontmatter tool aliases installing as globals is gone. - `Frontmatter` error causes in the `Prompt` parse docs no longer include a reserved tool alias. The missing delimiters, decode failure, non-plain Plugin entry, and duplicate Plugin causes stay. Plan: vibe/2026-10-10-3-prompt-reads-debt-removal.md --- .../engine/src/execute/tests/suite/exec_flow/fanout_arms.rs | 2 +- crates/promptforge-internal/lua/src/globals.rs | 5 ++--- crates/promptforge-internal/parser/src/parse.rs | 4 ++-- vibe/2026-10-10-3-prompt-reads-debt-removal.md | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_arms.rs b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_arms.rs index cfcc26975..2972afba3 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_arms.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/fanout_arms.rs @@ -255,7 +255,7 @@ jump('## Parent')\n\ /// worker's visible set both reject it: `call` errors not-found, which the /// arm catches with `pcall` and asserts on, and `prompt.list` reads `nil`. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn call_and_list_inside_a_fanout_arm_reject_non_visible_sections() { +async fn call_raises_and_list_reads_nil_for_non_visible_sections_in_a_fanout_arm() { let md = [ ARM_FANOUT_PARENT, "### Worker\n\n\ diff --git a/crates/promptforge-internal/lua/src/globals.rs b/crates/promptforge-internal/lua/src/globals.rs index e19510047..aab86c150 100644 --- a/crates/promptforge-internal/lua/src/globals.rs +++ b/crates/promptforge-internal/lua/src/globals.rs @@ -28,9 +28,8 @@ //! //! [`RESERVED_NAMES`] is the other half of `_G`'s contract: every name the //! globals table holds once section setup ends, before any Plugin -//! prelude installs, plus the Lua keywords. A frontmatter tool alias -//! installs as a global of its own name, so the parser refuses one that is -//! reserved, and a prelude global may not take one either. +//! prelude installs, plus the Lua keywords. A Plugin prelude global may not +//! take one. use std::fmt; use std::sync::LazyLock; diff --git a/crates/promptforge-internal/parser/src/parse.rs b/crates/promptforge-internal/parser/src/parse.rs index 7ec82b89f..128142ebc 100644 --- a/crates/promptforge-internal/parser/src/parse.rs +++ b/crates/promptforge-internal/parser/src/parse.rs @@ -29,8 +29,8 @@ impl Prompt { /// kind is: /// /// - `Frontmatter` when the `---` delimiters are missing, the frontmatter - /// fails to decode, a tool alias is a reserved name, a Plugin entry is - /// not a plain name, or a Plugin is declared twice. + /// fails to decode, a Plugin entry is not a plain name, or a Plugin is + /// declared twice. /// - `Structure` when the H1 is missing, there is more than one H1, the H1 /// title is empty, a section heading skips a level (such as an H4 /// directly under an H2), a section heading is empty, or two sibling diff --git a/vibe/2026-10-10-3-prompt-reads-debt-removal.md b/vibe/2026-10-10-3-prompt-reads-debt-removal.md index 59287a1ce..3f3a91300 100644 --- a/vibe/2026-10-10-3-prompt-reads-debt-removal.md +++ b/vibe/2026-10-10-3-prompt-reads-debt-removal.md @@ -247,7 +247,7 @@ isProject: false -### Step 4: Correct the fanout-arm test name, the reserved-alias docs, and the spec status lines, then run the exit gate +### Step 4: Correct the fanout-arm test name, the reserved-alias docs, and the spec status lines, then run the exit gate [completed] - Component: Names and docs - Placement: last. It depends on no other component, and the exit gate attached to it runs once after every work item lands, so any gate fix goes into a commit that is not yet finalized. From ceafc3a2ac7947a059af2543d289feefffb35fd7 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 16:21:54 -0700 Subject: [PATCH 09/34] Close plan: prompt-reads-debt-removal Plan: vibe/2026-10-10-3-prompt-reads-debt-removal.md --- vibe/2026-10-10-3-prompt-reads-debt-removal.md | 10 +++++----- vibe/ACTIVE | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/2026-10-10-3-prompt-reads-debt-removal.md b/vibe/2026-10-10-3-prompt-reads-debt-removal.md index 3f3a91300..f0e709578 100644 --- a/vibe/2026-10-10-3-prompt-reads-debt-removal.md +++ b/vibe/2026-10-10-3-prompt-reads-debt-removal.md @@ -4,19 +4,19 @@ overview: Remove the one cheap fix the prompt-namespace-reads work left, adopt t todos: - id: parser-cleanup content: "D1-5, C-1: split build_sections into four named checks; drop synthetic_section's items parameter and update its four call sites" - status: pending + status: completed - id: scope-module content: "D1-14, D1-15: add h1_visible_sections beside visible_sections and use it at the three H1 sites; rename resolve_sibling to resolve_visible with its tests and doc links" - status: pending + status: completed - id: names-docs content: "C-2, C-3, X-1: rename the fanout-arm test, fix the reserved tool-alias docs in globals.rs and parse.rs, set the two spec status lines to Active. (promptforge-design commit)" - status: pending + status: completed - id: plan-history content: "D1-16: vibe coder close step marks todos completed (tools-public commit); flip todo statuses in every closed vibe/ plan, preserving line endings" - status: pending + status: completed - id: exit-gate content: "Full exit gate: both nextest runs, both clippy runs, fmt, docs builds, xtask api --check, build-xtask tests, docs-claims, and the rg checks" - status: pending + status: completed isProject: false --- diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 5134f338e..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-10-10-3-prompt-reads-debt-removal.md From 05d130eea005b774ac63bee5b695297c25636ce6 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 10 Oct 2026 19:35:58 -0700 Subject: [PATCH 10/34] Tidy up vibe/ plans --- vibe/{ => 2026-10}/2026-10-01-1-remove-task-event-reads.md | 0 vibe/{ => 2026-10}/2026-10-01-2-store-to-vfs-inline.md | 0 vibe/{ => 2026-10}/2026-10-02-1-gateway-client.md | 0 vibe/{ => 2026-10}/2026-10-02-2-host-services.md | 0 vibe/{ => 2026-10}/2026-10-02-3-debt-removal.md | 0 vibe/{ => 2026-10}/2026-10-02-4-inference-broker.md | 0 vibe/{ => 2026-10}/2026-10-03-1-harness-per-run.md | 0 vibe/{ => 2026-10}/2026-10-03-2-debt-removal-6-9.md | 0 vibe/{ => 2026-10}/2026-10-03-3-agent-context-cleanup.md | 0 vibe/{ => 2026-10}/2026-10-04-1-tool-context.md | 0 vibe/{ => 2026-10}/2026-10-04-2-plugin-rename.md | 0 vibe/{ => 2026-10}/2026-10-05-1-debt-removal.md | 0 vibe/{ => 2026-10}/2026-10-05-1-whisper-cuda-backend-debt.md | 0 vibe/{ => 2026-10}/2026-10-06-1-stt-two-model-upgrade.md | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename vibe/{ => 2026-10}/2026-10-01-1-remove-task-event-reads.md (100%) rename vibe/{ => 2026-10}/2026-10-01-2-store-to-vfs-inline.md (100%) rename vibe/{ => 2026-10}/2026-10-02-1-gateway-client.md (100%) rename vibe/{ => 2026-10}/2026-10-02-2-host-services.md (100%) rename vibe/{ => 2026-10}/2026-10-02-3-debt-removal.md (100%) rename vibe/{ => 2026-10}/2026-10-02-4-inference-broker.md (100%) rename vibe/{ => 2026-10}/2026-10-03-1-harness-per-run.md (100%) rename vibe/{ => 2026-10}/2026-10-03-2-debt-removal-6-9.md (100%) rename vibe/{ => 2026-10}/2026-10-03-3-agent-context-cleanup.md (100%) rename vibe/{ => 2026-10}/2026-10-04-1-tool-context.md (100%) rename vibe/{ => 2026-10}/2026-10-04-2-plugin-rename.md (100%) rename vibe/{ => 2026-10}/2026-10-05-1-debt-removal.md (100%) rename vibe/{ => 2026-10}/2026-10-05-1-whisper-cuda-backend-debt.md (100%) rename vibe/{ => 2026-10}/2026-10-06-1-stt-two-model-upgrade.md (100%) diff --git a/vibe/2026-10-01-1-remove-task-event-reads.md b/vibe/2026-10/2026-10-01-1-remove-task-event-reads.md similarity index 100% rename from vibe/2026-10-01-1-remove-task-event-reads.md rename to vibe/2026-10/2026-10-01-1-remove-task-event-reads.md diff --git a/vibe/2026-10-01-2-store-to-vfs-inline.md b/vibe/2026-10/2026-10-01-2-store-to-vfs-inline.md similarity index 100% rename from vibe/2026-10-01-2-store-to-vfs-inline.md rename to vibe/2026-10/2026-10-01-2-store-to-vfs-inline.md diff --git a/vibe/2026-10-02-1-gateway-client.md b/vibe/2026-10/2026-10-02-1-gateway-client.md similarity index 100% rename from vibe/2026-10-02-1-gateway-client.md rename to vibe/2026-10/2026-10-02-1-gateway-client.md diff --git a/vibe/2026-10-02-2-host-services.md b/vibe/2026-10/2026-10-02-2-host-services.md similarity index 100% rename from vibe/2026-10-02-2-host-services.md rename to vibe/2026-10/2026-10-02-2-host-services.md diff --git a/vibe/2026-10-02-3-debt-removal.md b/vibe/2026-10/2026-10-02-3-debt-removal.md similarity index 100% rename from vibe/2026-10-02-3-debt-removal.md rename to vibe/2026-10/2026-10-02-3-debt-removal.md diff --git a/vibe/2026-10-02-4-inference-broker.md b/vibe/2026-10/2026-10-02-4-inference-broker.md similarity index 100% rename from vibe/2026-10-02-4-inference-broker.md rename to vibe/2026-10/2026-10-02-4-inference-broker.md diff --git a/vibe/2026-10-03-1-harness-per-run.md b/vibe/2026-10/2026-10-03-1-harness-per-run.md similarity index 100% rename from vibe/2026-10-03-1-harness-per-run.md rename to vibe/2026-10/2026-10-03-1-harness-per-run.md diff --git a/vibe/2026-10-03-2-debt-removal-6-9.md b/vibe/2026-10/2026-10-03-2-debt-removal-6-9.md similarity index 100% rename from vibe/2026-10-03-2-debt-removal-6-9.md rename to vibe/2026-10/2026-10-03-2-debt-removal-6-9.md diff --git a/vibe/2026-10-03-3-agent-context-cleanup.md b/vibe/2026-10/2026-10-03-3-agent-context-cleanup.md similarity index 100% rename from vibe/2026-10-03-3-agent-context-cleanup.md rename to vibe/2026-10/2026-10-03-3-agent-context-cleanup.md diff --git a/vibe/2026-10-04-1-tool-context.md b/vibe/2026-10/2026-10-04-1-tool-context.md similarity index 100% rename from vibe/2026-10-04-1-tool-context.md rename to vibe/2026-10/2026-10-04-1-tool-context.md diff --git a/vibe/2026-10-04-2-plugin-rename.md b/vibe/2026-10/2026-10-04-2-plugin-rename.md similarity index 100% rename from vibe/2026-10-04-2-plugin-rename.md rename to vibe/2026-10/2026-10-04-2-plugin-rename.md diff --git a/vibe/2026-10-05-1-debt-removal.md b/vibe/2026-10/2026-10-05-1-debt-removal.md similarity index 100% rename from vibe/2026-10-05-1-debt-removal.md rename to vibe/2026-10/2026-10-05-1-debt-removal.md diff --git a/vibe/2026-10-05-1-whisper-cuda-backend-debt.md b/vibe/2026-10/2026-10-05-1-whisper-cuda-backend-debt.md similarity index 100% rename from vibe/2026-10-05-1-whisper-cuda-backend-debt.md rename to vibe/2026-10/2026-10-05-1-whisper-cuda-backend-debt.md diff --git a/vibe/2026-10-06-1-stt-two-model-upgrade.md b/vibe/2026-10/2026-10-06-1-stt-two-model-upgrade.md similarity index 100% rename from vibe/2026-10-06-1-stt-two-model-upgrade.md rename to vibe/2026-10/2026-10-06-1-stt-two-model-upgrade.md From c41bfb81b3b7ca025c4380d53a1bc0014ccc1774 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 03:48:54 -0700 Subject: [PATCH 11/34] Add a one-line description to each Plugin package Every Plugin package can now carry a short line saying what it offers, so a Host listing the packages it can install has something to show beside each name. The line is optional and empty by default. The web search, user input, and MCP server packages each set one. - `Package::description` is both a public field and a `const fn` builder beside `prelude` and `needs`, so a Plugin crate sets it in the same chain that builds its `PACKAGE` constant. `Package::new` sets it to an empty string, so setting it is optional. - `promptforge/web`, `promptforge/user-input`, and `promptforge/mcp` now describe themselves as "Search the web and fetch pages", "Ask the person running the prompt", and "Tools from one MCP server". - `fmt::Debug` for `Package` now shows the description between the name and the prelude flag. - `description` is read only by `fmt::Debug` in the touched files, and the change adds no tests. Design: extends surface-growth @ crates/promptforge-plugin/src/plugin.rs::Package boundary: pub Design: new surface-growth @ crates/promptforge-plugin/src/plugin.rs::Package::description boundary: pub Design: extends surface-growth @ crates/plugin-web/src/web.rs::PACKAGE boundary: pub Design: extends surface-growth @ crates/plugin-user-input/src/lib.rs::PACKAGE boundary: pub Design: extends surface-growth @ crates/plugin-mcp/src/lib.rs::PACKAGE boundary: pub Deferred: Nothing reads Package::description beyond its Debug output yet Plan: vibe/2026-10-11-1-package-registry.md --- crates/plugin-mcp/src/lib.rs | 3 +- crates/plugin-user-input/src/lib.rs | 1 + crates/plugin-web/src/web.rs | 3 +- crates/promptforge-plugin/src/plugin.rs | 27 +- vibe/2026-10-11-1-package-registry.md | 514 ++++++++++++++++++++++++ vibe/ACTIVE | 1 + 6 files changed, 542 insertions(+), 7 deletions(-) create mode 100644 vibe/2026-10-11-1-package-registry.md create mode 100644 vibe/ACTIVE diff --git a/crates/plugin-mcp/src/lib.rs b/crates/plugin-mcp/src/lib.rs index 3de767ee1..7eae70aa8 100644 --- a/crates/plugin-mcp/src/lib.rs +++ b/crates/plugin-mcp/src/lib.rs @@ -85,7 +85,8 @@ use crate::server::{Server, State}; /// passing the server's `mcp.json` entry as the configuration, and /// provides the Plugin contract's [`promptforge_plugin::TOKIO_RUNTIME`] /// service. It has no prelude and needs no per-run service. -pub const PACKAGE: Package = Package::new("promptforge/mcp", construct); +pub const PACKAGE: Package = + Package::new("promptforge/mcp", construct).description("Tools from one MCP server"); /// The key `construct` reads the Host's runtime through. const RUNTIME: ServiceKey = ServiceKey::new(TOKIO_RUNTIME); diff --git a/crates/plugin-user-input/src/lib.rs b/crates/plugin-user-input/src/lib.rs index e26a10992..e762ee437 100644 --- a/crates/plugin-user-input/src/lib.rs +++ b/crates/plugin-user-input/src/lib.rs @@ -82,6 +82,7 @@ use promptforge_plugin::{Package, ServiceId, ServiceKey, ToolError}; /// installs it as `user-input`. It has a prelude and needs the run's /// [`INPUT_BROKER`]. pub const PACKAGE: Package = Package::new("promptforge/user-input", ask::construct) + .description("Ask the person running the prompt") .prelude(PRELUDE) .needs(NEEDS); diff --git a/crates/plugin-web/src/web.rs b/crates/plugin-web/src/web.rs index d7e2b4be0..940033b87 100644 --- a/crates/plugin-web/src/web.rs +++ b/crates/plugin-web/src/web.rs @@ -26,7 +26,8 @@ use crate::search::WebSearch; /// `construct` reads [`SEARCH_PROVIDER`] and the Plugin contract's /// [`promptforge_plugin::TOKIO_RUNTIME`] service from the Host-wide /// services and takes no configuration but `null` or `{}`. -pub const PACKAGE: Package = Package::new("promptforge/web", construct); +pub const PACKAGE: Package = + Package::new("promptforge/web", construct).description("Search the web and fetch pages"); /// The service key for the Host's search provider. /// diff --git a/crates/promptforge-plugin/src/plugin.rs b/crates/promptforge-plugin/src/plugin.rs index 0b6da97f5..aea4866b7 100644 --- a/crates/promptforge-plugin/src/plugin.rs +++ b/crates/promptforge-plugin/src/plugin.rs @@ -101,11 +101,13 @@ pub trait Plugin: Send + Sync { /// /// It is fixed data the Host reads before anything is built. Passing it to /// the Harness's install is also what links the Plugin crate into the -/// Host. A Plugin crate builds it with [`Package::new`], adding -/// [`Package::prelude`] and [`Package::needs`] only where it has them: +/// Host. A Plugin crate builds it with [`Package::new`], adding a +/// [`Package::description`], and [`Package::prelude`] and +/// [`Package::needs`] only where it has them: /// /// ```text /// pub const PACKAGE: Package = Package::new("acme/greeter", construct) +/// .description("Greet the person running the prompt") /// .prelude(PRELUDE) /// .needs(NEEDS); /// ``` @@ -139,11 +141,14 @@ pub struct Package { config: serde_json::Value, services: &HostServices, ) -> Result, ToolError>, + /// One line saying what the Plugin offers, for a Host listing the + /// packages it can install; empty unless the Plugin crate sets it. + pub description: &'static str, } impl Package { - /// Builds the label for the package `name`, with no prelude and no - /// needs. + /// Builds the label for the package `name`, with an empty description, + /// no prelude, and no needs. #[expect( clippy::type_complexity, reason = "spelled out so a Plugin author reads the signature construct implements in one place" @@ -162,6 +167,16 @@ impl Package { prelude: None, needs: &[], construct, + description: "", + } + } + + /// Sets the one line saying what the Plugin offers. + #[must_use] + pub const fn description(self, description: &'static str) -> Package { + Package { + description, + ..self } } @@ -182,10 +197,12 @@ impl Package { } impl fmt::Debug for Package { - /// Shows the name, whether a prelude exists, and the needs. + /// Shows the name, the description, whether a prelude exists, and the + /// needs. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Package") .field("name", &self.name) + .field("description", &self.description) .field("prelude", &self.prelude.is_some()) .field("needs", &self.needs) .finish_non_exhaustive() diff --git a/vibe/2026-10-11-1-package-registry.md b/vibe/2026-10-11-1-package-registry.md new file mode 100644 index 000000000..4e08efa2d --- /dev/null +++ b/vibe/2026-10-11-1-package-registry.md @@ -0,0 +1,514 @@ +--- +name: package registry and plugin kinds +overview: Add a string-keyed PackageRegistry to the Harness so a Host lists its Plugins and installs them by package name, and name three Plugin kinds, with promptforge/subagents built in as the one Engine-native package under a reserved name. tasks stays part of the language, in every prompt, and is not a Plugin. The native reuses the existing Package type and install path as a Plugin with no tools, so the run snapshot doesn't change. What subagents does doesn't change here. +todos: + - id: descriptions + content: "Step 1: Package::description field and builder in promptforge-plugin; descriptions on the web, user-input, and mcp packages" + status: pending + - id: registry + content: "Step 2: new runner packages.rs: PackageRegistry (new and Default hold the native, add, list yielding (&Package, PluginKind)), PluginKind { Native, Linked }, and the private native Package constant for promptforge/subagents with its tool-less Plugin; two guards: a doc comment on the stub saying built-in behavior lives in the Engine and the stub never gains tools, and a packages-tests.rs test that every native builds a Plugin with no tools, no prelude, and no needs" + status: pending + - id: install + content: "Step 2: HostContext::new(packages, services) takes the registry by value; install takes a package name; InstallError gains DuplicatePackage, UnknownPackage, and ReservedName, and InvalidPackage moves from install to add" + status: pending + - id: callers + content: "Step 2: runner and harness facade re-export PackageRegistry and PluginKind; Workshop with_plugins installs web, user-input, subagents, and each MCP server by name; run-prompt example; every test call site through a per-crate registry helper" + status: pending + - id: docs + content: "Step 2: AGENTS.md Plugin definition with three kinds, subagents the one native; host.rs, harness lib.rs, and Package docs say Hosts add packages to a registry and install by name" + status: pending + - id: verify + content: Both nextest runs, both clippy runs with warnings denied, fmt, docs and facade docs builds, xtask api --check, build-xtask tests, and the docs-claims test + status: pending + - id: close + content: "Step 3: mark the vibe/ copy completed and commit Close plan: package-registry with the Plan trailer; nothing is pushed" + status: pending +isProject: false +--- + +# Install Plugins by name from a package registry, with three Plugin kinds + +### Context for a fresh reader + +- Repository: `c:\Users\Vinnie\cursor\promptforge` (git). Every path in this plan is relative to it. +- Baseline: commit `e7c376a04` ("Tidy up vibe/ plans"). Line numbers are from that commit, so search by name if they have moved. +- Terms defined in `AGENTS.md`: + - The Engine runs prompts and performs no I/O itself. + - The Harness steps the Engine and performs its effects. + - The Host is the application (Workshop, or a command-line Host) that gives the Harness its model broker, timer, recorder, and installed Plugins. + - A Plugin is a package of tools installed under a local name such as `web`, whose tools sit under that name (`web/fetch`). +- Terms this plan adds: + - *package*: a Plugin crate's `Package` constant (a `vendor/name` name, prelude, needs, and `construct`), such as `plugin_web::PACKAGE`, named `promptforge/web`. + - *linked Plugin*: a `plugin-*` crate's package, which the Host adds to the registry. + - *native Plugin*, or *the native*: `promptforge/subagents`, built into every registry. Its future behavior and tools live in the Engine. + - *the stub*: the native's Plugin object, which offers no tools and whose `call` is unreachable. +- Why the stub's `call` is unreachable: today a tool call travels in three hops. + 1. The Engine's scheduler issues an `Effect::ToolCall` (`crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs`, lines 221-234). + 2. The Harness's effect loop performs it through `HostRunContext::call` (`crates/harness-internal/runner/src/effect_loop.rs`, lines 340-353). + 3. `HostRunContext::call` reaches a Plugin's `call` only for a tool id in that Plugin's own list (`crates/harness-internal/runner/src/host-run.rs`, lines 156-195). + + Calls the Engine answers itself never become effects, so they never reach the Harness. That includes today's task built-ins and the future subagent tools. +- Already in place and unchanged here: + - the Lua `plugins` table and tool objects from `vibe/2026-10-09-5-author-api-reshape.md`: `plugins.required()`, `plugins.extras()`, `plugins.get(name)` with its `.tools`, `tools.required()`, `tools.extras()`, and `tools.offer` + - the `tasks` Lua table, which the Engine installs in every prompt +- Plan record and commits: + - When execution starts, copy this plan to `vibe/` as `2026-10-
--package-registry.md`, using the next free number for that day. The latest plan record is `vibe/2026-10-10-3-prompt-reads-debt-removal.md`. + - Mark each step `[completed]` in that copy as it lands. + - Write each commit message by `c:\Users\Vinnie\cursor\.cursor\rules\commit.mdc`. Recent commits carry a one-line summary, a body, `Design:` lines, and a `Plan: vibe/` trailer. + - Close with a commit `Close plan: package-registry` carrying the same trailer. + - Nothing is pushed. + + + +## Product Requirements + +Hosts install Plugins by passing a compile-time `Package` constant, so nothing can list the available Plugins, install one by name, or pass it settings from a screen. This plan adds a `PackageRegistry` in the Harness that a Host builds once at startup, lists, and installs from by package-name string. It also names three Plugin kinds and builds `promptforge/subagents` into every registry as the one Engine-native package, under a reserved name. `tasks` stays part of the language, in every prompt as it is today, and is not a Plugin. What `subagents` does is unchanged here. + +- Problem and users: + - Hosts (Workshop, a command-line Host) call `HostContext::install(package: Package, ...)` with a constant such as `plugin_web::PACKAGE`. No list exists for a user interface, and nothing can install by name. + - `subagents`, the model-facing feature, needs to become a Plugin a Host installs or leaves out, under a name no other Plugin can take, so a Host can keep its tools out of a run. + - `tasks` does not. It is a language feature, and a prompt that doesn't want it just doesn't use it. +- Goals: + - A Host builds one registry, lists it (each package's name, description, needs, and kind), and installs by package name with a local name and a config value. + - Three kinds: linked (built now), library (named only), and Engine-native (`subagents`). + - A prompt declaring `subagents` passes preparation when the Host installed it, and is refused with `missing required Plugin: subagents` when not. +- Non-goals: + - The subagent tools and everything `subagents` does. + - Any change to the `tasks` table, which stays in every prompt. + - Library Plugins, per-package config schemas, and the Workshop checkbox screen. +- Success criteria: + - Workshop installs every Plugin by name, including the native. + - `registry.list()` yields the native with `PluginKind::Native` and each added package with `PluginKind::Linked`, each with its description. + - Each mistake fails with a named error: + - `install` with an unknown name: `UnknownPackage`. This includes `promptforge/tasks`, since `tasks` is not a package. + - `install` misusing the native's name: `ReservedName` + - `add` with a name already present, the native's name included: `DuplicatePackage` + - `add` with a malformed name: `InvalidPackage` + - Nothing in the run snapshot (`host-run.rs`) changes, and nothing about `tasks` changes. + - Every exit gate passes. +- Constraints: + - Engine crates don't depend on `promptforge-plugin`, and Harness crates name only `promptforge`, `promptforge-plugin`, and `workspace-hack` outside their family. So the registry lives in the runner, and the Engine learns nothing about the native in this plan. + - Every `.rs` file stays under 500 lines, and source directories stay flat. `host.rs` already has its two siblings (`host-run.rs`, `host-tests.rs` at 388 lines), so the registry is its own module. + - Plugin docs say installed, snapshotted, or declared, never activated. + - `unwrap_used` and `expect_used` are denied outside tests, and `missing_debug_implementations` warns, which is fatal under the gate. +- Open questions: none. + +## Functional Specification + +```rust +let mut packages = PackageRegistry::new(); // promptforge/subagents is built in +packages.add(plugin_web::PACKAGE)?; // linked packages, typed +for (package, kind) in packages.list() { /* package.name, package.description, package.needs */ } +let mut host = HostContext::new(packages, services); +host.install("promptforge/web", None, Value::Null)?; +host.install("promptforge/subagents", None, Value::Null)?; +``` + +- Registry: + - `new()` (and `Default`) holds `promptforge/subagents` as `PluginKind::Native`. + - `add` refuses a name that isn't a valid `vendor/name` pair (`InvalidPackage`) and a name already present (`DuplicatePackage`). The native is present from the start, so `add` also refuses its name. + - `list()` yields `(&Package, PluginKind)` in package-name order. +- Install: + - Looks the name up in the registry, and fails with `UnknownPackage` when it isn't there. + - With `name: None`, a package takes its name's second segment, as today. For the native, that's `subagents`. + - `ReservedName` covers the native package under any other local name, and a linked package whose local name, given or default, is `subagents`. + - Today's `NameTaken` twin rule is unchanged, so installing the native twice is `NameTaken`. One package may be installed many times (MCP once per server). +- The native's `construct` builds a Plugin with no tools, no prelude, and no needs. So install, the readiness wait, and the run snapshot treat it like any other Plugin: + - A declared `subagents` that is installed is usable and meets its requirement. + - A declared `subagents` that isn't installed is reported missing. + - Neither adds a catalog tool or a prelude. +- `tasks` is not in the registry, and it isn't a reserved Plugin name. The `tasks` Lua table stays in every prompt, and its global name stays in `RESERVED_NAMES`, as today. + + + + + +## Technical Design + +The registry and installation live in the Harness runner. The Engine sees only what it does today: the catalog, the preludes, and the requirements report. The native is an installed Plugin whose object, the stub, has an empty tool list. So nothing changes in `Installed`, `wait_ready`, `begin_run`, or anywhere in `host-run.rs`. One incidental change sits outside the registry and install: `HostContext`'s hand-written `Debug` must show the new field, because Clippy's pedantic `missing_fields_in_debug` lint, which the workspace denies, rejects a `debug_struct(...).finish()` that skips a field. + +```mermaid +flowchart LR + host[Host] -->|"add packages"| registry[PackageRegistry] + registry --> hostCtx[HostContext] + host -->|"install by name"| hostCtx + hostCtx -->|"per run"| snapshot[HostRunContext] + snapshot -->|"tools, preludes"| engine[Engine] +``` + +```rust +// crates/promptforge-plugin/src/plugin.rs +pub struct Package { /* name, prelude, needs, construct */ pub description: &'static str } // "" from new() +impl Package { pub const fn description(self, description: &'static str) -> Package; } + +// crates/harness-internal/runner/src/packages.rs (new), re-exported by harness::plugin +#[derive(Debug)] +pub struct PackageRegistry { + // keyed by package name, so list() runs in name order; the value is the package, its kind, + // and its default local name, parsed once by add (or by the native seeding) + entries: BTreeMap<&'static str, (Package, PluginKind, PluginId)>, +} +impl Default for PackageRegistry { fn default() -> PackageRegistry; } // holds the native +impl PackageRegistry { + pub fn new() -> PackageRegistry; // PackageRegistry::default() + pub fn add(&mut self, package: Package) -> Result<(), InstallError>; // InvalidPackage, DuplicatePackage + pub fn list(&self) -> impl Iterator; // name order + pub(crate) fn entry(&self, package: &str) -> Option<(Package, PluginKind, &PluginId)>; // install's lookup + pub(crate) fn reserves(&self, name: &PluginId) -> bool; // true when a native entry's default local name is `name` +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum PluginKind { + Native, // built into every registry; what it does lives in the Engine + Linked, // added by the Host with PackageRegistry::add +} // Library { path } lands with dynamic loading + +// private to packages.rs: the native, whose behavior lives in the Engine +const NATIVE: [Package; 1] = [Package::new("promptforge/subagents", native) + .description("Lets the model start, wait for, and cancel subagents")]; +fn native(name: &PluginId, config: Value, services: &HostServices) -> Result, ToolError>; // Ok(Arc::new(Native)) +struct Native; // tools() is empty; call() returns an error; never gains tools + +// crates/harness-internal/runner/src/host.rs +pub struct HostContext { packages: PackageRegistry, services: HostServices, installed: Vec } +impl HostContext { + pub fn new(packages: PackageRegistry, services: HostServices) -> HostContext; // was new(services) + pub fn install(&mut self, package: &str, name: Option, config: Value) -> Result; +} +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] // unchanged derives +#[non_exhaustive] +pub enum InstallError { + #[error("Plugin package name `{package}` is not a vendor/name pair")] // unchanged text; now returned by add + InvalidPackage { package: &'static str }, + #[error("Plugin package `{package}` is already in the registry")] // add + DuplicatePackage { package: &'static str }, + #[error("Plugin package `{package}` is not in the registry; add it with PackageRegistry::add before installing it")] // install + UnknownPackage { package: String }, + #[error("Plugin package `{package}` cannot be installed as {name}: a native Plugin keeps its own name, and no other package may take it")] // install + ReservedName { package: &'static str, name: PluginId }, + NameTaken { name: PluginId, existing: PluginId }, // unchanged, text included +} +``` + +`entry` returns a tuple rather than a private struct, so the `private_interfaces` lint has nothing to flag. `Package` is `Copy`, so `install` takes it by value. The error texts are suggestions in the house style: they name what was asked for and what is wrong. + +`NATIVE` stays an array of one, so the registry seeds it and the guard test loops over it the same way they would for a second native. + +Paths are relative to the `promptforge` repository. Line numbers are as of `e7c376a04`. + +- Step 1, package descriptions: + - [crates/promptforge-plugin/src/plugin.rs](crates/promptforge-plugin/src/plugin.rs): + - add the `description` field on `Package` (lines 114-142), set to `""` in `new` (lines 152-166) + - add the `description` builder beside `prelude` and `needs` + - show it in `Debug`, and add `.description(...)` to the `text` example in the doc comment (lines 107-111) + - Descriptions on `plugin_web::PACKAGE` ([web.rs](crates/plugin-web/src/web.rs) line 29), `plugin_user_input::PACKAGE` ([lib.rs](crates/plugin-user-input/src/lib.rs) line 84), and `plugin_mcp::PACKAGE` ([lib.rs](crates/plugin-mcp/src/lib.rs) line 88). Suggested text: "Search the web and fetch pages", "Ask the person running the prompt", and "Tools from one MCP server". +- Step 2, the registry and string installs: + - New `crates/harness-internal/runner/src/packages.rs` holds: + - the registry, `PluginKind`, and the private `NATIVE` constant, each public item with a doc comment (`missing_docs`) + - `native`, which builds `Arc::new(Native)` and carries the `#[expect(clippy::needless_pass_by_value, clippy::unnecessary_wraps, reason = ...)]` that `host-tests.rs`'s `listed` (lines 52-57) carries for the same fixed `construct` signature + - `Native`, whose `tools()` returns an empty `Vec`. Its `call` returns `ToolError::message`, naming `cx.tool()` and saying the Engine answers a native Plugin's calls. + - `package_name` (the `vendor/name` check, also returning the default local name), which moves here from `host.rs` (lines 229-235) and becomes `pub(crate)`. `normalize` (lines 237-245) stays in `host.rs` for the `NameTaken` check. + - a test sibling, `packages-tests.rs`, wired with `#[cfg(test)] #[path = "packages-tests.rs"] mod tests;` + - The constraint comment sits on `NATIVE`, and `Native` and `native` point to it. It is the one comment this plan adds, and it states what the code can't show: + - A built-in's behavior and tools live in the Engine. + - This Plugin exists only so install, readiness, and the run snapshot treat a built-in like any other Plugin. + - It must never offer tools, carry a prelude, or need services. + - Its `call` is unreachable, because the run snapshot routes a call only to a tool in a Plugin's own list. + - Registry behavior: + - `Default` seeds each `NATIVE` entry through `package_name`, inserting it as `PluginKind::Native` with its default local name. `expect` is denied, so a name that failed to parse would be skipped, not unwrapped. The integration test that lists `promptforge/subagents` pins that it parses. + - `new` returns `PackageRegistry::default()`. Clippy's `new_without_default` requires the `Default` impl. + - `add` runs three checks in order: + 1. Parse the name with `package_name`, or fail with `InvalidPackage { package: package.name }`. + 2. Refuse a name already present with `DuplicatePackage { package: package.name }`. This also covers the native's name. + 3. Insert the package as `PluginKind::Linked` with its default local name. + - `list` is `self.entries.values().map(|(package, kind, _)| (package, *kind))`. Edition 2024 captures the borrow of `self`, so the return type needs no `+ '_`. + - `entry` is a lookup in `entries`, and `reserves` is true when any `PluginKind::Native` entry's default local name equals `name`. `PluginId` is lowercase-only, so plain equality is exact. + - [host.rs](crates/harness-internal/runner/src/host.rs): + - `HostContext` (lines 42-45) gains `packages: PackageRegistry`, and `new` (lines 55-67) takes it. + - `install` (lines 69-119) runs these steps in order: + 1. Look the name up with `self.packages.entry(package)`, or fail with `UnknownPackage { package: package.to_owned() }`. + 2. Take the local name as `name.unwrap_or_else(|| default.clone())`. + 3. A `PluginKind::Native` entry whose local name differs from its default fails with `ReservedName { package: package.name, name }`. + 4. A `PluginKind::Linked` entry whose local name `self.packages.reserves(...)` fails the same way. + 5. Today's `NameTaken` twin check, `construct` call, and push follow, unchanged. So installing the native twice is `NameTaken`. + + Today's `package_name(...).ok_or(InvalidPackage)` at the top of `install` goes, because `add` already validated the name. + - `Debug` (lines 184-197) adds `.field("packages", &self.packages)` before `.finish()`. + - `InstallError` (lines 199-227) gains the three variants above. Its doc comment becomes "A mistake `PackageRegistry::add` or `HostContext::install` refuses outright, so the Host's author sees it at startup", with the construct-failure paragraph kept. Each new variant documents its fields. + - The module doc (lines 1-15) says the context is built once at startup over a [`PackageRegistry`], and that a Host installs a package by the name the registry holds, under a local name. The rest of the doc (construct runs once, the readiness wait, the per-run snapshot) stays. The `new` and `install` docs say the same: `new` takes the registry, and `install`'s `# Errors` lists `UnknownPackage`, `ReservedName`, and `NameTaken`. + - [runner lib.rs](crates/harness-internal/runner/src/lib.rs): + - Declare `mod packages;`, and re-export `PackageRegistry` and `PluginKind` beside the line-42 `pub use host::{HostContext, InstallError};`. + - Add a bullet to the crate doc's `## Invariants` (lines 13-25), which Harness crates must carry: a Host installs a Plugin only by a package name its `PackageRegistry` holds. The registry holds the native `promptforge/subagents` from the start and refuses a duplicate name, and install refuses the native's local name to every other package. So no other Plugin can take `subagents`. + - [crates/harness/src/lib.rs](crates/harness/src/lib.rs): + - Re-export `PackageRegistry` and `PluginKind` in `pub mod plugin` (lines 25-34). + - The crate doc's first sentence (lines 3-4) becomes: a Host adds its Plugin packages to a [`plugin::PackageRegistry`], builds one [`plugin::HostContext`] over it, and installs Plugins by package name. + - The `plugin` module doc (lines 26-28) adds "the registry a Host lists and installs packages from". + - [crates/workshop/server/src/agents.rs](crates/workshop/server/src/agents.rs), `with_plugins` (lines 140-179): + - Build `PackageRegistry::new()`, add the web, user-input, and mcp constants (warning on error), and build `HostContext::new(packages, services)`. + - Install `plugin_web::PACKAGE.name`, `plugin_user_input::PACKAGE.name`, `"promptforge/subagents"`, and `plugin_mcp::PACKAGE.name` per server. + - Its signature stays, so the callers in `agents/tests.rs` don't change. + - Name the local `packages`, since `registry` there is the gateway `workshop_registry::Registry`. + - The install order stays web, then user-input (whose returned name still builds the ask tool id), then the native, then each MCP server. A refused native install is logged with `tracing::warn!`, like the others, and the server still starts. + - The doc comment (lines 140-148) names the native among the installed Plugins, and so does the module doc (lines 12-14): "the server's `HostContext` with web, user-input, the native subagents, and the MCP servers of the configured `mcp.json` installed". + - [crates/harness/examples/run-prompt.rs](crates/harness/examples/run-prompt.rs) line 72: `HostContext::new(PackageRegistry::new(), HostServices::new())`. + - Test call sites. A test now adds each package to a registry before it builds the context, then installs by `package.name`. Paths are under `crates/`, with `runner` meaning `harness-internal/runner`. + - `runner/src/host-tests.rs`: `host()` (line 105) becomes `host(packages: &[Package]) -> HostContext`, and a sibling `host_with(services, packages)` serves the tests that build with Host-wide services, such as `provided` around line 213. Each test passes the packages it installs. + - `runner/tests/it/support.rs` (the shared helpers) gains `pub(crate) fn registry(packages: &[Package]) -> PackageRegistry`. It adds each package to `PackageRegistry::new()`, with `unwrap`, which tests allow. Every runner integration test builds its context as `HostContext::new(registry(&[...]), services)`. + - `runner/tests/it/prepare.rs`: + - `bare()` (line 90) becomes `HostContext::new(registry(&[]), HostServices::new())`. Its seven callers in `prepare.rs`, `prepare-files.rs`, and `prepare-ready.rs` don't change. + - `installing(package)` (line 95) builds over `registry(&[package])` and installs `package.name`. + - `runner/tests/it/prepare-prelude.rs` (line 81) builds over `registry(&[...])` of the echo packages it installs. + - `runner/tests/it/prepare-ready.rs` (line 146) builds `HostContext::new(registry(&[SLOW]), wide)`, keeping its Host-wide services, then installs `SLOW.name` under several local names (line 148). + - `runner/tests/it/scripted.rs`: + - `host_holding` (line 260) builds over `registry(&[HOLD, ASKER])`. That way `with_asker` (line 270), which installs `ASKER.name` into a context someone else built, works on every `hold_host` and `surviving_hold_host` result used by `harness-stop.rs`. + - `host_holding` installs `HOLD.name` (line 264). + - `runner/tests/it/harness-stop-timing.rs` (line 67) builds `HostContext::new(registry(&[ASKER]), HostServices::new())` before `with_asker`. + - `runner/tests/it/{harness,harness-errors}.rs` change only where they call `HostContext::new`: pass `registry(&[...])` of what they install, or `PackageRegistry::new()`. + - `harness/tests/suite/{host,launch,broker}.rs` only call `HostContext::new`: pass `PackageRegistry::new()`. `harness/tests/suite/asker.rs` adds `ASKER`, then installs `ASKER.name` (line 44). + - `workshop/agents/tests/it/support.rs` (line 63) and `workshop/agents/src/input-tests.rs` (line 270): add `plugin_user_input::PACKAGE`, then install `plugin_user_input::PACKAGE.name`. + - Docs: + - [AGENTS.md](AGENTS.md) line 12, the whole Plugin definition, becomes the following. It keeps today's last three sentences word for word, and it must pass the docs-claims scan: Plugin capitalized, and installed, snapshotted, or declared, never activated. + + > **Plugin**: a package of tools a Host adds to its package registry and installs into a `HostContext` by package name, such as `promptforge/web`, under a local name the Host chooses, such as `web`. Every tool it offers sits under that name (`web/fetch`). A Plugin is linked (a `plugin-*` crate implementing `promptforge-plugin`), library (loaded from a dynamic library; not built yet), or Engine-native (`promptforge/subagents`, implemented in the Engine under the fixed name `subagents`, which no other Plugin may take). Linked and library Plugins use only the `promptforge-plugin` API. Every run receives every usable Plugin's tools: those of each installed Plugin that built and whose per-run services the run has. Declaring a Plugin under `plugins:` in a prompt's frontmatter installs its Lua prelude and makes it required, and the prompt's Lua decides what the model sees. A Plugin has no other name: never capability, pack, or addon. + - The `Package` doc in [crates/promptforge-plugin/src/plugin.rs](crates/promptforge-plugin/src/plugin.rs) (lines 102-103, "Passing it to the Harness's install is also what links the Plugin crate into the Host") becomes "Adding it to the Harness's package registry is also what links the Plugin crate into the Host." The field docs for `name` and `construct` stay accurate. +- Deferred to the `tasks` and `subagents` overhaul, which adds the subagent tools: + - `subagents` is model-facing, so it behaves like a linked Plugin. When the Host installs it, its tools are in every run: in `tools.required()` when the prompt declares `subagents`, in `tools.extras()` when it doesn't, and the prompt's Lua offers them with `tools.offer`. When the Host doesn't install it, no run has them. + - So each run must tell the Engine which natives the Host installed. How it does that is the overhaul's decision. The Engine still answers the calls in its scheduler, so they never become effects. + - The overhaul also removes `tools.allow_tasks` and today's four model built-ins (`task`, `task_cancel`, `task_status`, `await_tasks`), so this plan doesn't gate them on the install. + + + + + +## Testing Plan + +- Unit, in the new `crates/harness-internal/runner/src/packages-tests.rs`: `natives_offer_no_tools_prelude_or_needs` loops over every entry in `NATIVE`. Each one's `construct`, called with the default local name `super::package_name` returns for it (`unwrap` is allowed in tests), `Value::Null`, and `HostServices::new()`, builds a Plugin whose `tools()` is empty, and the entry has no `prelude` and no `needs`. It loops over the constant, not one name, so a native added later is covered too. This is the guard that fails if anyone gives the stub tools. +- Integration, in a new `crates/harness-internal/runner/tests/it/packages.rs` with `mod packages;` in `main.rs` (beside `mod prepare;`). Everything it tests is public, and `host-tests.rs` (388 lines) has little room. Linked test packages come from `prepare.rs`'s existing `pub(crate) const fn echo_package(name, prelude)`, for example `echo_package("acme/web", None)`. A package with needs is built with `Package::new(...).needs(...)`, as `prepare-host-services.rs` does. The test names below are suggestions in the house style. + - Registry: + - `the_registry_lists_the_native_and_each_added_package_in_name_order`: `PackageRegistry::new().list()` yields exactly `promptforge/subagents` with `PluginKind::Native` and a non-empty description. After `add(echo_package("acme/web", None))`, it yields `acme/web` (`PluginKind::Linked`) before `promptforge/subagents`. A package added with needs lists those needs. + - `add_refuses_a_duplicate_package_including_the_native`: a second `acme/web` and a package named `promptforge/subagents` are both `DuplicatePackage`. + - `add_refuses_a_package_name_that_is_not_a_vendor_name_pair`: the six names of today's install test at `host-tests.rs` lines 142-158 (`web`, `acme/web/extra`, `/web`, `acme/`, `Acme/web`, `acme/Web`) are each `InvalidPackage`. That install test is deleted, because `install` no longer parses names. + - Install: + - `install_refuses_a_package_the_registry_does_not_hold`: `acme/missing` is `UnknownPackage`, and so is `promptforge/tasks`, which pins that `tasks` is not a package. + - `no_other_package_can_take_the_native_name`: `acme/subagents` with no name, and `acme/web` named `subagents`, are `ReservedName`, and so is `promptforge/subagents` named `other`. The test also checks the `ReservedName` message text once. + - `the_native_installs_once_under_its_own_name`: `promptforge/subagents` with no name returns `subagents`, and a second install is `NameTaken`. +- Unit, in `host-tests.rs`, using its existing `prompt()` and `catalog_ids` helpers and `begin_run`: + - `a_declared_native_is_met_when_installed_and_missing_when_not`: with `promptforge/subagents` installed, a prompt with `plugins:\n - subagents\n` reports no missing requirement. Without that install, `requirements.missing_required` holds `subagents`. + - `an_installed_native_adds_no_catalog_tool_and_no_prelude`: `catalog_ids` is unchanged by installing the native, and `run.preludes(&prompt)` is empty for a prompt declaring `subagents`. + + The existing tests switch to `host(packages)` and `host_with(services, packages)` as described under Step 2. +- Workshop, in `crates/workshop/server/src/agents/tests.rs`: `a_prompt_declaring_subagents_prepares_on_the_servers_installed_plugins` mirrors `a_prompt_declaring_web_prepares_on_the_servers_installed_plugins` (lines 127-137). It builds `host_context(&Registry::new(), None)`, then expects `run_prompt(&Arc::new(host), requiring("subagents"))` to return `Some(RunOutcome::Completed { final_text: "browsed".to_owned() })`. `requiring` (line 37) declares the Plugin and returns `'browsed'` without calling a tool, so it works for a Plugin with no tools. This also pins the literal native name in `with_plugins`. +- Regression: every existing install, snapshot, and preparation test passes through the registry helper, and every test that uses `tasks` passes unchanged. +- Exit criteria: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` + - both clippy runs with `CARGO_BUILD_WARNINGS=deny` + - `cargo fmt --all --check` + - `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` and `cargo doc -p promptforge --no-deps`, both with `RUSTDOCFLAGS="-D warnings"` + - `cargo +nightly-2026-09-05 xtask api --check`, which should pass unchanged, since the `promptforge` facade doesn't change + - `cargo test -p build-xtask` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + + + + + +## Decision Record + +- Decisions: + - The scope is the package registry and the Plugin kinds alone. User: "just the package registry and plugin kinds". + - `tasks` is part of the language, not a Plugin, and `subagents` is the one Engine-native Plugin. Rationale: `tasks` became a Plugin when it was both Lua-facing and model-facing. A Host has a reason to keep the model-facing tools out of a run, but no one has a reason to make the `tasks` table optional. User: "tasks should not even be a plugin anymore it should just be a language feature. a prompt that doesn't want tasks... just doesn't use it." and "There is a use-case for not having the model-facing version in tools.required(). But there is no use-case for making the `tasks` table itself optional. and I think it creates more problems than it solves." + - Making the Host's install of `subagents` control which runs get its tools waits for the `tasks` and `subagents` overhaul. Rationale: `subagents` is model-facing, so it must behave like a linked Plugin, not be switched on by the prompt's declaration. The overhaul removes `tools.allow_tasks` anyway. User: "We are going to overhaul `tasks` and `subagents` later anyway. so the tools.allow_tasks problem is going to just vanish" and "Fix it later." + - Hosts install by package-name string through a registry. User: "it has to be strings, because there needs to be a way for a user interface to show a list of available packages, and then let the user check or uncheck the ones that they want". + - Loading and installing are separate steps, every entry records its kind, and installing walks the Host's selection, not the registry. User: "How can it know the names up front?" and "how does it distinguish between the registry entries that came from the directory versus the ones that are built in?" + - The registry is built once at startup and reused. User: "the workshop will build the registry once, when it launches and it'll use it over and over again". + - There are three kinds, and only the native reaches Engine internals. User: "1 and 2 strictly follow the normalized plugin API and do not get special access to engine implementation, while 3 is the special one." + - The native installs like any package, by string, and its name is reserved. User: "it needs to be impossible for another [...] plugin to take the name". + - No separate crate for the native Plugin. User: "but we aren't creating a plugin-tasks crate are we?" + - The registry lives in the Harness runner, and the Engine never sees it. Rationale: it holds `Package` values from `promptforge-plugin`, which Engine crates may not depend on, and installing is already the Harness's job. + - The API is compressed to what this plan's behavior reads, reusing existing structures. User: "see if there are any structures that can be reused instead of creating new ones. Preserve the functionality, see if you can compress the API." +- Defaults chosen in planning, open to override: + - The native is a private `Package` constant in the runner, whose `construct` builds a Plugin with no tools. So install, readiness, and the snapshot treat it like any Plugin. Its behavior and tools will live in the Engine. + - `list()` yields `(&Package, PluginKind)`, reusing `Package` for the name, description, and needs. + - `add` and `install` share `InstallError`. `InvalidPackage` moves from install to add unchanged, and the native's name is a plain `DuplicatePackage`. + - `HostContext::new` takes the registry by value. + - The kind enum is `PluginKind`, matching the AGENTS.md wording and leaving `harness::vfs::Origin` as the facade's only `Origin`. + - `PluginKind` is `#[non_exhaustive]`, so `Library { path }` can land later. + - `tasks` is not a reserved Plugin name. A Plugin installed as `tasks` would be its own namespace (`tasks/...` tools, `plugins.get('tasks')`), and preludes already can't take the `tasks` global. Reserving it would be code for a confusion no Host has hit. + - Workshop installs the native by default: the linked packages by `PACKAGE.name`, and the native by its literal name, which its test pins. + - Workshop keeps no separate registry handle until a user interface reads one. +- Rejected alternatives: + - Keeping `tasks` as a native Plugin that the Host installs and a prompt declares. Reason: no use case for an optional `tasks` table, and gating it would add a declaration to about 50 test files and prompts for nothing. + - A closed `NativePlugin` enum, with `Installed::Native` and `RunPlugin::Native` arms across install, readiness, snapshot, catalog, preludes, requirements, and offering, plus a `promptforge` facade export and a `local_name()` with no infallible way to build a `PluginId`. Reason: special cases with no reader in this plan. How the Engine learns which natives are installed is decided by the `tasks` and `subagents` overhaul, which is the first reader. + - A `PackageEntry` type with four getters. Reason: `Package` already carries the name, description, and needs. + - A separate `RegistryError` with `InvalidPackage`, `DuplicatePackage`, and `NativePackage`. Reason: `InstallError` already holds `InvalidPackage`, and a seeded native name is just a duplicate. + - Folding the registry into `HostContext`. Reason: the registry is loaded once and outlives each selection a Host installs, and later folder loads add entries to it. + - `Arc` in `HostContext::new`. Reason: the context reads it only while installing. A `Clone` can come when a Host first builds a second selection. + - Keeping `install(package: Package)` beside a registry lookup. Reason: anyone can build a `Package` named `promptforge/subagents`. Resolving names only through the registry is what keeps the native's name unforgeable. + - A Host-side method or run option to switch `subagents` on; installing by constant, or by string for the native only; installing a library by file path; listing libraries from a manifest, lazy activation, or scanning them in a separate process. +- Risks: + - String installs move a typo from compile time to startup. Typed `add`, the Workshop test, and named errors limit it. + - The native is installable before it does anything, so a prompt declaring `subagents` prepares but gets nothing until the `subagents` behavior lands. + - Until the overhaul, leaving the native out only refuses prompts that declare `subagents`. A prompt that calls `tools.allow_tasks()` still offers the model the four task built-ins without the Host's say. The overhaul removes `allow_tasks`. + - The cost of making the native a tool-less Plugin is confined to future code changes, by a person or an LLM. At runtime it costs nothing: + - `Plugin::call` has one call site (`host-run.rs` line 193), reached only for a tool in the Plugin's list, so the stub's `call` never runs. + - Only the registry can hold the native's name, so it can't be forged. + - Run records hold no Plugin list. + - The overhead is one empty object. + + Two mistakes remain possible: + - New code that walks the installed Plugins treats the native as a Plugin with no tools without the compiler asking. That is usually right, and wrong only silently. + - Someone hangs native behavior on the stub in the Harness by giving it tools and a real `call`. This is the worst plausible damage: it breaks the rule that the native is implemented in the Engine, and it would send model calls out as outside-world requests, which breaks replay, because a recorded answer can't stand in for a subagent the Engine must start. + + The doc comment on `Native` warns the writer, and `natives_offer_no_tools_prelude_or_needs` fails the change. User: "the only cost is if a developer makes a mistake, and that developer could be an LLM." +- Deferred: library Plugins, per-package config schemas, the Workshop checkbox screen, and everything `subagents` does. + + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build --locked -p gateway`. + - The gateway is the workspace's only default member, so plain `cargo build` builds the same thing. + - The desktop app builds with `cargo workshop`, an alias for `run -p build-workshop --`. It builds the gateway, stages it as the Tauri sidecar, builds `workshop`, and removes the sidecar. + - The `gateway-config-ui` and `workshop-server` build scripts bundle their UIs with esbuild. They need Node.js 22 and one `npm ci --prefix crates/workshop` plus one `npm ci --prefix crates/gateway/config-ui/ui` per checkout. Both installs, Node.js, cargo-nextest, and the pinned nightly are present on this machine. +- Focused test command pattern: `cargo nextest run --locked -p --all-features `. Drop `--all-features` for `workshop`, `workshop-server`, and `workshop-server-api`, which CI never builds with all features. +- Component test command pattern: `cargo nextest run --locked -p --all-features`, with the same three exceptions. `cargo test -p build-xtask` runs the boundary and structural checks. +- Full-suite test command: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api`. The first run includes `build-xtask`. + - CI also runs `cargo nextest run --locked -p workshop-server --features headless`, which serves without the UI bundle. + - The wording and rulebook scan: `node --test crates/workshop/ui/test/docs-claims.mjs`. It needs no `npm ci` and runs from any directory. + - The UI suites: `npm test --workspaces --if-present` in `crates/workshop` and `npm test` in `crates/gateway/config-ui/ui`. +- Linter command: + - `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, both with `CARGO_BUILD_WARNINGS=deny` (in PowerShell, `$env:CARGO_BUILD_WARNINGS='deny'`). + - Never run a standalone `cargo check --workspace` beside them. The one exception is `cargo check -p gateway --no-default-features`, the headless build shape clippy does not cover. + - UI type checks: `npm run typecheck --workspaces --if-present` in `crates/workshop` and `npm run typecheck` in `crates/gateway/config-ui/ui`. + - Supply chain: `cargo deny check`, `cargo audit`, and `cargo hakari verify`. +- Formatter check command: `cargo fmt --all --check`, with `style_edition = "2024"`. The pre-commit hook runs it. +- Docs command: + - `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api`, `cargo doc -p promptforge --no-deps`, `cargo doc -p harness --no-deps`, `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items`, and `cargo doc --locked --no-deps -p workshop-server --document-private-items`, all with `RUSTDOCFLAGS="-D warnings"`. The facade docs build with default features on purpose. + - The facade surface check: `cargo +nightly-2026-09-05 xtask api --check` against `crates/promptforge/public-api.txt`, and `--bless` regenerates the listing. The nightly is pinned in `crates/build-xtask/src/api/toolchain.rs`. Its fixtures run with `cargo +nightly-2026-09-05 nextest run --locked -p build-xtask --run-ignored only`. + - The documentation site: `cargo xtask site`, which needs `PROMPTFORGE_DOCS` naming a `promptforge-docs` checkout and an mdBook binary (`MDBOOK`, default `mdbook`). +- Test placement and naming conventions: + - Unit tests sit in a sibling file wired with `#[cfg(test)] #[path = "-tests.rs"] mod tests;`. A module whose tests reach three files keeps them in a `tests/` subdirectory beside it, one file per topic, as in `crates/promptforge-internal/engine/src/execute/tests/`. A few `build-*` crates, `gateway-api-types`, and `gateway-api-discovery` still hold inline `mod tests { }` blocks. + - Integration tests compile as one binary per crate: from `tests/it/main.rs` in most crates, and from `tests/suite/main.rs` in `harness` and `promptforge`. Each topic is one module file, and shared helpers go in `support.rs` or a `support/` directory. + - A few crates keep standalone test files directly in `tests/`: `gateway-stt-engine`, `gateway-stt-backend-whisper`, `gateway-cloud-providers`, and `build-workshop`. + - Test names are snake_case sentences stating the behavior, such as `a_call_without_the_per_run_service_it_needs_fails`. + - Doctests are banned. `build-xtask`'s `no_doctests` check fails any doc comment holding a code block rustdoc would compile, so examples use `text`, `json`, or `toml` fences. + - Nightly-only `build-xtask` fixtures and the native whisper tests are `#[ignore]`d. `.config/nextest.toml` puts the STT crates in a `heavy` test group. +- Directory map: + - `crates/`: all 52 crates. The top level holds the public and shared layer: the `promptforge` facade, `promptforge-plugin`, `harness`, `harness-gateway-client`, `gateway-api-types`, `gateway-api-discovery`, the `plugin-*` crates (`plugin-web`, `plugin-user-input`, `plugin-mcp`), `shared-error-source`, `shared-loopback`, the `build-*` tooling, and `workspace-hack`. + - `crates/promptforge-internal/`: the Engine's private crates `types`, `engine`, `lua`, `parser`, `vfs`, and `model-client`. + - `crates/harness-internal/runner`: the Harness's private per-run runner and effect loop. + - `crates/gateway/`: the gateway's private crates `app`, `cloud-providers`, `config`, `config-ui`, `local`, `logging`, `progress`, `protocol`, `routing`, and `web-search`, plus the `stt/` subsystem (`api`, `engine`, `backend-whisper`, `whisper-ffi`). + - `crates/workshop/`: Workshop's crates `desktop` (package `workshop`), `server`, `server-api`, `agents`, `gateway`, `menu`, `protocol`, `registry`, `run-log`, `status`, `support`, `user-state`, and `workspace`, plus the npm workspace packages `ui`, `look`, and `platform`. + - `crates/shared-ui`: a TypeScript and CSS package the gateway config UI consumes, not a Rust crate. + - `guide/`: mdBook `book.toml` files for the gateway, language, and Workshop books, the shared page chrome, and the landing page style. The chapters live in the separate `promptforge-docs` checkout. + - `prompts/`: example prompt files. `images/`: README art. + - `tools/`: Node scripts for staging the gateway sidecar and a live TTS check, each with a `.test.mjs`. + - `vibe/`: plan records, recent ones at the top level and older ones filed by month. + - `.github/workflows/`: `ci.yml` plus release, nightly, site, and native-library workflows. `ci-green` is the single required check. + - `.config/`: `nextest.toml` and `hakari.toml`. `.cargo/config.toml`: the `xtask` and `workshop` aliases and the Windows `rust-lld` linker with the static CRT. `.githooks/`: pre-commit (format) and pre-push (wording scan, headless check, clippy, deny), with `core.hooksPath` set to it. `.cursor/rules/`: the Workshop architecture and SPA rules. + - Untracked: `local/` (operator config, profiles, and fixtures), `cabinet/`, `target/`, and `target-msrv/`. +- Component boundaries, which `cargo test -p build-xtask` enforces across normal, dev, and build dependencies: + - The Engine (`promptforge*`) depends on no gateway, Workshop, Harness, or Plugin crate. Outside crates reach it only through `promptforge` and `promptforge-plugin`, the only two crates that may enter `promptforge-internal/`. + - The Harness (`harness*`) names only `promptforge`, `promptforge-plugin`, and `workspace-hack` outside its family, and no shared or `build-*` crate. The exception is `harness-gateway-client`, which depends on `plugin-web` to implement its `SearchProvider`. Outside crates reach the Harness only through `harness` and `harness-gateway-client`, and only `harness` may enter `harness-internal/`. The chain runs `harness-gateway-client` on `harness`, `harness` on `harness-runner`, and `harness-runner` on `promptforge` and `promptforge-plugin`. + - Plugin crates depend only on `promptforge-plugin`, `shared-*` crates, `workspace-hack`, and outside libraries. No Engine, gateway, or Harness crate except `harness-gateway-client` names a Plugin crate. Today `workshop-server` and `workshop-agents` are the other dependents. + - The gateway (`gateway*`) depends on no Engine, Workshop, Harness, or Plugin crate. Outside crates reach it only through `gateway-api-types` and `gateway-api-discovery`. Inside the family, only `gateway-stt` is visible outside `gateway/stt/`. + - Workshop is a Host. It reaches the gateway only through that public pair, the Engine only through `promptforge` and `promptforge-plugin`, and the Harness only through `harness` and `harness-gateway-client`. Its tiers depend one way: the server on features (`agents`, `workspace`, `user-state`, `run-log`), features on services (`menu`, `gateway`, `status`), and services on vocabulary (`protocol`, `registry`, `support`). Same-tier crates meet only through `workshop-protocol` wire types and `workshop-registry` slots. The desktop app names only `workshop-server-api`, which names only `workshop-server`. + - `shared-*` crates depend on no product crate. `build-*` crates are tooling and may depend into any container. +- Conventions summary: + - Engine, Harness, Host, and Plugin are capitalized with one meaning each. Engine crates say "the caller" and never mention the Host. `docs-claims.mjs` enforces what a scan can see. + - Every crate's build script calls `build_ceiling::check()`, which fails any Rust file over 500 lines. Splits go into private child modules without widening visibility. + - Clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, and `allow_attributes` deny, so suppressions use `#[expect(..., reason = "...")]`. `missing_docs`, `missing_debug_implementations`, and `unreachable_pub` warn, which is fatal under the gate. `unsafe_code` is denied outside its owned boundaries, and `clippy.toml` bans process-global installers outside binary entry points. + - Every `workshop-*` `lib.rs` opens with a `//!` doc carrying a `## Invariants` marker naming its tier and allowed dependencies, and `lib.rs` holds only docs, attributes, `mod` declarations, and re-exports. + - Source directories stay flat until a group reaches three files. Below that, kebab siblings such as `foo-bar.rs` are wired with `#[path]`. + - Comments explain only non-obvious constraints, and every workaround cites its upstream issue URL. + - Error messages are written for a model to read: concise, naming what was required and what was found. + - Behavior changes ship with tests in the same change. A Cargo feature gates only a real build constraint, never product shape. + - JSON that reaches a recorder round-trips exactly, with sorted keys and finite numbers. + - Workspace dependencies are declared once in the root `Cargo.toml`, with a comment on every non-obvious pin. Every member inherits `workspace-hack`, which `cargo hakari` manages. + - Commits carry a one-line summary, a body, optional `Design:` lines, and a `Plan: vibe/` trailer. + + + + + +## Execution Instructions + + + +### Step 1: Describe each Plugin package [completed] + +- Component: `none` +- Work item: `descriptions`, detailed in the Technical Design's Step 1 list. Additive only: nothing reads `Package::description` yet, and the field is public, so it raises no dead-code warning. The `promptforge` facade does not list `Package`, so `crates/promptforge/public-api.txt` stays unchanged. +- `crates/promptforge-plugin/src/plugin.rs`: + - `Package` gains `pub description: &'static str`, which `Package::new` sets to `""`. + - `Package::description(self, description: &'static str) -> Package`, a `const fn` builder beside `prelude` and `needs`. + - `Package`'s `Debug` shows the new field, and the doc comment's `text` example adds `.description(...)`. +- Descriptions, using the Technical Design's suggested text: + - `plugin_web::PACKAGE` in `crates/plugin-web/src/web.rs` + - `plugin_user_input::PACKAGE` in `crates/plugin-user-input/src/lib.rs` + - `plugin_mcp::PACKAGE` in `crates/plugin-mcp/src/lib.rs` +- Tests: none new. The existing `promptforge-plugin` and Plugin crate tests pass. +- Verify: + - `cargo nextest run --locked -p promptforge-plugin -p plugin-web -p plugin-user-input -p plugin-mcp --all-features` + - the Project Survey's first clippy run, with `CARGO_BUILD_WARNINGS=deny` + - `cargo fmt --all --check` +- Commit: one `promptforge` commit, `Describe each Plugin package`. + + + + + +### Step 2: Install Plugins by name from a package registry + +- Component: `none` +- Work items: `registry`, `install`, `callers`, `docs`, and `verify`. They land together, because changing `HostContext::new` and `install` breaks every caller at once. The Technical Design's Step 2 list holds the details for each artifact below. +- New `crates/harness-internal/runner/src/packages.rs`: + - `pub struct PackageRegistry` over `BTreeMap<&'static str, (Package, PluginKind, PluginId)>`, with `Default` (seeds each `NATIVE` entry through `package_name`, skipping one that fails to parse because `expect` is denied), `new`, `add`, `list`, and the `pub(crate)` `entry` and `reserves` + - `#[non_exhaustive] pub enum PluginKind { Native, Linked }` + - the private `const NATIVE: [Package; 1]` for `promptforge/subagents`, carrying the plan's one constraint comment; the `native` construct function with its `#[expect(...)]`; and the private `struct Native`, whose `tools()` is empty and whose `call` returns `ToolError::message` + - `pub(crate) fn package_name`, moved from `host.rs` + - the `packages-tests.rs` sibling, wired with `#[cfg(test)] #[path = "packages-tests.rs"] mod tests;` +- `crates/harness-internal/runner/src/host.rs`: + - `HostContext` gains `packages: PackageRegistry`. `HostContext::new(packages, services)` takes it by value, and `install(&mut self, package: &str, name: Option, config: Value)` resolves through `self.packages.entry` in the Technical Design's five-step order, ending in today's unchanged `NameTaken` check, `construct` call, and push. + - `InstallError` gains `DuplicatePackage`, `UnknownPackage`, and `ReservedName`. `InvalidPackage` keeps its text and is now returned only by `add`. + - The hand-written `Debug` adds `.field("packages", &self.packages)`. `normalize` stays here. + - The module, `new`, `install`, and `InstallError` docs change as the Technical Design says. +- Re-exports and crate docs: + - `crates/harness-internal/runner/src/lib.rs`: `mod packages;`, `pub use packages::{PackageRegistry, PluginKind};` beside the `host` re-export, and the new `## Invariants` bullet + - `crates/harness/src/lib.rs`: `PackageRegistry` and `PluginKind` in `pub mod plugin`, the crate doc's first sentence, and the `plugin` module doc +- Hosts: + - `crates/workshop/server/src/agents.rs` `with_plugins`: builds a local `packages` registry from the web, user-input, and mcp constants, then installs web, user-input, `"promptforge/subagents"`, and each MCP server by name, in that order, logging each refusal with `tracing::warn!`. Its signature stays. Its doc comment and the module doc name the native. + - `crates/harness/examples/run-prompt.rs`: `HostContext::new(PackageRegistry::new(), HostServices::new())` +- Test call sites. Each builds its context over a registry of the packages it installs, then installs by `package.name`. Paths are under `crates/`: + - `harness-internal/runner/src/host-tests.rs`: `host(packages: &[Package])` and `host_with(services, packages)`, which every existing test switches to. Delete the install-time invalid-name test (lines 142-158); its six names move to the `add` test below. + - `harness-internal/runner/tests/it/support.rs`: `pub(crate) fn registry(packages: &[Package]) -> PackageRegistry` + - `harness-internal/runner/tests/it/prepare.rs`: `bare()` over `registry(&[])`, and `installing(package)` over `registry(&[package])`. The extras test near line 382 installs `echo_package("tests/extra", None)` into an `installing(TOOLS)` context, so it builds its own context over `registry(&[TOOLS, echo_package("tests/extra", None)])` and installs both by name. + - `harness-internal/runner/tests/it/prepare-prelude.rs` (line 81): replace `bare()` with a context over `registry(&[...])` of the `tests/first` and `tests/second` echo packages + - `harness-internal/runner/tests/it/prepare-ready.rs` (line 146): `HostContext::new(registry(&[SLOW]), wide)`, installing `SLOW.name` + - `harness-internal/runner/tests/it/scripted.rs`: `host_holding` over `registry(&[HOLD, ASKER])`, installing `HOLD.name`. `with_asker` installs `ASKER.name`. + - `harness-internal/runner/tests/it/harness-stop-timing.rs`: line 67 over `registry(&[ASKER])`, and line 141 over `PackageRegistry::new()` + - `harness-internal/runner/tests/it/harness.rs` (line 66) and `harness-errors.rs` (line 49): `PackageRegistry::new()` + - `harness/tests/suite/host.rs` (lines 158 and 268), `launch.rs` (line 29), and `broker.rs` (line 91): `PackageRegistry::new()`. `asker.rs` (line 43) adds `ASKER` to a `PackageRegistry::new()`, then installs `ASKER.name`. + - `workshop/agents/tests/it/support.rs` (line 63) and `workshop/agents/src/input-tests.rs` (line 270): add `plugin_user_input::PACKAGE`, then install `plugin_user_input::PACKAGE.name` +- Docs: + - `AGENTS.md` line 12: the Plugin definition becomes the Technical Design's quoted text, word for word. + - `crates/promptforge-plugin/src/plugin.rs`: the `Package` doc's linking sentence becomes "Adding it to the Harness's package registry is also what links the Plugin crate into the Host." +- Tests, named as in the Testing Plan: + - `packages-tests.rs`: `natives_offer_no_tools_prelude_or_needs`, looping over `NATIVE` + - new `crates/harness-internal/runner/tests/it/packages.rs`, declared with `mod packages;` beside `mod prepare;` in `main.rs`: the three registry tests and the three install tests, built from `echo_package` + - `host-tests.rs`: `a_declared_native_is_met_when_installed_and_missing_when_not` and `an_installed_native_adds_no_catalog_tool_and_no_prelude` + - `crates/workshop/server/src/agents/tests.rs`: `a_prompt_declaring_subagents_prepares_on_the_servers_installed_plugins` + - Every existing install, snapshot, preparation, and `tasks` test passes with only its call site changed. +- Verify, from the `promptforge` root after this step's edits; fixes land in this step's commit: + - every exit criterion in the Testing Plan + - `rg -n "HostContext::new\(HostServices|HostContext::new\(services\)|HostContext::new\(wide\)" crates` prints nothing + - `git diff --stat e7c376a04 -- crates/harness-internal/runner/src/host-run.rs crates/promptforge-internal` prints nothing, so the run snapshot, the Engine, the `tasks` table, and `RESERVED_NAMES` are unchanged +- Commit: one `promptforge` commit, `Install Plugins by name from a package registry`. + + + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..7b1126a8c --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-10-11-1-package-registry.md From e2a5ad4d6d638c3c0714a3fe53f03506e370ffa0 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 04:09:30 -0700 Subject: [PATCH 12/34] Install Plugins by name from a package registry A Host now adds the Plugin packages it links to a package registry, builds its context over that registry, and installs each Plugin by package name. Every registry starts out holding the Engine-native subagents package, which offers no tools of its own and keeps its own local name, so no other package can install under that name. An unknown package, a duplicate package, or a misused native name is refused at startup, and a Host can list every package it may install along with its kind. Workshop now installs the native subagents Plugin alongside web and user-input. - `PackageRegistry` maps each package name to its `Package`, its `PluginKind`, and its default local name, which `add` parses once. `HostContext::install` looks the name up there and no longer re-checks the `vendor/name` form. - `HostContext::new` takes the registry by value beside the Host-wide services, and `install` takes a package name as `&str` instead of a `Package`. Every Host and test now adds its packages before building the context, then installs by `package.name`. - `NATIVE` defines `promptforge/subagents` as an ordinary `Package` with a `Native` object, so it goes through the same install path as a linked package. - `InstallError` adds `DuplicatePackage` and `UnknownPackage`, and `InvalidPackage` now comes only from `add`. `add` refuses a native's name as a duplicate, and the unknown-package message tells the Host to call `PackageRegistry::add` first. - `ReservedName` refuses the native under any name but `subagents` and refuses every other package under `subagents`, whether the Host picks that name or it is the default. Installing the native twice is `NameTaken`. - `with_plugins` adds the web, user-input, and MCP packages to a new registry and installs the native after user-input. A refused add or native install only logs a warning, and the remaining installs still run. - `PackageRegistry::default` skips a native whose name fails to parse instead of panicking. The registry listing test pins that `promptforge/subagents` parses. - `Native` offers no tools, prelude, or needs, and its `call` returns an error saying the Engine answers native calls. Installing it satisfies a prompt that declares `subagents` but adds nothing to the run's catalog or preludes. - `PluginKind` has only `Native` and `Linked`; the library kind that the Plugin definition in `AGENTS.md` names is not built yet. Design: new registry @ crates/harness-internal/runner/src/packages.rs::PackageRegistry boundary: pub Design: new pure-function @ crates/harness-internal/runner/src/packages.rs::native deps: HostServices,PluginId,Value Design: replaces pure-function @ crates/harness-internal/runner/src/packages.rs::package_name deps: str was: crates/harness-internal/runner/src/host.rs::package_name Design: extends constructor-injection @ crates/harness-internal/runner/src/host.rs::HostContext boundary: pub Design: new stringly-typed @ crates/harness-internal/runner/src/host.rs::HostContext::install boundary: pub Design: new shotgun-surgery @ crates/harness-internal/runner/src/host.rs::HostContext::new Design: extends surface-growth @ crates/harness-internal/runner/src/host.rs::InstallError boundary: pub Design: extends facade @ crates/harness/src/lib.rs::plugin boundary: pub Design: extends swallowed-exception @ crates/workshop/server/src/agents.rs::with_plugins deps: HostServices,Vec Plan: vibe/2026-10-11-1-package-registry.md --- AGENTS.md | 2 +- .../harness-internal/runner/src/host-tests.rs | 151 ++++++++------- crates/harness-internal/runner/src/host.rs | 119 ++++++++---- crates/harness-internal/runner/src/lib.rs | 7 + .../runner/src/packages-tests.rs | 18 ++ .../harness-internal/runner/src/packages.rs | 160 ++++++++++++++++ .../runner/tests/it/harness-errors.rs | 7 +- .../runner/tests/it/harness-stop-timing.rs | 14 +- .../runner/tests/it/harness.rs | 9 +- .../harness-internal/runner/tests/it/main.rs | 1 + .../runner/tests/it/packages.rs | 177 ++++++++++++++++++ .../runner/tests/it/prepare-prelude.rs | 9 +- .../runner/tests/it/prepare-ready.rs | 4 +- .../runner/tests/it/prepare.rs | 16 +- .../runner/tests/it/scripted.rs | 10 +- .../runner/tests/it/support.rs | 15 +- crates/harness/examples/run-prompt.rs | 7 +- crates/harness/src/lib.rs | 23 ++- crates/harness/tests/suite/asker.rs | 10 +- crates/harness/tests/suite/broker.rs | 7 +- crates/harness/tests/suite/host.rs | 12 +- crates/harness/tests/suite/launch.rs | 7 +- crates/promptforge-plugin/src/plugin.rs | 6 +- crates/workshop/agents/src/input-tests.rs | 16 +- crates/workshop/agents/tests/it/support.rs | 16 +- crates/workshop/server/src/agents.rs | 44 +++-- crates/workshop/server/src/agents/tests.rs | 15 +- vibe/2026-10-11-1-package-registry.md | 9 +- 28 files changed, 713 insertions(+), 178 deletions(-) create mode 100644 crates/harness-internal/runner/src/packages-tests.rs create mode 100644 crates/harness-internal/runner/src/packages.rs create mode 100644 crates/harness-internal/runner/tests/it/packages.rs diff --git a/AGENTS.md b/AGENTS.md index c66af4a68..754d0d2cf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ Four words have exactly one meaning each, everywhere in this repository: code co - **Engine**: the `promptforge` and `promptforge-*` crates. The Engine parses a prompt and steps a run. Whenever a run needs a model reply, a tool result, a timer, or a file, the Engine emits an effect and waits for its caller to answer it. In production the caller is the Harness. - **Harness**: the `harness` and `harness-*` crates. The Harness steps the Engine, performs every effect, returns each answer, and records every run through the recorder the Host supplies. Production code runs prompts only through the Harness. - **Host**: an application that runs prompts through the Harness, such as Workshop or Papergate. The Host makes every policy decision. It runs prompts only through the Harness; it may also use the Engine's parser and types to read prompts and show events. -- **Plugin**: a `plugin-*` crate implementing `promptforge-plugin`, installed into a `HostContext` under a local name the Host chooses, such as `web`. Every tool it offers sits under that name (`web/fetch`). Every run receives every usable Plugin's tools: those of each installed Plugin that built and whose per-run services the run has. Declaring a Plugin under `plugins:` in a prompt's frontmatter installs its Lua prelude and makes it required, and the prompt's Lua decides what the model sees. A Plugin has no other name: never capability, pack, or addon. +- **Plugin**: a package of tools a Host adds to its package registry and installs into a `HostContext` by package name, such as `promptforge/web`, under a local name the Host chooses, such as `web`. Every tool it offers sits under that name (`web/fetch`). A Plugin is linked (a `plugin-*` crate implementing `promptforge-plugin`), library (loaded from a dynamic library; not built yet), or Engine-native (`promptforge/subagents`, implemented in the Engine under the fixed name `subagents`, which no other Plugin may take). Linked and library Plugins use only the `promptforge-plugin` API. Every run receives every usable Plugin's tools: those of each installed Plugin that built and whose per-run services the run has. Declaring a Plugin under `plugins:` in a prompt's frontmatter installs its Lua prelude and makes it required, and the prompt's Lua decides what the model sees. A Plugin has no other name: never capability, pack, or addon. ### Using the terms diff --git a/crates/harness-internal/runner/src/host-tests.rs b/crates/harness-internal/runner/src/host-tests.rs index 0633aeaf4..0c5391a0f 100644 --- a/crates/harness-internal/runner/src/host-tests.rs +++ b/crates/harness-internal/runner/src/host-tests.rs @@ -1,7 +1,7 @@ //! Tests for the Host's install and each run's snapshot: the default name, -//! the refused packages and names, a construct failure stored as -//! unavailable, the requirements a run reports, the tools its catalog -//! keeps, the preludes it installs, and where each call goes. +//! the refused names, a construct failure stored as unavailable, the +//! requirements a run reports, the native that offers nothing, the tools +//! its catalog keeps, the preludes it installs, and where each call goes. use std::collections::BTreeMap; use std::sync::Arc; @@ -16,6 +16,7 @@ use promptforge_plugin::{ use serde_json::{Value, json}; use super::{HostContext, InstallError}; +use crate::packages::PackageRegistry; use crate::performers::ToolPerformer; /// A Host-wide service a fixture's `construct` reads. @@ -102,8 +103,19 @@ fn name(text: &str) -> PluginId { PluginId::parse(text).unwrap() } -fn host() -> HostContext { - HostContext::new(HostServices::new()) +/// A Host with no Host-wide services over a registry holding `packages`. +fn host(packages: &[Package]) -> HostContext { + host_with(HostServices::new(), packages) +} + +/// A Host over the Host-wide `services` and a registry holding +/// `packages`. +fn host_with(services: HostServices, packages: &[Package]) -> HostContext { + let mut registry = PackageRegistry::new(); + for package in packages { + registry.add(*package).unwrap(); + } + HostContext::new(registry, services) } /// A prompt with the given `plugins:` and `tools:` frontmatter lines. @@ -128,54 +140,36 @@ fn catalog_ids(host: &HostContext, services: HostServices, prompt: &Prompt) -> V #[test] fn install_names_the_plugin_after_its_package_names_second_segment_unless_the_host_picks_one() { - let mut host = host(); + let mut host = host(&[package("acme/web")]); + assert_eq!(host.install("acme/web", None, Value::Null), Ok(name("web"))); assert_eq!( - host.install(package("acme/web"), None, Value::Null), - Ok(name("web")) - ); - assert_eq!( - host.install(package("acme/web"), Some(name("browse")), Value::Null), + host.install("acme/web", Some(name("browse")), Value::Null), Ok(name("browse")) ); } -#[test] -fn install_refuses_a_package_name_that_is_not_a_vendor_name_pair() { - for bad in [ - "web", - "acme/web/extra", - "/web", - "acme/", - "Acme/web", - "acme/Web", - ] { - assert_eq!( - host().install(package(bad), None, Value::Null), - Err(InstallError::InvalidPackage { package: bad }), - "{bad}" - ); - } -} - #[test] fn install_refuses_a_name_already_installed_and_its_punctuation_twin() { - let mut host = host(); - host.install(package("acme/user-input"), None, Value::Null) - .unwrap(); + let mut host = host(&[ + package("acme/user-input"), + package("other/user_input"), + package("other/ask"), + ]); + host.install("acme/user-input", None, Value::Null).unwrap(); let taken = |name_: &str| InstallError::NameTaken { name: name(name_), existing: name("user-input"), }; assert_eq!( - host.install(package("acme/user-input"), None, Value::Null), + host.install("acme/user-input", None, Value::Null), Err(taken("user-input")) ); assert_eq!( - host.install(package("other/user_input"), None, Value::Null), + host.install("other/user_input", None, Value::Null), Err(taken("user_input")) ); assert_eq!( - host.install(package("other/ask"), Some(name("user.input")), Value::Null), + host.install("other/ask", Some(name("user.input")), Value::Null), Err(taken("user.input")) ); assert_eq!( @@ -187,9 +181,9 @@ fn install_refuses_a_name_already_installed_and_its_punctuation_twin() { #[test] fn a_construct_failure_is_stored_and_reported_as_unavailable_with_its_reason() { let backed_package = Package::new("acme/backed", backed); - let mut bare = host(); + let mut bare = host(&[backed_package]); assert_eq!( - bare.install(backed_package, None, Value::Null), + bare.install(backed_package.name, None, Value::Null), Ok(name("backed")), "a construct failure is not an install error" ); @@ -209,8 +203,10 @@ fn a_construct_failure_is_stored_and_reported_as_unavailable_with_its_reason() { let mut wide = HostServices::new(); wide.provide(&BACKEND, Arc::from("up")).unwrap(); - let mut provided = HostContext::new(wide); - provided.install(backed_package, None, Value::Null).unwrap(); + let mut provided = host_with(wide, &[backed_package]); + provided + .install(backed_package.name, None, Value::Null) + .unwrap(); let (_run, _env, requirements) = provided.begin_run( HostServices::new(), BTreeMap::new(), @@ -224,13 +220,12 @@ fn a_construct_failure_is_stored_and_reported_as_unavailable_with_its_reason() { #[test] fn begin_run_reports_every_declared_plugin_it_cannot_serve() { - let mut host = host(); let needy = package("acme/needy").needs(NEEDS_SESSION); let broken = Package::new("acme/broken", backed); - host.install(needy, None, Value::Null).unwrap(); - host.install(broken, None, Value::Null).unwrap(); - host.install(package("acme/fine"), None, Value::Null) - .unwrap(); + let mut host = host(&[needy, broken, package("acme/fine")]); + host.install(needy.name, None, Value::Null).unwrap(); + host.install(broken.name, None, Value::Null).unwrap(); + host.install("acme/fine", None, Value::Null).unwrap(); let (_run, _env, requirements) = host.begin_run( HostServices::new(), @@ -250,11 +245,40 @@ fn begin_run_reports_every_declared_plugin_it_cannot_serve() { assert_eq!(requirements.unavailable[0].plugin, name("broken")); } +#[test] +fn a_declared_native_is_met_when_installed_and_missing_when_not() { + let declares = prompt("plugins:\n - subagents\n"); + let mut host = host(&[]); + let (_run, _env, requirements) = + host.begin_run(HostServices::new(), BTreeMap::new(), &declares); + assert_eq!(requirements.missing_required, [name("subagents")]); + + assert_eq!( + host.install("promptforge/subagents", None, Value::Null), + Ok(name("subagents")) + ); + let (_run, _env, requirements) = + host.begin_run(HostServices::new(), BTreeMap::new(), &declares); + assert!(requirements.is_satisfied(), "{requirements:?}"); +} + +#[test] +fn an_installed_native_adds_no_catalog_tool_and_no_prelude() { + let mut host = host(&[]); + host.install("promptforge/subagents", None, Value::Null) + .unwrap(); + let declares = prompt("plugins:\n - subagents\n"); + assert!(catalog_ids(&host, HostServices::new(), &declares).is_empty()); + let (run, _env, _requirements) = + host.begin_run(HostServices::new(), BTreeMap::new(), &declares); + assert!(run.preludes(&declares).is_empty()); +} + #[test] fn begin_run_drops_a_tool_outside_its_plugin_and_a_repeated_id() { - let mut host = host(); + let mut host = host(&[package("acme/kit")]); let config = json!([tool("kit/good"), tool("other/escape"), tool("kit/good")]); - host.install(package("acme/kit"), None, config).unwrap(); + host.install("acme/kit", None, config).unwrap(); assert_eq!( catalog_ids(&host, HostServices::new(), &prompt("")), ["kit/good"] @@ -263,9 +287,9 @@ fn begin_run_drops_a_tool_outside_its_plugin_and_a_repeated_id() { #[test] fn a_plugin_whose_needs_the_run_lacks_offers_it_no_tools() { - let mut host = host(); let needy = package("acme/needy").needs(NEEDS_SESSION); - host.install(needy, None, json!([tool("needy/use")])) + let mut host = host(&[needy]); + host.install(needy.name, None, json!([tool("needy/use")])) .unwrap(); assert!(catalog_ids(&host, HostServices::new(), &prompt("")).is_empty()); @@ -276,12 +300,13 @@ fn a_plugin_whose_needs_the_run_lacks_offers_it_no_tools() { #[test] fn an_undeclared_plugins_tools_reach_the_catalog_but_its_prelude_does_not() { - let mut host = host(); let declared = package("acme/declared").prelude("declared = {}\n"); let extra = package("acme/extra").prelude("extra = {}\n"); - host.install(declared, None, json!([tool("declared/a")])) + let mut host = host(&[declared, extra]); + host.install(declared.name, None, json!([tool("declared/a")])) + .unwrap(); + host.install(extra.name, None, json!([tool("extra/b")])) .unwrap(); - host.install(extra, None, json!([tool("extra/b")])).unwrap(); let prompt = prompt("plugins:\n - declared\n"); assert_eq!( catalog_ids(&host, HostServices::new(), &prompt), @@ -298,13 +323,13 @@ fn an_undeclared_plugins_tools_reach_the_catalog_but_its_prelude_does_not() { #[test] fn preludes_follow_the_prompts_declaration_order_and_carry_the_installed_name() { - let mut host = host(); + let with_prelude = + |package_name: &'static str| package(package_name).prelude("local plugin = ...\n"); + let mut host = host(&[with_prelude("acme/first"), with_prelude("acme/second")]); for package_name in ["acme/first", "acme/second"] { - let with_prelude = package(package_name).prelude("local plugin = ...\n"); - host.install(with_prelude, None, Value::Null).unwrap(); + host.install(package_name, None, Value::Null).unwrap(); } - let renamed = package("acme/first").prelude("local plugin = ...\n"); - host.install(renamed, Some(name("third")), Value::Null) + host.install("acme/first", Some(name("third")), Value::Null) .unwrap(); let prompt = prompt("plugins:\n - third\n - second\n - first\n"); let (run, _env, _requirements) = host.begin_run(HostServices::new(), BTreeMap::new(), &prompt); @@ -337,10 +362,10 @@ async fn call(run: &dyn ToolPerformer, tool: &str) -> Result, } @@ -53,22 +58,24 @@ struct Installed { } impl HostContext { - /// A context with no Plugins, whose installs hand `services` to each - /// package's `construct`. + /// A context with no Plugins, which installs from the packages + /// `packages` holds and hands `services` to each package's + /// `construct`. /// /// `services` are Host-wide, such as a search provider. They reach /// only `construct`; a tool call reads the run's own services instead. #[must_use] - pub fn new(services: HostServices) -> HostContext { + pub fn new(packages: PackageRegistry, services: HostServices) -> HostContext { HostContext { + packages, services, installed: Vec::new(), } } - /// Builds the Plugin with `package.construct` and adds it under `name`, - /// or under the package name's second segment when `name` is `None`, - /// and returns the name it used. + /// Builds the Plugin of the package the registry holds as `package` + /// and adds it under `name`, or under the package name's second + /// segment when `name` is `None`, and returns the name it used. /// /// `construct` runs once, here, with the name, `config`, and the /// Host-wide services. A `construct` failure is not an error here: the @@ -76,20 +83,34 @@ impl HostContext { /// run that needs it is refused. /// /// # Errors - /// Returns [`InstallError::InvalidPackage`] when the package name is - /// not a `vendor/name` pair of valid one-segment names, and - /// [`InstallError::NameTaken`] when the name is installed already or - /// differs from an installed name only by `-`, `_`, or `.`. + /// Returns [`InstallError::UnknownPackage`] when the registry does not + /// hold `package`, [`InstallError::ReservedName`] when a native is + /// asked for under a name other than its own or another package under + /// a native's name, and [`InstallError::NameTaken`] when the name is + /// installed already or differs from an installed name only by `-`, + /// `_`, or `.`. pub fn install( &mut self, - package: Package, + package: &str, name: Option, config: serde_json::Value, ) -> Result { - let default = package_name(package.name).ok_or(InstallError::InvalidPackage { - package: package.name, - })?; - let name = name.unwrap_or(default); + let Some((package, kind, default)) = self.packages.entry(package) else { + return Err(InstallError::UnknownPackage { + package: package.to_owned(), + }); + }; + let name = name.unwrap_or_else(|| default.clone()); + let reserved = match kind { + PluginKind::Native => name != *default, + PluginKind::Linked => self.packages.reserves(&name), + }; + if reserved { + return Err(InstallError::ReservedName { + package: package.name, + name, + }); + } let twin = normalize(&name); if let Some(existing) = self .installed @@ -182,7 +203,8 @@ impl HostContext { } impl fmt::Debug for HostContext { - /// Shows the services and each installed name with whether it built. + /// Shows the services, each installed name with whether it built, and + /// the registry. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let installed: Vec<(&PluginId, bool)> = self .installed @@ -192,25 +214,52 @@ impl fmt::Debug for HostContext { f.debug_struct("HostContext") .field("services", &self.services) .field("installed", &installed) + .field("packages", &self.packages) .finish() } } -/// A mistake [`HostContext::install`] refuses outright, so the Host's -/// author sees it at startup. +/// A mistake [`PackageRegistry::add`] or [`HostContext::install`] refuses +/// outright, so the Host's author sees it at startup. /// /// A Plugin whose `construct` fails is not one of these; it is installed /// as unavailable instead. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] #[non_exhaustive] pub enum InstallError { - /// The package name is not `vendor/name`, or a segment of it is not a - /// valid one-segment name. + /// The added package's name is not `vendor/name`, or a segment of it + /// is not a valid one-segment name. #[error("Plugin package name `{package}` is not a vendor/name pair")] InvalidPackage { /// The package name as the label spells it. package: &'static str, }, + /// The registry holds the added package's name already, as a native or + /// as a package added before. + #[error("Plugin package `{package}` is already in the registry")] + DuplicatePackage { + /// The package name as the label spells it. + package: &'static str, + }, + /// The registry holds no package by the name the install asked for. + #[error( + "Plugin package `{package}` is not in the registry; add it with PackageRegistry::add before installing it" + )] + UnknownPackage { + /// The package name the install asked for. + package: String, + }, + /// A native was asked for under a name other than its own, or another + /// package under a native's name. + #[error( + "Plugin package `{package}` cannot be installed as {name}: a native Plugin keeps its own name, and no other package may take it" + )] + ReservedName { + /// The package name as the label spells it. + package: &'static str, + /// The name the install asked for. + name: PluginId, + }, /// The name is installed already (`existing` equals `name`), or it /// differs from the installed `existing` only by `-`, `_`, or `.`, as /// `user-input` and `user_input` do, which a model reading the tool @@ -226,14 +275,6 @@ pub enum InstallError { }, } -/// The default local name for the package name `vendor/name`: its second -/// segment, when both segments are valid one-segment names. -fn package_name(package: &str) -> Option { - let (vendor, name) = package.split_once('/')?; - PluginId::parse(vendor).ok()?; - PluginId::parse(name).ok() -} - /// A name with each `-`, `_`, and `.` mapped to `-`, so two names that /// differ only by that punctuation compare equal. Names are lowercase /// only, so case needs no handling. diff --git a/crates/harness-internal/runner/src/lib.rs b/crates/harness-internal/runner/src/lib.rs index d4606ac4d..5a0ea80d3 100644 --- a/crates/harness-internal/runner/src/lib.rs +++ b/crates/harness-internal/runner/src/lib.rs @@ -26,6 +26,11 @@ //! - The loop never reads an event to decide anything; control comes //! from the run's own word (`Step`, `Run::decided`), the cancel flag, //! and the Host's stop. +//! - A Host installs a Plugin only by a package name its +//! [`PackageRegistry`] holds. The registry holds the native +//! `promptforge/subagents` from the start and refuses a duplicate name, +//! and install refuses the native's local name to every other package, +//! so no other Plugin can take `subagents`. mod display_chain; pub mod effect_loop; @@ -33,6 +38,7 @@ pub mod environment; pub mod files; mod harness; mod host; +mod packages; pub mod performers; pub mod prepare; pub mod recorder; @@ -40,6 +46,7 @@ pub mod recorder; pub use display_chain::display_chain; pub use harness::{Harness, HarnessError, RunControl, RunReport, RunRequest}; pub use host::{HostContext, InstallError}; +pub use packages::{PackageRegistry, PluginKind}; pub use recorder::{ MemoryRecorder, Record, RecordKind, RecorderError, RecorderFuture, RunId, RunMeta, RunOutcome, RunRecorder, diff --git a/crates/harness-internal/runner/src/packages-tests.rs b/crates/harness-internal/runner/src/packages-tests.rs new file mode 100644 index 000000000..e99f55bf3 --- /dev/null +++ b/crates/harness-internal/runner/src/packages-tests.rs @@ -0,0 +1,18 @@ +//! Tests for the natives every registry holds: each builds a Plugin that +//! offers no tools, and carries no prelude and no needs. + +use promptforge_plugin::HostServices; +use serde_json::Value; + +use super::{NATIVE, package_name}; + +#[test] +fn natives_offer_no_tools_prelude_or_needs() { + for package in NATIVE { + assert!(package.prelude.is_none(), "{} has a prelude", package.name); + assert!(package.needs.is_empty(), "{} has needs", package.name); + let name = package_name(package.name).unwrap(); + let plugin = (package.construct)(&name, Value::Null, &HostServices::new()).unwrap(); + assert!(plugin.tools().is_empty(), "{} offers tools", package.name); + } +} diff --git a/crates/harness-internal/runner/src/packages.rs b/crates/harness-internal/runner/src/packages.rs new file mode 100644 index 000000000..80d9424b1 --- /dev/null +++ b/crates/harness-internal/runner/src/packages.rs @@ -0,0 +1,160 @@ +//! The packages a Host can install: [`PackageRegistry`], holding the +//! natives from the start and each linked package the Host adds, and the +//! [`PluginKind`] that says which a package is. + +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use std::sync::Arc; + +use promptforge_plugin::{ + HostServices, Package, Plugin, PluginFuture, PluginId, ToolContext, ToolDescriptor, ToolError, + ToolOutput, +}; +use serde_json::Value; + +use crate::host::InstallError; + +#[cfg(test)] +#[path = "packages-tests.rs"] +mod tests; + +/// The packages a Host can install, by package name. +/// +/// Every registry holds the natives, such as `promptforge/subagents`, from +/// the start. A Host adds the `PACKAGE` constant of each Plugin crate it +/// links, lists what the registry holds, and builds its +/// [`HostContext`](crate::HostContext) over it, which installs Plugins by +/// package name. +#[derive(Debug)] +pub struct PackageRegistry { + entries: BTreeMap<&'static str, (Package, PluginKind, PluginId)>, +} + +/// Where a package's Plugin comes from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum PluginKind { + /// Built into every registry; what it does lives in the Engine. + Native, + /// Added by the Host with [`PackageRegistry::add`]. + Linked, +} + +impl Default for PackageRegistry { + /// A registry holding only the natives. + fn default() -> PackageRegistry { + let entries = NATIVE + .into_iter() + .filter_map(|package| { + let default = package_name(package.name)?; + Some((package.name, (package, PluginKind::Native, default))) + }) + .collect(); + PackageRegistry { entries } + } +} + +impl PackageRegistry { + /// A registry holding only the natives. + #[must_use] + pub fn new() -> PackageRegistry { + PackageRegistry::default() + } + + /// Adds `package` as a linked package, so a Host can install it by its + /// name. + /// + /// # Errors + /// Returns [`InstallError::InvalidPackage`] when the package name is + /// not a `vendor/name` pair of valid one-segment names, and + /// [`InstallError::DuplicatePackage`] when the registry holds the name + /// already, as it holds each native's from the start. + pub fn add(&mut self, package: Package) -> Result<(), InstallError> { + let default = package_name(package.name).ok_or(InstallError::InvalidPackage { + package: package.name, + })?; + match self.entries.entry(package.name) { + Entry::Occupied(_) => Err(InstallError::DuplicatePackage { + package: package.name, + }), + Entry::Vacant(slot) => { + slot.insert((package, PluginKind::Linked, default)); + Ok(()) + } + } + } + + /// Every package the registry holds, natives included, with its kind, + /// in package name order. + pub fn list(&self) -> impl Iterator { + self.entries + .values() + .map(|(package, kind, _)| (package, *kind)) + } + + /// The package named `package`, its kind, and its default local name. + pub(crate) fn entry(&self, package: &str) -> Option<(Package, PluginKind, &PluginId)> { + let (package, kind, default) = self.entries.get(package)?; + Some((*package, *kind, default)) + } + + /// Whether `name` is a native's default local name, which only that + /// native may be installed under. + pub(crate) fn reserves(&self, name: &PluginId) -> bool { + self.entries + .values() + .any(|(_, kind, default)| *kind == PluginKind::Native && default == name) + } +} + +/// The natives every registry holds from the start. +/// +/// A native Plugin's behavior and tools live in the Engine. Its Plugin +/// object exists only so install, readiness, and the run snapshot treat it +/// like any other Plugin, so a native must never offer tools, carry a +/// prelude, or need services. Its `call` is unreachable, because the run +/// snapshot routes a call only to a tool in a Plugin's own list. +const NATIVE: [Package; 1] = [Package::new("promptforge/subagents", native) + .description("Lets the model start, wait for, and cancel subagents")]; + +/// Builds a native's [`Native`] object; see [`NATIVE`]. +#[expect( + clippy::unnecessary_wraps, + reason = "the Package construct signature fixes the return type" +)] +fn native( + _name: &PluginId, + _config: Value, + _services: &HostServices, +) -> Result, ToolError> { + Ok(Arc::new(Native)) +} + +/// The object of a native Plugin, which offers no tools; see [`NATIVE`]. +struct Native; + +impl Plugin for Native { + fn tools(&self) -> Vec { + Vec::new() + } + + fn call<'a>( + &'a self, + cx: ToolContext<'a>, + _args: Value, + ) -> PluginFuture<'a, Result> { + let message = format!( + "{} belongs to a native Plugin, whose calls the Engine answers", + cx.tool() + ); + Box::pin(async move { Err(ToolError::message(message)) }) + } +} + +/// The default local name for the package name `vendor/name`: its second +/// segment, when both segments are valid one-segment names. +pub(crate) fn package_name(package: &str) -> Option { + let (vendor, name) = package.split_once('/')?; + PluginId::parse(vendor).ok()?; + PluginId::parse(name).ok() +} diff --git a/crates/harness-internal/runner/tests/it/harness-errors.rs b/crates/harness-internal/runner/tests/it/harness-errors.rs index f20652844..56178f649 100644 --- a/crates/harness-internal/runner/tests/it/harness-errors.rs +++ b/crates/harness-internal/runner/tests/it/harness-errors.rs @@ -13,7 +13,7 @@ use std::task::Poll; use harness_runner::display_chain; use harness_runner::environment::CurrentModelError; use harness_runner::recorder::MemoryRecorder; -use harness_runner::{Harness, HarnessError, HostContext}; +use harness_runner::{Harness, HarnessError, HostContext, PackageRegistry}; use promptforge::model::{Completion, CompletionError, CompletionErrorKind}; use promptforge_plugin::HostServices; @@ -46,7 +46,10 @@ fn harness_over(recorder: &Arc, broker: ScriptedBroker) -> Harn recorder.clone(), Arc::new(broker), Arc::new(PendingTimer::default()), - Arc::new(HostContext::new(HostServices::new())), + Arc::new(HostContext::new( + PackageRegistry::new(), + HostServices::new(), + )), HostServices::new(), ) } diff --git a/crates/harness-internal/runner/tests/it/harness-stop-timing.rs b/crates/harness-internal/runner/tests/it/harness-stop-timing.rs index fc4f72497..1e44033c9 100644 --- a/crates/harness-internal/runner/tests/it/harness-stop-timing.rs +++ b/crates/harness-internal/runner/tests/it/harness-stop-timing.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; use harness_runner::recorder::MemoryRecorder; -use harness_runner::{Harness, HostContext, RunRequest}; +use harness_runner::{Harness, HostContext, PackageRegistry, RunRequest}; use promptforge::vfs::VfsRef; use promptforge_plugin::HostServices; use serde_json::json; @@ -18,7 +18,9 @@ use super::{ HookedBackend, PendingTimer, RunOutcome, answers_to, answers_where, calls, plain_harness, request, run_beside, until, }; +use crate::asker::ASKER; use crate::scripted::{Held, HeldTimer, Operator, ScriptedBroker, held_broker, with_asker}; +use crate::support::registry; /// A main section parked under a `pcall` on a 30-second timed /// `tasks.join_any` over a child that asks the operator. Once the wait @@ -64,7 +66,10 @@ const WRITES_THEN_INFERS: &str = "---\nname: writes-infers\ndescription: d\nprom /// A Host holding only the fixture ask Plugin, as `user-input`. fn asker_host() -> Arc { - Arc::new(with_asker(HostContext::new(HostServices::new()))) + Arc::new(with_asker(HostContext::new( + registry(&[ASKER]), + HostServices::new(), + ))) } /// A store whose every write raises a stop through `harness`'s control. @@ -138,7 +143,10 @@ async fn a_stop_seen_before_a_steps_effects_start_drops_the_round_in_flight_and_ recorder.clone(), Arc::new(held_broker(&chat)), Arc::new(PendingTimer::default()), - Arc::new(HostContext::new(HostServices::new())), + Arc::new(HostContext::new( + PackageRegistry::new(), + HostServices::new(), + )), HostServices::new(), ); let vfs = stopping_store(&harness); diff --git a/crates/harness-internal/runner/tests/it/harness.rs b/crates/harness-internal/runner/tests/it/harness.rs index 84cf8036d..8787b68d1 100644 --- a/crates/harness-internal/runner/tests/it/harness.rs +++ b/crates/harness-internal/runner/tests/it/harness.rs @@ -18,7 +18,9 @@ use std::time::Duration; use harness_runner::environment::HostSnapshot; use harness_runner::files::OutputError; use harness_runner::recorder::{MemoryRecorder, Record, RecordKind, RunOutcome}; -use harness_runner::{Harness, HarnessError, HostContext, RunControl, RunReport, RunRequest}; +use harness_runner::{ + Harness, HarnessError, HostContext, PackageRegistry, RunControl, RunReport, RunRequest, +}; use promptforge::vfs::VfsRef; use promptforge_plugin::HostServices; use serde_json::{Value, json}; @@ -63,7 +65,10 @@ fn plain_harness(recorder: &Arc, broker: ScriptedBroker) -> Harn recorder.clone(), Arc::new(broker), Arc::new(PendingTimer::default()), - Arc::new(HostContext::new(HostServices::new())), + Arc::new(HostContext::new( + PackageRegistry::new(), + HostServices::new(), + )), HostServices::new(), ) } diff --git a/crates/harness-internal/runner/tests/it/main.rs b/crates/harness-internal/runner/tests/it/main.rs index 901211b8e..2f5892d82 100644 --- a/crates/harness-internal/runner/tests/it/main.rs +++ b/crates/harness-internal/runner/tests/it/main.rs @@ -11,6 +11,7 @@ mod asker; mod effect_loop; mod harness; +mod packages; mod performers; mod prepare; mod recorder; diff --git a/crates/harness-internal/runner/tests/it/packages.rs b/crates/harness-internal/runner/tests/it/packages.rs new file mode 100644 index 000000000..a79e61900 --- /dev/null +++ b/crates/harness-internal/runner/tests/it/packages.rs @@ -0,0 +1,177 @@ +//! The package registry and installs by name: a new registry holds the +//! native `promptforge/subagents` and lists every package in name order, +//! `add` refuses a name that is not a vendor/name pair and a name the +//! registry holds already, and install refuses a package the registry +//! does not hold, keeps the native's name to the native, and refuses the +//! native's second install as a taken name. + +use harness_runner::{HostContext, InstallError, PackageRegistry, PluginKind}; +use promptforge_plugin::{HostServices, PluginId, ServiceId, ServiceKey}; +use serde_json::Value; + +use crate::prepare::echo_package; +use crate::support::registry; + +/// The native's package name. +const SUBAGENTS: &str = "promptforge/subagents"; + +/// The test-only service a listed package needs. +const GREETING: ServiceKey = ServiceKey::new("tests/greeting"); + +const NEEDS: &[ServiceId] = &[GREETING.id()]; + +const NO_NEEDS: &[ServiceId] = &[]; + +fn name(text: &str) -> PluginId { + PluginId::parse(text).unwrap() +} + +#[test] +fn a_new_registry_holds_the_native_and_lists_every_package_in_name_order() { + let mut packages = PackageRegistry::new(); + packages + .add(echo_package("tests/zeta", None).needs(NEEDS)) + .unwrap(); + packages.add(echo_package("acme/alpha", None)).unwrap(); + let listed: Vec<(&str, PluginKind, &[ServiceId])> = packages + .list() + .map(|(package, kind)| (package.name, kind, package.needs)) + .collect(); + assert_eq!( + listed, + [ + ("acme/alpha", PluginKind::Linked, NO_NEEDS), + (SUBAGENTS, PluginKind::Native, NO_NEEDS), + ("tests/zeta", PluginKind::Linked, NEEDS), + ], + "a package added with needs lists those needs" + ); + let (native, _kind) = packages.list().nth(1).unwrap(); + assert!( + !native.description.is_empty(), + "a Host listing packages can say what the native does" + ); +} + +#[test] +fn add_refuses_a_package_name_that_is_not_a_vendor_name_pair() { + for bad in [ + "web", + "acme/web/extra", + "/web", + "acme/", + "Acme/web", + "acme/Web", + ] { + assert_eq!( + PackageRegistry::new().add(echo_package(bad, None)), + Err(InstallError::InvalidPackage { package: bad }), + "{bad}" + ); + } +} + +#[test] +fn add_refuses_a_package_name_the_registry_holds_already_the_natives_included() { + let mut packages = registry(&[echo_package("tests/tools", None)]); + for taken in ["tests/tools", SUBAGENTS] { + assert_eq!( + packages.add(echo_package(taken, None)), + Err(InstallError::DuplicatePackage { package: taken }), + "{taken}" + ); + } + assert!( + packages + .list() + .any(|(package, kind)| package.name == SUBAGENTS && kind == PluginKind::Native), + "the refused add left the native in place" + ); +} + +#[test] +fn install_refuses_a_package_the_registry_does_not_hold() { + let mut host = HostContext::new( + registry(&[echo_package("tests/tools", None)]), + HostServices::new(), + ); + for missing in ["tests/other", "promptforge/tasks"] { + assert_eq!( + host.install(missing, None, Value::Null), + Err(InstallError::UnknownPackage { + package: missing.to_owned() + }), + "{missing}" + ); + } + let message = host + .install("tests/other", None, Value::Null) + .unwrap_err() + .to_string(); + assert!( + message.contains("tests/other") && message.contains("PackageRegistry::add"), + "the refusal names the package and how to add it: {message}" + ); +} + +#[test] +fn install_keeps_the_natives_name_to_the_native_alone() { + let mut host = HostContext::new( + registry(&[ + echo_package("tests/tools", None), + echo_package("acme/subagents", None), + ]), + HostServices::new(), + ); + assert_eq!( + host.install(SUBAGENTS, Some(name("helpers")), Value::Null), + Err(InstallError::ReservedName { + package: SUBAGENTS, + name: name("helpers"), + }), + "the native keeps its own name" + ); + assert_eq!( + host.install("tests/tools", Some(name("subagents")), Value::Null), + Err(InstallError::ReservedName { + package: "tests/tools", + name: name("subagents"), + }), + "a linked package may not pick the native's name" + ); + let refused = host.install("acme/subagents", None, Value::Null); + assert_eq!( + refused, + Err(InstallError::ReservedName { + package: "acme/subagents", + name: name("subagents"), + }), + "a linked package may not take the native's name by default" + ); + assert_eq!( + refused.unwrap_err().to_string(), + "Plugin package `acme/subagents` cannot be installed as subagents: \ + a native Plugin keeps its own name, and no other package may take it" + ); + assert_eq!( + host.install(SUBAGENTS, Some(name("subagents")), Value::Null), + Ok(name("subagents")), + "the native installs under its own name" + ); +} + +#[test] +fn installing_the_native_twice_is_a_taken_name() { + let mut host = HostContext::new(PackageRegistry::new(), HostServices::new()); + assert_eq!( + host.install(SUBAGENTS, None, Value::Null), + Ok(name("subagents")) + ); + assert_eq!( + host.install(SUBAGENTS, None, Value::Null), + Err(InstallError::NameTaken { + name: name("subagents"), + existing: name("subagents"), + }) + ); +} diff --git a/crates/harness-internal/runner/tests/it/prepare-prelude.rs b/crates/harness-internal/runner/tests/it/prepare-prelude.rs index 10bc3e93a..0361bfc62 100644 --- a/crates/harness-internal/runner/tests/it/prepare-prelude.rs +++ b/crates/harness-internal/runner/tests/it/prepare-prelude.rs @@ -78,10 +78,11 @@ async fn a_preludes_tool_calls_are_recorded_as_script_calls_from_the_section_tha #[tokio::test] async fn preludes_install_in_the_prompts_declaration_order_not_the_install_order() { - let mut host = bare(); - for name in ["tests/first", "tests/second"] { - host.install(echo_package(name, Some("shared = 1\n")), None, Value::Null) - .unwrap(); + let packages = + ["tests/first", "tests/second"].map(|name| echo_package(name, Some("shared = 1\n"))); + let mut host = HostContext::new(registry(&packages), HostServices::new()); + for package in packages { + host.install(package.name, None, Value::Null).unwrap(); } let declares_second_first = "---\nname: order\ndescription: d\npromptforge: 0\n\ plugins:\n - second\n - first\n---\n\n# Title\n\n## Only\n\n```lua\nreturn 'ran'\n```\n"; diff --git a/crates/harness-internal/runner/tests/it/prepare-ready.rs b/crates/harness-internal/runner/tests/it/prepare-ready.rs index d973420bd..1edab58c3 100644 --- a/crates/harness-internal/runner/tests/it/prepare-ready.rs +++ b/crates/harness-internal/runner/tests/it/prepare-ready.rs @@ -143,9 +143,9 @@ fn starting_host(names: &[&str], phase: Phase) -> (HostContext, Vec Vec /// A Host with no Plugins. pub(crate) fn bare() -> HostContext { - HostContext::new(HostServices::new()) + HostContext::new(registry(&[]), HostServices::new()) } /// A Host with `package` installed under its default name. pub(crate) fn installing(package: Package) -> HostContext { - let mut host = bare(); - host.install(package, None, Value::Null).unwrap(); + let mut host = HostContext::new(registry(&[package]), HostServices::new()); + host.install(package.name, None, Value::Null).unwrap(); host } @@ -379,9 +379,11 @@ async fn the_lists_split_the_offering_by_declared_plugin_and_a_script_calls_a_to local extra = extras[1]\n\ return #required .. ':' .. required[1].id .. '|' .. #extras .. ':' .. extra.id .. '|' .. \ tools.call(extra, { value = 'hi' })\n```\n"; - let mut host = installing(TOOLS); - host.install(echo_package("tests/extra", None), None, Value::Null) - .unwrap(); + let extra = echo_package("tests/extra", None); + let mut host = HostContext::new(registry(&[TOOLS, extra]), HostServices::new()); + for package in [TOOLS, extra] { + host.install(package.name, None, Value::Null).unwrap(); + } assert_eq!( completed(drive_over(offering, host).await), "1:tools/echo|1:extra/echo|hi", diff --git a/crates/harness-internal/runner/tests/it/scripted.rs b/crates/harness-internal/runner/tests/it/scripted.rs index 8dd7193b1..329e6b533 100644 --- a/crates/harness-internal/runner/tests/it/scripted.rs +++ b/crates/harness-internal/runner/tests/it/scripted.rs @@ -23,6 +23,7 @@ use serde_json::{Value, json}; use tokio::sync::mpsc; use crate::asker::{ASKER, AskBroker, broker_services}; +use crate::support::registry; /// The one model the scripted broker lists. pub(crate) const MODEL: &str = "m"; @@ -260,15 +261,16 @@ pub(crate) fn surviving_hold_host(held: &Arc) -> HostContext { fn host_holding(held: &Arc, survives_stop: bool) -> HostContext { let mut services = HostServices::new(); services.provide(&HELD, Arc::clone(held)).unwrap(); - let mut host = HostContext::new(services); - host.install(HOLD, None, json!({ "survives_stop": survives_stop })) + let mut host = HostContext::new(registry(&[HOLD, ASKER]), services); + host.install(HOLD.name, None, json!({ "survives_stop": survives_stop })) .unwrap(); host } -/// `host` with the fixture ask Plugin installed as `user-input`. +/// `host`, whose registry holds the fixture ask Plugin, with it installed +/// as `user-input`. pub(crate) fn with_asker(mut host: HostContext) -> HostContext { - host.install(ASKER, None, Value::Null).unwrap(); + host.install(ASKER.name, None, Value::Null).unwrap(); host } diff --git a/crates/harness-internal/runner/tests/it/support.rs b/crates/harness-internal/runner/tests/it/support.rs index 0f899cde5..4ead70f2d 100644 --- a/crates/harness-internal/runner/tests/it/support.rs +++ b/crates/harness-internal/runner/tests/it/support.rs @@ -1,10 +1,12 @@ //! Fixtures shared by the runner's suites: a one-section prompt, a run -//! over it, and fake performers that answer by script. +//! over it, fake performers that answer by script, and a package registry +//! holding a suite's packages. use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::task::Poll; +use harness_runner::PackageRegistry; use harness_runner::performers::{BoxFuture, InferenceBroker, Performers, Timer, ToolPerformer}; use harness_runner::recorder::{ MemoryRecorder, Record, RecorderError, RecorderFuture, RunId, RunMeta, RunOutcome, RunRecorder, @@ -20,11 +22,22 @@ use promptforge::vfs::{ VfsRef, }; use promptforge::{Environment, Prompt, Run, RunContext}; +use promptforge_plugin::Package; use serde_json::{Value, json}; /// The run's execution identifier. pub(crate) const EXECUTION: &str = "runner-test"; +/// A registry holding the natives and each of `packages`, which a suite +/// builds its Host over before it installs them by name. +pub(crate) fn registry(packages: &[Package]) -> PackageRegistry { + let mut registry = PackageRegistry::new(); + for package in packages { + registry.add(*package).unwrap(); + } + registry +} + /// A section body that calls the fixture wait tool by its full id and /// returns its answer: the section parks until the test's tool performer /// answers. diff --git a/crates/harness/examples/run-prompt.rs b/crates/harness/examples/run-prompt.rs index a4038b0e7..1c81f0182 100644 --- a/crates/harness/examples/run-prompt.rs +++ b/crates/harness/examples/run-prompt.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::time::Duration; -use harness::plugin::{HostContext, HostServices}; +use harness::plugin::{HostContext, HostServices, PackageRegistry}; use harness::record::{MemoryRecorder, RunOutcome}; use harness::vfs::VfsRef; use harness::{BoxFuture, Harness, HostSnapshot, InferenceBroker, RunRequest, Timer}; @@ -69,7 +69,10 @@ async fn main() -> Result<(), Box> { recorder.clone(), Arc::new(Offline), Arc::new(Clock), - Arc::new(HostContext::new(HostServices::new())), + Arc::new(HostContext::new( + PackageRegistry::new(), + HostServices::new(), + )), HostServices::new(), ); diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index 5d8ea3431..38b91f324 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -1,11 +1,13 @@ //! The Harness, which runs one prompt for a Host and records the run. //! -//! A Host installs its Plugins once, in a [`plugin::HostContext`], and -//! builds one Harness for each run. [`Harness::new`] takes the Host's -//! recorder, inference broker, timer, installed Plugins, and the run's own -//! services. [`Harness::run`] runs the prompt to its end and returns its -//! [`RunReport`]. While the run goes on, the Host steers it through the -//! [`RunControl`] it took from `Harness::control` before the run started. +//! A Host adds its Plugin packages to a [`plugin::PackageRegistry`], +//! builds one [`plugin::HostContext`] over it, and installs Plugins by +//! package name. It installs them once and builds one Harness for each +//! run. [`Harness::new`] takes the Host's recorder, inference broker, +//! timer, installed Plugins, and the run's own services. [`Harness::run`] +//! runs the prompt to its end and returns its [`RunReport`]. While the run +//! goes on, the Host steers it through the [`RunControl`] it took from +//! `Harness::control` before the run started. //! //! The crate's `run-prompt` example runs one prompt from start to end. @@ -23,12 +25,15 @@ pub use harness_runner::performers::InferenceBroker; pub use harness_runner::performers::Timer; pub mod plugin { - //! The context a Host installs its Plugins in, the services maps a - //! Host fills, and the names Plugins are installed under. The Plugin - //! crates themselves are the Host's own dependencies. + //! The registry a Host lists and installs packages from, the context + //! a Host installs its Plugins in, the services maps a Host fills, and + //! the names Plugins are installed under. The Plugin crates themselves + //! are the Host's own dependencies. pub use harness_runner::HostContext; pub use harness_runner::InstallError; + pub use harness_runner::PackageRegistry; + pub use harness_runner::PluginKind; pub use promptforge_plugin::HostServices; pub use promptforge_plugin::PluginId; } diff --git a/crates/harness/tests/suite/asker.rs b/crates/harness/tests/suite/asker.rs index a4754c671..cd7b94918 100644 --- a/crates/harness/tests/suite/asker.rs +++ b/crates/harness/tests/suite/asker.rs @@ -5,7 +5,7 @@ use std::sync::Arc; -use harness::plugin::{HostContext, HostServices}; +use harness::plugin::{HostContext, HostServices, PackageRegistry}; use promptforge_plugin::{ Package, Plugin, PluginFuture, PluginId, ServiceId, ServiceKey, ToolContext, ToolDescriptor, ToolError, ToolId, ToolOutput, @@ -40,8 +40,12 @@ pub(crate) trait AskBroker: Send + Sync { /// A Host with the fixture installed as `user-input`. pub(crate) fn asker_host() -> Arc { - let mut host = HostContext::new(HostServices::new()); - if let Err(error) = host.install(ASKER, None, Value::Null) { + let mut packages = PackageRegistry::new(); + if let Err(error) = packages.add(ASKER) { + panic!("a new registry takes the fixture: {error}"); + } + let mut host = HostContext::new(packages, HostServices::new()); + if let Err(error) = host.install(ASKER.name, None, Value::Null) { panic!("an empty Host installs the fixture: {error}"); } Arc::new(host) diff --git a/crates/harness/tests/suite/broker.rs b/crates/harness/tests/suite/broker.rs index e5c92aaa2..b4303983d 100644 --- a/crates/harness/tests/suite/broker.rs +++ b/crates/harness/tests/suite/broker.rs @@ -5,7 +5,7 @@ use std::num::NonZeroU32; use std::sync::{Arc, Mutex, PoisonError}; -use harness::plugin::{HostContext, HostServices}; +use harness::plugin::{HostContext, HostServices, PackageRegistry}; use harness::record::{MemoryRecorder, RunOutcome}; use harness::vfs::VfsRef; use harness::{BoxFuture, Harness, HostSnapshot, InferenceBroker, RunRequest}; @@ -88,7 +88,10 @@ async fn a_sections_round_reaches_the_broker_as_chat_and_a_nested_infer_round_as Arc::new(MemoryRecorder::new()), broker.clone(), Arc::new(Clock), - Arc::new(HostContext::new(HostServices::new())), + Arc::new(HostContext::new( + PackageRegistry::new(), + HostServices::new(), + )), HostServices::new(), ); let report = harness diff --git a/crates/harness/tests/suite/host.rs b/crates/harness/tests/suite/host.rs index 6875dda07..8a008fb09 100644 --- a/crates/harness/tests/suite/host.rs +++ b/crates/harness/tests/suite/host.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::task::{Poll, Waker}; use async_trait::async_trait; -use harness::plugin::{HostContext, HostServices}; +use harness::plugin::{HostContext, HostServices, PackageRegistry}; use harness::record::{MemoryRecorder, RecordKind, RunOutcome}; use harness::vfs::VfsRef; use harness::{BoxFuture, Harness, HostSnapshot, InferenceBroker, RunRequest}; @@ -155,7 +155,10 @@ fn stuck_harness(catalog: ModelCatalog) -> (Harness, Arc) { Arc::new(MemoryRecorder::new()), Arc::new(broker), Arc::new(Clock), - Arc::new(HostContext::new(HostServices::new())), + Arc::new(HostContext::new( + PackageRegistry::new(), + HostServices::new(), + )), HostServices::new(), ); (harness, started) @@ -265,7 +268,10 @@ async fn a_hosts_own_broker_streams_a_section_round_under_the_round_id_the_run_r recorder.clone(), Arc::new(broker), Arc::new(Clock), - Arc::new(HostContext::new(HostServices::new())), + Arc::new(HostContext::new( + PackageRegistry::new(), + HostServices::new(), + )), HostServices::new(), ); let report = harness diff --git a/crates/harness/tests/suite/launch.rs b/crates/harness/tests/suite/launch.rs index 905f42cdd..f29066171 100644 --- a/crates/harness/tests/suite/launch.rs +++ b/crates/harness/tests/suite/launch.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use harness::plugin::{HostContext, HostServices}; +use harness::plugin::{HostContext, HostServices, PackageRegistry}; use harness::record::{MemoryRecorder, RunOutcome}; use harness::vfs::{Origin, VfsError, VfsRef}; use harness::{Harness, HostSnapshot, OutputError, RunRequest}; @@ -26,7 +26,10 @@ fn harness() -> Harness { Arc::new(MemoryRecorder::new()), Arc::new(Offline), Arc::new(Clock), - Arc::new(HostContext::new(HostServices::new())), + Arc::new(HostContext::new( + PackageRegistry::new(), + HostServices::new(), + )), HostServices::new(), ) } diff --git a/crates/promptforge-plugin/src/plugin.rs b/crates/promptforge-plugin/src/plugin.rs index aea4866b7..7ec614c10 100644 --- a/crates/promptforge-plugin/src/plugin.rs +++ b/crates/promptforge-plugin/src/plugin.rs @@ -99,9 +99,9 @@ pub trait Plugin: Send + Sync { /// A Plugin crate's label, exported as a constant such as /// `plugin_web::PACKAGE`. /// -/// It is fixed data the Host reads before anything is built. Passing it to -/// the Harness's install is also what links the Plugin crate into the -/// Host. A Plugin crate builds it with [`Package::new`], adding a +/// It is fixed data the Host reads before anything is built. Adding it to +/// the Harness's package registry is also what links the Plugin crate into +/// the Host. A Plugin crate builds it with [`Package::new`], adding a /// [`Package::description`], and [`Package::prelude`] and /// [`Package::needs`] only where it has them: /// diff --git a/crates/workshop/agents/src/input-tests.rs b/crates/workshop/agents/src/input-tests.rs index 03c726594..094f22c0e 100644 --- a/crates/workshop/agents/src/input-tests.rs +++ b/crates/workshop/agents/src/input-tests.rs @@ -4,7 +4,7 @@ use super::*; use std::sync::Arc; -use harness::plugin::{HostContext, HostServices}; +use harness::plugin::{HostContext, HostServices, PackageRegistry}; use harness::record::{MemoryRecorder, RunOutcome}; use harness::vfs::VfsRef; use harness::{BoxFuture, Harness, HostSnapshot, InferenceBroker, RunRequest}; @@ -267,9 +267,17 @@ async fn an_ask_is_answered_when_the_registry_receives_the_text() { let source = "---\nname: ask\ndescription: asks the operator\npromptforge: 0\n\ plugins:\n - user-input\n---\n\n\ # Ask\n\n## Only\n\n```lua\nreturn (input.ask())\n```\n"; - let mut host = HostContext::new(HostServices::new()); - host.install(plugin_user_input::PACKAGE, None, serde_json::Value::Null) - .expect("an empty Host installs the Plugin"); + let mut packages = PackageRegistry::new(); + packages + .add(plugin_user_input::PACKAGE) + .expect("a new registry takes the Plugin"); + let mut host = HostContext::new(packages, HostServices::new()); + host.install( + plugin_user_input::PACKAGE.name, + None, + serde_json::Value::Null, + ) + .expect("an empty Host installs the Plugin"); let (broker, registry, frames) = broker_fixture(); let mut socket = frames.subscribe(); let recorder = Arc::new(MemoryRecorder::new()); diff --git a/crates/workshop/agents/tests/it/support.rs b/crates/workshop/agents/tests/it/support.rs index 8d8743000..62cac0529 100644 --- a/crates/workshop/agents/tests/it/support.rs +++ b/crates/workshop/agents/tests/it/support.rs @@ -11,7 +11,7 @@ use std::sync::Arc; -use harness::plugin::{HostContext, HostServices}; +use harness::plugin::{HostContext, HostServices, PackageRegistry}; use harness::record::MemoryRecorder; use harness::vfs::VfsRef; use harness::{BoxFuture, Harness, HostSnapshot, InferenceBroker, RunRequest}; @@ -60,9 +60,17 @@ pub(crate) fn harness_for( recorder: Arc, source: String, ) -> (Harness, RunRequest) { - let mut host = HostContext::new(HostServices::new()); - host.install(plugin_user_input::PACKAGE, None, serde_json::Value::Null) - .expect("an empty Host installs the Plugin"); + let mut packages = PackageRegistry::new(); + packages + .add(plugin_user_input::PACKAGE) + .expect("a new registry takes the Plugin"); + let mut host = HostContext::new(packages, HostServices::new()); + host.install( + plugin_user_input::PACKAGE.name, + None, + serde_json::Value::Null, + ) + .expect("an empty Host installs the Plugin"); let harness = Harness::new( conversation.recorder(recorder), Arc::new(Offline), diff --git a/crates/workshop/server/src/agents.rs b/crates/workshop/server/src/agents.rs index 1fab9a624..7e180c796 100644 --- a/crates/workshop/server/src/agents.rs +++ b/crates/workshop/server/src/agents.rs @@ -10,8 +10,8 @@ //! recorder tee on a handle of the run log's [`TursoRecorder`] that names //! the launched agent, a per-run broker over //! the server's inference broker ([`broker`]), the tokio timer, the -//! server's [`HostContext`] with web, user-input, and the MCP servers of -//! the configured `mcp.json` installed, and the +//! server's [`HostContext`] with web, user-input, the native subagents, +//! and the MCP servers of the configured `mcp.json` installed, and the //! conversation's own services, which hold its input broker. The Harness reaches the gateway //! only through that broker, which follows the live gateway binding and //! sends each round to the dropdown's current pick. The @@ -43,7 +43,7 @@ use std::io; use std::path::{Path, PathBuf}; use std::sync::Arc; -use harness::plugin::{HostContext, HostServices}; +use harness::plugin::{HostContext, HostServices, PackageRegistry}; use harness::record::RunId; use harness::vfs::VfsRef; use harness::{Harness, RunRequest}; @@ -138,23 +138,34 @@ fn read_mcp_servers(setting: Option<&Path>) -> Vec<(String, Value)> { } /// A Host over the Host-wide `services` with the Plugins agents may -/// declare installed: `web`, and `user-input`, so they can ask the -/// operator, under their default names, then each of `servers` as an MCP -/// Plugin under its name lowercased in ASCII, with its entry as the -/// configuration. A name that does not parse as a Plugin id, or that is -/// taken or differs from an installed name only by punctuation, is skipped -/// with a warning, and the other servers still install. The ask tool's id -/// is `/ask` under the name the user-input install returned, and -/// `None` only if that install was refused. +/// declare installed: `web`, `user-input`, so they can ask the operator, +/// and the native `subagents`, under their default names, then each of +/// `servers` as an MCP Plugin under its name lowercased in ASCII, with its +/// entry as the configuration. A name that does not parse as a Plugin id, +/// or that is taken, reserved for the native, or differs from an installed +/// name only by punctuation, is skipped with a warning, and the other +/// servers still install. The ask tool's id is `/ask` under the name +/// the user-input install returned, and `None` only if that install was +/// refused. fn with_plugins( services: HostServices, servers: Vec<(String, Value)>, ) -> (HostContext, Option) { - let mut host = HostContext::new(services); - if let Err(error) = host.install(plugin_web::PACKAGE, None, Value::Null) { + let mut packages = PackageRegistry::new(); + for package in [ + plugin_web::PACKAGE, + plugin_user_input::PACKAGE, + plugin_mcp::PACKAGE, + ] { + if let Err(error) = packages.add(package) { + tracing::warn!(%error, "the Plugin package was not added"); + } + } + let mut host = HostContext::new(packages, services); + if let Err(error) = host.install(plugin_web::PACKAGE.name, None, Value::Null) { tracing::warn!(%error, "the web Plugin was not installed"); } - let ask = match host.install(plugin_user_input::PACKAGE, None, Value::Null) { + let ask = match host.install(plugin_user_input::PACKAGE.name, None, Value::Null) { Ok(name) => ToolId::parse(&format!("{name}/{}", plugin_user_input::ASK)) .inspect_err(|error| tracing::warn!(%error, "the ask tool's id does not parse")) .ok(), @@ -163,6 +174,9 @@ fn with_plugins( None } }; + if let Err(error) = host.install("promptforge/subagents", None, Value::Null) { + tracing::warn!(%error, "the subagents Plugin was not installed"); + } for (server, entry) in servers { let id = match PluginId::parse(&server.to_ascii_lowercase()) { Ok(id) => id, @@ -171,7 +185,7 @@ fn with_plugins( continue; } }; - if let Err(error) = host.install(plugin_mcp::PACKAGE, Some(id), entry) { + if let Err(error) = host.install(plugin_mcp::PACKAGE.name, Some(id), entry) { tracing::warn!(%server, %error, "the MCP server was not installed"); } } diff --git a/crates/workshop/server/src/agents/tests.rs b/crates/workshop/server/src/agents/tests.rs index 5bd8c8f61..0c3a0f6cf 100644 --- a/crates/workshop/server/src/agents/tests.rs +++ b/crates/workshop/server/src/agents/tests.rs @@ -1,5 +1,6 @@ //! The agent conversations' Harness setup: a prompt declaring -//! `web` prepares over the server's installed Plugins, a Host without the +//! `web` or the native `subagents` prepares over the server's installed +//! Plugins, a Host without the //! search provider installs web as unavailable and refuses the prompt //! naming that service, outside a runtime the services leave the runtime //! out, and the ask id names the ask tool under the name user-input was @@ -136,6 +137,18 @@ async fn a_prompt_declaring_web_prepares_on_the_servers_installed_plugins() { ); } +#[tokio::test] +async fn a_prompt_declaring_subagents_prepares_on_the_servers_installed_plugins() { + let (host, _ask) = host_context(&Registry::new(), None); + assert_eq!( + run_prompt(&Arc::new(host), requiring("subagents")).await, + Some(RunOutcome::Completed { + final_text: "browsed".to_owned(), + }), + "the server installs the native subagents" + ); +} + #[tokio::test] async fn a_host_without_the_search_provider_refuses_a_prompt_requiring_web() { let mut runtime_only = HostServices::new(); diff --git a/vibe/2026-10-11-1-package-registry.md b/vibe/2026-10-11-1-package-registry.md index 4e08efa2d..6961041e8 100644 --- a/vibe/2026-10-11-1-package-registry.md +++ b/vibe/2026-10-11-1-package-registry.md @@ -462,7 +462,7 @@ Paths are relative to the `promptforge` repository. Line numbers are as of `e7c3 -### Step 2: Install Plugins by name from a package registry +### Step 2: Install Plugins by name from a package registry [completed] - Component: `none` - Work items: `registry`, `install`, `callers`, `docs`, and `verify`. They land together, because changing `HostContext::new` and `install` breaks every caller at once. The Technical Design's Step 2 list holds the details for each artifact below. @@ -499,7 +499,12 @@ Paths are relative to the `promptforge` repository. Line numbers are as of `e7c3 - `crates/promptforge-plugin/src/plugin.rs`: the `Package` doc's linking sentence becomes "Adding it to the Harness's package registry is also what links the Plugin crate into the Host." - Tests, named as in the Testing Plan: - `packages-tests.rs`: `natives_offer_no_tools_prelude_or_needs`, looping over `NATIVE` - - new `crates/harness-internal/runner/tests/it/packages.rs`, declared with `mod packages;` beside `mod prepare;` in `main.rs`: the three registry tests and the three install tests, built from `echo_package` + - new `crates/harness-internal/runner/tests/it/packages.rs`, declared with `mod packages;` beside `mod prepare;` in `main.rs`: the three registry tests and the three install tests, built from `echo_package`, which together cover every case below: + - `PackageRegistry::new().list()` yields exactly `promptforge/subagents` as `PluginKind::Native` with a non-empty description. After adding `acme/web`, it yields `acme/web` (`PluginKind::Linked`) before `promptforge/subagents`. A package added with needs lists those needs. + - `add`: a second `acme/web` and a package named `promptforge/subagents` are both `DuplicatePackage`. The six names `web`, `acme/web/extra`, `/web`, `acme/`, `Acme/web`, and `acme/Web` are each `InvalidPackage`. + - `install`: `acme/missing` and `promptforge/tasks` are both `UnknownPackage`; the second pins that `tasks` is not a package. + - `install`: `acme/subagents` with no name, `acme/web` named `subagents`, and `promptforge/subagents` named `other` are each `ReservedName`, and the test checks the `ReservedName` message text once. + - `install`: `promptforge/subagents` with no name returns `subagents`, and a second install is `NameTaken`. - `host-tests.rs`: `a_declared_native_is_met_when_installed_and_missing_when_not` and `an_installed_native_adds_no_catalog_tool_and_no_prelude` - `crates/workshop/server/src/agents/tests.rs`: `a_prompt_declaring_subagents_prepares_on_the_servers_installed_plugins` - Every existing install, snapshot, preparation, and `tasks` test passes with only its call site changed. From 6ee1885b7ae17838560527539750f6338563d157 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 04:30:27 -0700 Subject: [PATCH 13/34] Close plan: package-registry Plan: vibe/2026-10-11-1-package-registry.md --- vibe/2026-10-11-1-package-registry.md | 14 +++++++------- vibe/ACTIVE | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/2026-10-11-1-package-registry.md b/vibe/2026-10-11-1-package-registry.md index 6961041e8..88d2a926c 100644 --- a/vibe/2026-10-11-1-package-registry.md +++ b/vibe/2026-10-11-1-package-registry.md @@ -4,25 +4,25 @@ overview: Add a string-keyed PackageRegistry to the Harness so a Host lists its todos: - id: descriptions content: "Step 1: Package::description field and builder in promptforge-plugin; descriptions on the web, user-input, and mcp packages" - status: pending + status: completed - id: registry content: "Step 2: new runner packages.rs: PackageRegistry (new and Default hold the native, add, list yielding (&Package, PluginKind)), PluginKind { Native, Linked }, and the private native Package constant for promptforge/subagents with its tool-less Plugin; two guards: a doc comment on the stub saying built-in behavior lives in the Engine and the stub never gains tools, and a packages-tests.rs test that every native builds a Plugin with no tools, no prelude, and no needs" - status: pending + status: completed - id: install content: "Step 2: HostContext::new(packages, services) takes the registry by value; install takes a package name; InstallError gains DuplicatePackage, UnknownPackage, and ReservedName, and InvalidPackage moves from install to add" - status: pending + status: completed - id: callers content: "Step 2: runner and harness facade re-export PackageRegistry and PluginKind; Workshop with_plugins installs web, user-input, subagents, and each MCP server by name; run-prompt example; every test call site through a per-crate registry helper" - status: pending + status: completed - id: docs content: "Step 2: AGENTS.md Plugin definition with three kinds, subagents the one native; host.rs, harness lib.rs, and Package docs say Hosts add packages to a registry and install by name" - status: pending + status: completed - id: verify content: Both nextest runs, both clippy runs with warnings denied, fmt, docs and facade docs builds, xtask api --check, build-xtask tests, and the docs-claims test - status: pending + status: completed - id: close content: "Step 3: mark the vibe/ copy completed and commit Close plan: package-registry with the Plan trailer; nothing is pushed" - status: pending + status: completed isProject: false --- diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 7b1126a8c..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-10-11-1-package-registry.md From 7fb07696b636f56e91c23c08475eabb95ea4e84f Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 07:59:07 -0700 Subject: [PATCH 14/34] Correct the native subagents and Plugin package docs The subagents package that every registry holds was described as letting the model start, wait for, and cancel subagents, but it offers no tools yet. Its description, the documentation of its kind, and the repository's Plugin definition now say it only reserves the subagents name for tools the Engine will implement later. The label docs of the three Plugin crates now say a Host adds the label to its package registry, which matches how a Host installs Plugins by package name. - `NATIVE` now describes `promptforge/subagents` as reserving the subagents name for the Engine's subagent tools, which are not built yet. `PackageRegistry::list` returns this package with every other, so the corrected line is what a Host listing a registry gets. - `PluginKind` documents `Native` as a kind whose behavior will live in the Engine, and says today's one native, `promptforge/subagents`, has none yet. - `AGENTS.md` changes only the Engine-native clause of the Plugin definition, which now says `promptforge/subagents` is reserved for subagent tools the Engine will implement under the fixed name `subagents` and is not built yet. - `crates/plugin-web/src/web.rs`, `crates/plugin-user-input/src/lib.rs`, and `crates/plugin-mcp/src/lib.rs` each document the Plugin's label as something a Host adds to its package registry, instead of something it passes to its install. - `NATIVE` keeps its doc comment word for word, including the rules every native must follow and the line that a native's behavior and tools live in the Engine. The change adds no tests. Plan: vibe/2026-10-11-2-native-docs-debt-removal.md --- AGENTS.md | 2 +- .../harness-internal/runner/src/packages.rs | 8 +- crates/plugin-mcp/src/lib.rs | 2 +- crates/plugin-user-input/src/lib.rs | 2 +- crates/plugin-web/src/web.rs | 2 +- vibe/2026-10-11-2-native-docs-debt-removal.md | 489 ++++++++++++++++++ vibe/ACTIVE | 1 + 7 files changed, 499 insertions(+), 7 deletions(-) create mode 100644 vibe/2026-10-11-2-native-docs-debt-removal.md create mode 100644 vibe/ACTIVE diff --git a/AGENTS.md b/AGENTS.md index 754d0d2cf..815d32ea7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ Four words have exactly one meaning each, everywhere in this repository: code co - **Engine**: the `promptforge` and `promptforge-*` crates. The Engine parses a prompt and steps a run. Whenever a run needs a model reply, a tool result, a timer, or a file, the Engine emits an effect and waits for its caller to answer it. In production the caller is the Harness. - **Harness**: the `harness` and `harness-*` crates. The Harness steps the Engine, performs every effect, returns each answer, and records every run through the recorder the Host supplies. Production code runs prompts only through the Harness. - **Host**: an application that runs prompts through the Harness, such as Workshop or Papergate. The Host makes every policy decision. It runs prompts only through the Harness; it may also use the Engine's parser and types to read prompts and show events. -- **Plugin**: a package of tools a Host adds to its package registry and installs into a `HostContext` by package name, such as `promptforge/web`, under a local name the Host chooses, such as `web`. Every tool it offers sits under that name (`web/fetch`). A Plugin is linked (a `plugin-*` crate implementing `promptforge-plugin`), library (loaded from a dynamic library; not built yet), or Engine-native (`promptforge/subagents`, implemented in the Engine under the fixed name `subagents`, which no other Plugin may take). Linked and library Plugins use only the `promptforge-plugin` API. Every run receives every usable Plugin's tools: those of each installed Plugin that built and whose per-run services the run has. Declaring a Plugin under `plugins:` in a prompt's frontmatter installs its Lua prelude and makes it required, and the prompt's Lua decides what the model sees. A Plugin has no other name: never capability, pack, or addon. +- **Plugin**: a package of tools a Host adds to its package registry and installs into a `HostContext` by package name, such as `promptforge/web`, under a local name the Host chooses, such as `web`. Every tool it offers sits under that name (`web/fetch`). A Plugin is linked (a `plugin-*` crate implementing `promptforge-plugin`), library (loaded from a dynamic library; not built yet), or Engine-native (`promptforge/subagents`, reserved for subagent tools the Engine will implement under the fixed name `subagents`, which no other Plugin may take; not built yet). Linked and library Plugins use only the `promptforge-plugin` API. Every run receives every usable Plugin's tools: those of each installed Plugin that built and whose per-run services the run has. Declaring a Plugin under `plugins:` in a prompt's frontmatter installs its Lua prelude and makes it required, and the prompt's Lua decides what the model sees. A Plugin has no other name: never capability, pack, or addon. ### Using the terms diff --git a/crates/harness-internal/runner/src/packages.rs b/crates/harness-internal/runner/src/packages.rs index 80d9424b1..aa05fd16c 100644 --- a/crates/harness-internal/runner/src/packages.rs +++ b/crates/harness-internal/runner/src/packages.rs @@ -34,7 +34,8 @@ pub struct PackageRegistry { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum PluginKind { - /// Built into every registry; what it does lives in the Engine. + /// Built into every registry; its behavior will live in the Engine, and + /// today's one native, `promptforge/subagents`, has none yet. Native, /// Added by the Host with [`PackageRegistry::add`]. Linked, @@ -114,8 +115,9 @@ impl PackageRegistry { /// like any other Plugin, so a native must never offer tools, carry a /// prelude, or need services. Its `call` is unreachable, because the run /// snapshot routes a call only to a tool in a Plugin's own list. -const NATIVE: [Package; 1] = [Package::new("promptforge/subagents", native) - .description("Lets the model start, wait for, and cancel subagents")]; +const NATIVE: [Package; 1] = [Package::new("promptforge/subagents", native).description( + "Reserves the subagents name for the Engine's subagent tools, which are not built yet", +)]; /// Builds a native's [`Native`] object; see [`NATIVE`]. #[expect( diff --git a/crates/plugin-mcp/src/lib.rs b/crates/plugin-mcp/src/lib.rs index 7eae70aa8..183054e3f 100644 --- a/crates/plugin-mcp/src/lib.rs +++ b/crates/plugin-mcp/src/lib.rs @@ -79,7 +79,7 @@ use tokio::sync::watch; use crate::entry::Vars; use crate::server::{Server, State}; -/// The Plugin's label, which a Host passes to its install. +/// The Plugin's label, which a Host adds to its package registry. /// /// Its name is `promptforge/mcp`. The Host installs it once per server, /// passing the server's `mcp.json` entry as the configuration, and diff --git a/crates/plugin-user-input/src/lib.rs b/crates/plugin-user-input/src/lib.rs index e762ee437..b8327ae56 100644 --- a/crates/plugin-user-input/src/lib.rs +++ b/crates/plugin-user-input/src/lib.rs @@ -76,7 +76,7 @@ mod ask; use promptforge_plugin::{Package, ServiceId, ServiceKey, ToolError}; -/// The Plugin's label, which a Host passes to its install. +/// The Plugin's label, which a Host adds to its package registry. /// /// Its name is `promptforge/user-input`, so a Host that picks no name /// installs it as `user-input`. It has a prelude and needs the run's diff --git a/crates/plugin-web/src/web.rs b/crates/plugin-web/src/web.rs index 940033b87..fb6b5341f 100644 --- a/crates/plugin-web/src/web.rs +++ b/crates/plugin-web/src/web.rs @@ -19,7 +19,7 @@ use crate::fetch::{FetchClient, WebFetch}; use crate::provider::SearchProvider; use crate::search::WebSearch; -/// The Plugin's label, which a Host passes to its install. +/// The Plugin's label, which a Host adds to its package registry. /// /// Its name is `promptforge/web`, so a Host that picks no name installs it /// as `web`. It has no prelude and needs no per-run service; its diff --git a/vibe/2026-10-11-2-native-docs-debt-removal.md b/vibe/2026-10-11-2-native-docs-debt-removal.md new file mode 100644 index 000000000..6c1b90ed8 --- /dev/null +++ b/vibe/2026-10-11-2-native-docs-debt-removal.md @@ -0,0 +1,489 @@ +--- +name: native docs debt removal +overview: "Remove the debt found in the 13 unpushed promptforge commits, plus the visible-set copying the user added to scope: correct the native subagents text, the PACKAGE docs, and the UnknownPackage advice; keep each prompt's body as one shared read-only string with sections as offsets into it; and read the shared prompt in place instead of copying sections." +todos: + - id: native-text + content: "D1-1: rewrite the NATIVE description, the PluginKind::Native doc, and the AGENTS.md Engine-native clause to say the native reserves the subagents name and offers nothing yet" + status: pending + - id: package-docs + content: "CH-1: change the three PACKAGE doc lines (plugin-web, plugin-user-input, plugin-mcp) to say a Host adds the constant to its package registry" + status: pending + - id: unknown-advice + content: "D1-10: change the UnknownPackage error text in host.rs to say add the package before building the HostContext" + status: pending + - id: visible-in-place + content: "D1-17 part 1: replace the copied Vec
visible sets with a borrowed Visible view; reads, jump, call, and fanout look sections up in the shared Arc; move SlicePath to walk_target.rs" + status: pending + - id: shared-body + content: "D1-17 part 2: keep the parsed body as one Arc; Section markdown, items, and prose become BodyText offsets into it; names stay owned; H1 prose points into its masked copy; pending prose shares instead of copying" + status: pending + - id: verify + content: "Run the Testing Plan exit checks: both full nextest runs, both clippy runs, fmt, docs builds, xtask api --check, build-xtask, docs-claims, and the rg checks" + status: pending +isProject: false +--- + +# Correct the Plugin docs and stop copying prompt sections + +### Context for a fresh reader + +- Repository: `c:\Users\Vinnie\cursor\promptforge` (git), branch `master`. Every path in this plan is relative to it unless it is absolute. +- Baseline: `HEAD` is `72fc0f5bb` ("Close plan: package-registry"), 13 commits ahead of `origin/master` (`abe218a36`). Nothing is pushed. Line numbers are as of `72fc0f5bb`; search by name if they have moved. +- Terms from `AGENTS.md`: the Engine runs prompts and performs no I/O; the Harness steps the Engine and performs its effects; the Host is the application (Workshop, or a command-line Host); a Plugin is a package of tools a Host installs. +- Terms this plan uses: + - *section*: one Markdown heading (H2 to H6) of a prompt and the text under it, up to the next heading. Sections form a tree: an H2 holds the H3s under it, and so on. The parser's `Section` type holds one, with its `children`. + - *H1 pass*: the prompt's title section, which runs first and is not a `Section`. + - *walk*: the Engine running sections in order, top to bottom. + - *section entry*: the Engine starting a section. That happens for every walk step, every re-entry after a jump, every `call` child, and the first section of every task or fanout arm. Each entry builds a frame with `SectionContext::new`. + - *`jump`, `call`, `fanout`*: Lua globals that name another section by a heading string such as `'## Retry'`. `jump` continues the walk there, `call` runs it and returns its result, and `fanout` runs it once per item. + - *home slice*: the list of sections a section sits in, meaning it and its siblings. + - *visible set*: the sections a running section may address by heading: its home slice minus itself, plus its own direct children. Never its parent, cousins, or grandchildren. + - *`prompt.section(h)` and `prompt.list(h)`*: Lua reads that return the visible section `h`'s own raw markdown, or its bullet items, or nil when no visible section matches. + - *body*: the prompt text after the frontmatter, which the parser rebuilds once with LF line endings and parses. + - *`SlicePath`*: the scheduler's record of where a slice sits in the tree, as the index of each ancestor from the top level down. It resolves against the shared `Arc` on demand. +- Debt IDs (D1-1, CH-1, D1-10, D1-17) come from a debt analysis of the 13 commits. Each is defined with its evidence in the Debt Inventory below, so no other record is needed. +- Running it: execute this plan with the vibe coder (`c:\Users\Vinnie\cursor\tools-public\planning\vibe-coder.md`), whose survey fills the Project Survey section. The plan record goes in `vibe/` as `2026-10-
--.md`, using the next free number for that day; the latest record is `vibe/2026-10-11-1-package-registry.md`. Commit messages follow `c:\Users\Vinnie\cursor\.cursor\rules\commit.mdc` and carry a `Plan: vibe/` trailer. Nothing is pushed. +- The stale user guide in the separate `promptforge-docs` repository is recorded under Deferred and Out of Scope. It is not part of this run. + + + +## Product Requirements + +- Scope and target work: + - Repository `c:\Users\Vinnie\cursor\promptforge`, branch `master`. The analyzed work is the 13 unpushed commits `abe218a36..72fc0f5bb`: the prompt namespace reads plan, its debt removal plan, a `vibe/` tidy-up, and the package registry plan (`vibe/2026-10-11-1-package-registry.md`, commits `9a9a18525` and `92bff15b4`). + - Kept findings D1-1 and CH-1 come from the package registry work. The user chose to fix D1-10 from the rejected candidates. + - The user expanded the scope to D1-17: the Engine copies sections out of the shared parsed prompt every time a section starts and every time it resolves a `jump`, `call`, or fanout target. The pattern predates the target, and `475fc39cb` made each copy larger. +- Why, in the user's words: + - On the copies: "When the prompt is parsed, it should go into a shared string." "Why are we borrowing at all? prompt text is read-only." The parsed prompt never changes while it runs, so no part of the Engine needs its own copy of any section. + - On what a read returns: "when someone asks for an H2, they should not get the child sections." Reads already return only the section's own text, so copying children to answer them is pure waste. + - On the storage: "I like the idea of having one big shared string, and then computing offsets into it, to constructed a hierarchal table of sections, mirroring the markdown structure. That feels right." + - On the native's text: `subagents` is model-facing, unlike `tasks`, which is pure Lua. So the native must eventually behave like a linked Plugin, with installing it controlling what the model gets. Until the overhaul delivers that, its text must not claim it. +- Cleanup goals: + - Every description of the native `promptforge/subagents` package says what it does today: it reserves the `subagents` name and offers nothing yet. + - Every Plugin `PACKAGE` doc describes the install that exists: a Host adds the constant to its package registry and installs by package name. + - The `UnknownPackage` refusal tells a Host author a step they can still take. + - A parsed prompt stores its text once. The parser keeps the prompt body as one shared, read-only string, and every section's own text (its raw markdown, prose blocks, and list items) is a stretch of that string recorded by byte offsets. The sections still form a tree that mirrors the Markdown headings: each H2 holds its H3s, and so on. + - A running prompt reads the shared tree in place. No section start, `prompt.section` or `prompt.list` read, `jump`, `call`, or fanout lookup copies a section. +- Non-goals: + - Making the Host's install of `subagents` control the model's task tools. That belongs to the `tasks` and `subagents` overhaul. + - Any change a prompt author or Host can observe. Heading addressing, read results, nil on a miss, jump, call, and fanout targets, error messages (including the ones that list the visible set), prose templates, list items, parse errors, spans, and line numbers all behave exactly as today. + - Changing how `LuaProgram` stores its source and bytecode. + - Any public API change. The `promptforge` facade exposes neither `Section` nor `Block`, and `crates/promptforge/public-api.txt` stays unchanged. +- Success criteria: + - No text in the repository outside `vibe/` says the native lets the model start, wait for, or cancel subagents, or that the Engine implements it today. + - No doc says a Host passes a `PACKAGE` constant to its install. + - No Engine code builds a `Vec
`. Visible sets are borrowed views over the shared prompt. + - `Section` owns no text except its heading name, and `Block::Prose` owns none. + - Every existing test passes and still asserts the same facts, the new tests pass, and every exit check passes. A test may change shape only where a signature or a field type changed: a call site, or the expression that builds an expected value. + +## Functional Specification + +### Debt Inventory + +- Debt added: + - D1-1 (introduced, `92bff15b4` and `9a9a18525`): the native package's text claims behavior it doesn't have. + - Evidence: `NATIVE` in `crates/harness-internal/runner/src/packages.rs` line 118 has the description "Lets the model start, wait for, and cancel subagents". `Package::description`'s field doc says it is "One line saying what the Plugin offers, for a Host listing the packages it can install", and `PackageRegistry::list()` is public through `harness::plugin`. `AGENTS.md` line 12 says the native is "implemented in the Engine under the fixed name `subagents`", and `PluginKind::Native`'s doc (`packages.rs` line 37) says "what it does lives in the Engine". No Engine crate contains the string `subagent`. + - Code facts: `Native::tools()` is empty, so installing the native adds no tool. The Engine's four task built-ins (`task`, `task_cancel`, `task_status`, `await_tasks`) are offered whenever a section calls `tools.allow_tasks()`, without consulting the Host, so leaving the native out removes nothing. + - Impact: a Host listing packages shows a line that is false both ways. No production code reads it yet (Workshop installs the native unconditionally and keeps no registry handle), so severity is low. + - Reversal cost: low. Text in three places, no interface change. + - Target state: the description, the `AGENTS.md` clause, and the `PluginKind::Native` doc say the native reserves the name for Engine-implemented subagent tools that are not built yet. + - CH-1 (introduced, `92bff15b4`; `9a9a18525` edited the declarations right below without fixing the doc): three `PACKAGE` docs describe the old install. + - Evidence: `crates/plugin-web/src/web.rs` line 22, `crates/plugin-user-input/src/lib.rs` line 79, and `crates/plugin-mcp/src/lib.rs` line 82 each read "The Plugin's label, which a Host passes to its install." `HostContext::install` now takes a package name `&str`, and the constant goes to `PackageRegistry::add`. The package registry plan rewrote the matching sentence in the `Package` doc but missed these three. + - Impact: a Host author following the doc gets a compile error. Low severity. + - Target state: each says a Host adds the constant to its package registry. + - D1-1's doc half and CH-1 share one cause: the doc sweep in `92bff15b4` was incomplete. One doc pass fixes both. +- Cheap fixes: + - D1-10 (selected by the user from the rejected candidates; introduced in `92bff15b4`): the `UnknownPackage` message gives advice a Host cannot follow. + - Evidence: `crates/harness-internal/runner/src/host.rs` line 246 reads "Plugin package `{package}` is not in the registry; add it with PackageRegistry::add before installing it". `HostContext::new(packages, services)` takes the registry by value and exposes no accessor, so by the time `install` refuses a name, no registry is left to call `add` on. + - Target state: the advice says to add the package before building the `HostContext`. +- User-expanded scope (pre-existing, worsened by `475fc39cb`): + - D1-17: the Engine copies sections out of the shared parsed prompt. + - Storage: `Section` (`crates/promptforge-internal/parser/src/prompt.rs` lines 31-45) owns `name`, `markdown`, and `items` as `String`s, owns its `blocks` (prose `String`s and `LuaProgram`s holding source and bytecode), and owns its `children`. Cloning one copies its whole subtree. The parser builds the body once (`split_frontmatter` in `build-frontmatter.rs` lines 215-246) but drops it after parsing, and every heading's content is copied out of it (`build.rs` lines 136-146). + - Copy sites, all non-test Engine code: + - Every section entry. `SectionContext::new` (`crates/promptforge-internal/engine/src/execute/section_context-construct.rs` line 91) calls `visible_sections(siblings, section)`. That runs for every walk step, every re-entry after a jump, every `call` child, and the first section of every task or fanout arm. `home_without` (`walk_target.rs` lines 28-35) clones each sibling with its whole subtree, and `visible_sections` (lines 39-44) clones the section's direct children. The copy moves into the `prompt.section` and `prompt.list` callback whether or not the section ever reads. + - The H1 pass. `SectionContext::new_live_h1` (line 168) copies the whole top-level slice with `.to_vec()`, once per run. + - Every target lookup. `resolve_jump_target` (`walk_target.rs` lines 90-108) builds the same copy for every `jump`, `call`, and fanout target that `Scheduler::resolve_chain_target` resolves (`scheduler/walk.rs` line 303), searches it, and throws it away. + - Every executed prose block. `scheduler/step.rs` line 88 copies the block's prose `String` into the chain's `pending_prose`. + - A read only ever returns the matched section's own markdown and list items (`read_visible`, `walk_target.rs` lines 59-66). Children are never part of a read result, so copying them buys nothing. + - History: the same copy backed `list_from_section` at the baseline and goes back at least to 2026-09-30. `475fc39cb` added `markdown` to every `Section`, so each copy grew. The prompt reads debt removal plan deferred it once. + - Impact: the copying grows with sections entered times document size. It has no measured cost on today's prompts of a few KB, but it is pure waste, it grows with prompt size and fanout width, and it contradicts the shared, read-only parsed prompt the run already holds as `Arc`. + - Reversal cost: low. `Section` and `Block` are internal to the Engine crates. The facade doesn't expose them, and no crate outside `crates/promptforge-internal` reads them. + - Target state: the goals above. +- Exposed pre-existing debt: none beyond D1-17. +- Rejected candidates: 20. + - 14 residual-but-acceptable: intended contracts already bounded by `#[non_exhaustive]`, the compiler, or an existing test. They cover string-keyed install, the three new `InstallError` variants, Workshop's log-and-continue, registry ownership, the call-site fan-out, the silent native skip in `Default`, the reserved MCP server name, the `Package::description` surface, the `prompt` namespace and nil on a miss, string heading addresses, the test stub fan-out, the ambiguity branch's coverage, the `vibe/` status flip, and month-filed plans. + - 2 weak or speculative: exact-equality reservation versus normalized twins (no present name collides), and doc drift across corrective commits as a recurring cause. + - 4 false: the parser helpers as hard-to-reverse, `build_sections` as oversized (37 lines at the endpoint), `prompt` as an unprotected global, and `Native::call` as removable dead code. + + + + +## Technical Design + +### Doc and message fixes (D1-1, CH-1, D1-10) + +- No interface change. Every change here is a string literal or doc text. +- `crates/harness-internal/runner/src/packages.rs`: + - `NATIVE`'s `.description(...)` (line 118) states present behavior. Suggested: "Reserves the subagents name for the Engine's subagent tools, which are not built yet". + - `PluginKind::Native`'s doc (line 37). Suggested: "Built into every registry; its behavior will live in the Engine, and today's one native, `promptforge/subagents`, has none yet." + - Leave the constraint comment on `NATIVE` unchanged. It states rules for future code, not present behavior. +- `AGENTS.md` line 12: replace only the Engine-native clause. Suggested: "or Engine-native (`promptforge/subagents`, reserved for subagent tools the Engine will implement under the fixed name `subagents`, which no other Plugin may take; not built yet)". Keep every other sentence word for word. The wording must pass the docs-claims scan: Plugin capitalized, and never "activated". +- The three `PACKAGE` doc lines become "The Plugin's label, which a Host adds to its package registry." Plugin crates don't depend on `harness`, so the doc uses no intra-doc link to `PackageRegistry`. +- `InstallError::UnknownPackage`'s `#[error(...)]` text in `crates/harness-internal/runner/src/host.rs` (line 246) becomes "Plugin package `{package}` is not in the registry; add it with PackageRegistry::add before building the HostContext". The variant, its field, and its doc comment stay as they are, apart from any doc sentence that repeats the old advice. + +### Read visible sets in place (D1-17, Engine) + +A section's visible set is the sections it may address by heading: its home slice (its siblings) minus itself, plus its own direct children. Today that set is built as an owned copy. It becomes a borrowed view over the shared prompt instead. + +- `crates/promptforge-internal/engine/src/heading_address.rs` (116 lines) gains the view, beside the matching code that reads it: + +```rust +/// A section's visible set, read in place: its home slice minus itself, +/// plus its direct children. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Visible<'a> { + home: &'a [Section], + caller: Option, + children: &'a [Section], +} + +impl<'a> Visible<'a> { + /// A walked section's set: `home` minus `home[caller]`, plus `home[caller]`'s children. + pub(crate) fn of(home: &'a [Section], caller: usize) -> Visible<'a>; + /// A whole slice and nothing else: the H1 pass's top-level set, and synthetic sets in tests. + pub(crate) fn all(slice: &'a [Section]) -> Visible<'a>; + /// The sections in today's order: the home slice in order with the caller skipped, then the children. + pub(crate) fn iter(self) -> impl Iterator; +} +``` + +- `find_visible` (lines 73-90) and `resolve_visible` (lines 100-112) take `Visible<'a>` in place of `&'a [Section]`, with the same matching, ambiguity detection, and error text. `find_visible` walks `visible.iter()` once for the match and the ambiguity check. `resolve_visible`'s not-found message calls `visible.iter()` again to list the visible sections, which works because `Visible` is `Copy`, in the same order the copied `Vec` had. +- Update the module doc and function docs that describe the visible set as a slice, so they describe the view. +- `crates/promptforge-internal/engine/src/execute/walk_target.rs` (108 lines): + - Delete `home_without` and `visible_sections`. + - Move `SlicePath` here from `scheduler.rs` (lines 103-144: the struct, its derives, its doc comment, and its four methods `root`, `child`, `resolve`, and `name_at`). Make the struct and all four methods `pub(super)`; today the methods are private to the scheduler module. Then the scheduler's files and `SectionContext` construction both reach it, because code under `execute/` sees a `pub(super)` item of `execute::walk_target`. The scheduler's files import it from `walk_target` in place of their current `super::SlicePath` paths. + - `read_visible(heading: &str, visible: Visible<'_>) -> Result>`. + - `resolve_jump_target(heading: &str, siblings: &[Section], jumper: usize) -> Result`: it resolves against `Visible::of(siblings, jumper)` and maps the match to `JumpTarget::Child` or `JumpTarget::Sibling` as today, keeping its internal-invariant error. + - `h1_visible_sections` and `section_position` stay. Both already borrow. +- `crates/promptforge-internal/engine/src/execute/section_context-construct.rs` (194 lines): + - `SectionContext::new` takes `home: &SlicePath` and `index: usize` in place of `section: &Section` and `siblings: &[Section]`, so its argument count is unchanged. It finds its section with `home.resolve(ctx.prompt()).get(index)`, and a miss returns `Error::internal("an entered section is absent from its slice")`; the caller resolved the same slice and index just before, so a miss is an internal invariant violation. Update the `#[expect(clippy::too_many_arguments, reason = ...)]` text to name the slice path and index, and the doc comment (lines 35-37) that says the visible set "is built" from `siblings`. + - Its read callback captures `Arc::clone(ctx.prompt_arc())`, a clone of the `SlicePath`, and `index`. On each call it resolves the home slice from the shared prompt and reads through `Visible::of`. The callback bound, `Fn(String) -> Result, Error> + Send + Sync + 'static` (`section_vm.rs` lines 116-122), is unchanged. `Arc`, `SlicePath`, and `usize` are all `Send + Sync + 'static`. + - `new_live_h1`'s callback captures `Arc::clone(ctx.prompt_arc())` and reads through `Visible::all(h1_visible_sections(&prompt))`, replacing the `.to_vec()` at line 168. +- `crates/promptforge-internal/engine/src/execute/scheduler/walk.rs`: `enter_section` (around line 160) passes `&chain.slice` and `chain.index` to `SectionContext::new`. `resolve_chain_target` (line 303) passes `index` to `resolve_jump_target`, and its H1 branch (line 290) resolves against `Visible::all(sections)`. `scheduler/h1.rs` line 148 does the same. +- `SectionText` (`crates/promptforge-internal/lua/src/handles.rs` lines 144-149) stays an owned value. It is the one copy left, made only when a read is called, of the one section asked for, on its way into Lua's own strings. + +### One shared body string (D1-17, parser) + +- New module `crates/promptforge-internal/parser/src/body_text.rs`, declared in `lib.rs`, with its tests in a sibling `body_text-tests.rs`. `BodyText` is re-exported beside `Section` and `Block`: + +```rust +/// A stretch of a prompt's text, read in place from the one shared copy. +#[derive(Clone)] +pub struct BodyText { + text: Arc, + range: Range, +} + +impl BodyText { + /// The text this stretch covers. + pub fn as_str(&self) -> &str; + /// `part` located inside `text` by its byte offset; `None` when `part` is not a subslice of `text`. + pub(crate) fn within(text: &Arc, part: &str) -> Option; + /// A whole owned string, for synthetic sections in test support. + pub(crate) fn from_owned(text: String) -> BodyText; +} +``` + + - `PartialEq` and `Eq` compare the text read, not the pointer or the range, so the derived equality on `Section`, `Block`, and `Prompt` still compares content. + - `Debug` prints exactly what the `String` field printed (delegate to `::fmt` on `as_str()`), so any debug output or `{:?}` text built from sections and blocks is unchanged. + - Add `AsRef`, `PartialEq`, and `PartialEq<&str>`, so tests can compare a `BodyText`, or a slice of them, against string literals. + - `within` finds `part`'s byte offset from the two strings' addresses (`str::as_ptr` with `.addr()`, stable since Rust 1.84; the toolchain is stable 1.99 and the workspace sets no minimum version). It uses no `unsafe`, and checks that `part` lies wholly inside `text`. +- The body: `split_frontmatter` already rebuilds the body as one LF-normalized `String`, and every later offset, span, and line number already counts from it. `parse_inner` (`parse.rs` line 64) turns it into one `Arc` right after the split, and parsing reads that. So spans, line numbers, and parse errors don't change. +- `parse_body` and the section builders receive the shared `&Arc` beside the `&str` they read today, so each builder can make a `BodyText`. +- `Heading.content` (`build.rs` lines 136-146, today `body[start..end].to_string()`) becomes a borrowed `&str` slice of the shared body, so `Heading` gains a lifetime. +- The intermediate values stop copying too: + - `RawBlock::Prose` (from `split_section_blocks` in `fence.rs`) carries a `&str` in place of a `String`. Drop the `.to_string()` after each `pending_prose(...).trim()` at `fence.rs` lines 259, 292, and 305. + - `parse_bullet_items` in `list.rs` yields `&str` items in place of `content.to_string()` (lines 23-29 and 93-136). + - The markdown trim at `build.rs` line 317 keeps the `&str` instead of calling `.to_owned()`. +- Every one of those `&str`s is then a subslice of the shared body. Each becomes a `BodyText` through `BodyText::within`, where `Section` and `Block` are built (`build.rs` lines 171 and 311-318). A `None` there is a parser bug, and it returns `Error::Internal("parser: section text is not a slice of the prompt body")` (the variant at `error.rs` line 68), never a panic, because `expect` is denied. +- Lua fence sources keep going to `LuaProgram::compile(&source, ...)`, which copies them as it does today. Passing a `&str` there in place of a freshly built `String` is fine either way. +- `Section` fields (`prompt.rs` lines 31-45): + - `markdown: BodyText`. `markdown()` still returns `&str`. + - `items: Vec`. `items()` returns `&[BodyText]`, and readers call `as_str()`. + - `name: String`, unchanged: names are decoded text, not stretches of the source (user decision). + - `blocks` and `children` keep their shape. +- `Block::Prose { text: BodyText }`. `Section::prose()` still returns `&str`. `Block::Lua(LuaProgram)` is unchanged. +- H1: `split_h1` always splits a copy of the H1 content, not the body (`fence.rs` lines 53-84). It copies the content into `h1_content` at line 53, blanks the shared Lua fence out of that copy when one exists (line 63), and splits the copy at line 84. So H1 prose can hold blanked stretches the body doesn't. Turn that copy into its own `Arc` before the split (once per prompt), and make H1 `Block::Prose` text through `BodyText::within` against it (line 88). The empty-prose check (lines 104-109) and `description_text` (lines 110-117) read `as_str()`. `title` and `description_text` stay owned `String`s, since they are decoded or joined text built once per prompt. +- Test support: `synthetic_section` and `prose_block` (`test_support.rs` lines 14-29) build their text with `BodyText::from_owned`, so Engine tests keep calling them the same way. +- Engine readers: + - `read_visible` builds `SectionText` from `markdown().to_owned()` and each item's `as_str().to_owned()`. + - `scheduler/step.rs` lines 85-98: `pending_prose` (`chain_record.rs` line 185, today `Option`) becomes `Option` and takes a clone, which increments a shared count instead of copying the prose. The chain record is never saved or serialized (it derives no `Serialize`), so this changes no stored format. `start_lua` (line 150) passes `as_str()` to `install_lazy_prose`. `chain.rs` line 102 and `walk.rs` line 136 already set it to `None` and need no change. + - `tasks.rs` lines 199-212 read `items()` for the list-worker checks. Adapt them to `&[BodyText]`. +- Size ceiling: every Rust file stays under 500 lines (`build_ceiling::check()`). Today `build.rs` has 325, `fence.rs` 327, `prompt.rs` 200, `scheduler.rs` 364, and `walk.rs` 316. `BodyText` gets its own module, and moving `SlicePath` shrinks `scheduler.rs`. + + + + +## Testing Plan + +- Doc and message fixes: no new tests. The facts the new text states are already pinned: + - `natives_offer_no_tools_prelude_or_needs` (`crates/harness-internal/runner/src/packages-tests.rs`) and `an_installed_native_adds_no_catalog_tool_and_no_prelude` (`crates/harness-internal/runner/src/host-tests.rs`) pin that the native offers nothing (D1-1). + - `a_new_registry_holds_the_native_and_lists_every_package_in_name_order` (`crates/harness-internal/runner/tests/it/packages.rs`) asserts the native's description is non-empty, and must still pass (D1-1). + - The compiler's check on `install`'s `&str` parameter already stops the misuse the old docs described (CH-1). + - `install_refuses_a_package_the_registry_does_not_hold` (`crates/harness-internal/runner/tests/it/packages.rs`) asserts the message contains the package name and `PackageRegistry::add`, so it keeps passing with the new wording (D1-10). +- D1-17 changes no behavior, so the existing tests are its regression suite. Each must still assert the same facts. Only call sites and the expressions that build expected values may change shape, for example `Block::Prose { text: BodyText::from_owned(...) }` in parser tests, which can call the crate-private constructor: + - Reads: `prompt_reads_resolve_over_the_visible_set` and `prompt_reads_raise_only_for_mistakes_in_the_call` (`crates/promptforge-internal/engine/src/execute/tests/suite/exec_flow/prompt_reads.rs`), and the fanout collection tests in `exec_flow/fanout_collections.rs` and `exec_flow/fanout_arms.rs`. + - Addressing and ambiguity: `heading_address-tests.rs`, including `resolve_visible_rejects_more_than_one_match`, which moves to `Visible::all`, and `duplicate_top_level_section_names_error_loudly` (`execute/tests/exec_flow.rs`), which passes the jumper's index. + - Jumps, calls, and fanout targets: `execute/tests/scheduler/walk/jumps.rs`, `execute/tests/scheduler/fanout/arms.rs` (including `a_jump_inside_a_fanout_arm_drives_a_child_walk` and `a_jump_from_an_arm_to_a_worker_child_walks_the_child_slice`), and `tasks-chain-end.rs`. + - Parsing: every parser test of markdown, items, prose, thematic breaks, fences, and line mapping, including the CRLF cases in `parser/src/tests/h1_fences.rs` and `section_fences.rs` and the H1 prose around a shared fence in `h1_fences.rs`. +- New unit tests in `crates/promptforge-internal/parser/src/body_text-tests.rs`: + - `a_body_text_reads_back_the_part_it_was_made_from`: `within` on a subslice returns a `BodyText` whose `as_str()` is that subslice, and which equals a plain `&str` with the same text. + - `a_string_outside_the_shared_text_is_refused`: `within` on an unrelated `String` with the same characters returns `None`. + - `equal_text_from_different_stretches_compares_equal`: two `BodyText`s over different ranges, or different shared strings, holding the same characters are equal, so derived equality on sections compares content. +- Compiler checks carry the structural goals, so no shape test is added: `Section` and `Block::Prose` hold `BodyText` fields, and `home_without` and `visible_sections` no longer exist to call. +- Exit checks: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` + - both clippy runs from the Project Survey, with `CARGO_BUILD_WARNINGS=deny` + - `cargo fmt --all --check` + - `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` and `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items`, both with `RUSTDOCFLAGS="-D warnings"` + - `cargo +nightly-2026-09-05 xtask api --check` passes unchanged, since the facade exposes neither `Section` nor `Block` + - `cargo test -p build-xtask` (crate boundaries and the 500-line ceiling) + - `node --test crates/workshop/ui/test/docs-claims.mjs` (D1-1's `AGENTS.md` wording) + - `rg -n "Lets the model start|passes to its install|implemented in the Engine under the fixed name|before installing it" crates AGENTS.md` prints nothing (D1-1, CH-1, D1-10) + - `rg -n "fn visible_sections|fn home_without|Vec
" crates/promptforge-internal/engine/src --glob '!**/*tests*' --glob '!**/tests/**'` prints nothing (D1-17). Today it prints exactly `walk_target.rs` lines 28 and 39. + + + + +## Decision Record + +- User-resolved choices: + - One shared body string, with sections recording offsets into it to form a tree that mirrors the Markdown structure. User: "I like the idea of having one big shared string, and then computing offsets into it, to constructed a hierarchal table of sections, mirroring the markdown structure." + - Read the shared prompt in place, with no copies. User: "Why are we borrowing at all? prompt text is read-only. And when someone asks for an H2, they should not get the child sections." Reads already return only the section's own text, so the copies are pure waste. + - Keep heading names as owned, decoded strings. `## Q&A` is addressed as `Q&A`, exactly as today. The user chose this over raw-text names, which would change how authors address headings. + - Fold D1-17 into this plan rather than giving it a plan of its own, so the doc fixes and the redesign land in one run. The user chose this knowing the doc fixes then wait for the redesign to be built and verified. + - Fix D1-10 now, though both analysis passes rejected it as wording-only. The user agreed when shown that the advice names a step a Host can no longer take. + - Leave the task built-ins ungated until the `tasks` and `subagents` overhaul. User: "We are going to overhaul `tasks` and `subagents` later anyway. so the tools.allow_tasks problem is going to just vanish" and "Fix it later." The reasoning: `tasks` is a language feature, pure Lua, and a prompt that doesn't want it just doesn't use it. `subagents` is model-facing, so it must behave like a linked Plugin, with the Host's install deciding whether runs get its tools. That makes D1-1's present-tense text wrong today and right only after the overhaul. +- Reversible decisions and consequences: + - Each text field holds a `BodyText`: a handle to the shared string plus a byte range. Consequence: one shared pointer per text field. In exchange, `markdown()` and `prose()` keep their signatures, and synthetic test sections stay easy to build. + - `SectionText` stays owned. That is the only copy left: made when a read is called, of the requested section only, on its way into Lua. + - `LuaProgram` is unchanged. It lives in `promptforge-lua`, which does not depend on the parser. It compiles to bytecode once per parse, and only tests and diagnostics read its source. + - H1 prose points into its own masked copy of the H1 content, because the parser blanks the shared fence out of that copy before splitting it. + - `SlicePath` moves to `walk_target.rs`, beside the visible-set code, so the scheduler and frame construction share it. + - Rewrite the native's text in the present tense now, rather than leaving the planned wording until the overhaul. Consequence: the overhaul rewrites these three lines again when the subagent tools land. Accepted, because `list()` is public and the deferred checkbox screen would show the false line. + - No test pins the exact description or doc wording. Pinning text would be a source-shape ratchet with no product contract behind it. + - One doc pass covers D1-1 and CH-1, since they share a cause. + - The `promptforge/tasks` case of D1-10 keeps the same message. It is still true for that name, just unhelpful. + - `SectionContext::new` resolves its section by slice path and index, rather than taking both the section and the path. Consequence: one lookup it already had in hand is repeated, in exchange for an argument list that stays the same length and a single source for which section is entered. + - The section entry's lookup miss is an internal error, not a panic, matching `resolve_jump_target`'s existing internal-invariant error. +- No remediation changes a public interface, persisted or wire format, component ownership, dependency direction, or trust boundary. +- Rejected alternatives: + - Bare byte ranges, with every text accessor taking the prompt or body as an argument. Rejected: every reader and many parser tests change signature, for no gain over a shared handle. + - Heading names as raw-text offsets. Rejected by the user: authors would have to write `jump('## Q&A')`. + - Only one half of D1-17. The shared string alone still builds and discards a list of sections on every section start. In-place reads alone leave the parsed tree storing some text twice. + - `Arc
` children, so clones become cheap. Rejected: lists would still be built and thrown away, and reading in place removes them. + - Writing read results straight into Lua strings. Rejected for now: it changes the `promptforge-lua` callback interface, to save one copy per read call. + - A test asserting that section text points into the shared body. Rejected: it checks implementation shape, and the types already guarantee no owned text. + - Keep the forward-looking native description and record a `Deferred:` note. Rejected: the field's documented meaning is what the Plugin offers, and a note in history does not reach a Host reading `list()`. + - Gate the task built-ins on the install now. Rejected: that is the overhaul's coordinated Engine and Harness change, and the user chose to fix it later. + - Give the native an empty description. Rejected: the integration test asserts a non-empty one so a listing can say what the package is. +- Assumptions and risks: + - `BodyText::within` relies on every derived `&str` being a subslice of the shared string. Making `Heading.content` a borrowed slice establishes that, and a stray copy fails as an internal parse error instead of quietly pointing at the wrong text. + - An error that lists the visible set must keep today's order. `Visible::iter` yields the home slice in order with the caller skipped, then the children, which is the order the copied `Vec` had. + - `Debug` output for sections and blocks must not change. `BodyText`'s `Debug` delegates to the text's own, so a `{:?}` of a section prints what it printed before. + - Moving `SlicePath` touches every scheduler file that names it. The compiler finds each one, and each change is an import path. + - The suggested doc wording may need adjusting to pass the docs-claims scan. The scan decides. + - The analysis did not inspect repositories outside `promptforge`. A follow-up search found no stale install wording in `promptforge-docs`, and only historical hits (old chats, scratch, earlier drafts) in `promptforge-design`. + +### Deferred and Out of Scope + +- Gating the four task built-ins on the native's install. Revisit in the `tasks` and `subagents` overhaul, which also rewrites the native's description, the `AGENTS.md` clause, and the `PluginKind::Native` doc to describe real behavior. +- `LuaProgram` keeping its own copy of each fence's source. Revisit if Lua source memory matters, or when `LuaProgram` is next reworked. +- Writing read results directly into Lua strings, which would remove the last copy per read. Revisit if reads show up in profiles. +- Exporting the native's package name as a constant from `harness::plugin` (D1-2). Revisit when a second Host installs the native by name. +- Normalized comparison in `PackageRegistry::reserves` (D1-7). Revisit when a second native, or a native name containing `-`, `_`, or `.`, is added. +- The user guide in `promptforge-docs` still teaches the removed `list_from_section` global: 35 mentions across nine `src/language/` chapters (most in `14-fanout.md`, `08-jump-and-call.md`, and `03-blocks-and-prose.md`), and no mention of `prompt.section` or `prompt.list`. Its last commit (`44ff253`, 2026-10-09) predates the removal in `7a80775b7`. It is a separate repository, so it gets its own change, not a work item here. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build --locked -p gateway`. + - `crates/gateway/app` is the workspace's only default member, so plain `cargo build` builds the same thing. The Engine and Plugin crates compile under any test, clippy, or docs command that names them. + - The desktop app builds with `cargo workshop`, an alias for `run -p build-workshop --` in `.cargo/config.toml`. + - The `gateway-config-ui` and `workshop-server` build scripts bundle their UIs with esbuild, so workspace-wide commands need Node.js plus one `npm ci --prefix crates/workshop` and one `npm ci --prefix crates/gateway/config-ui/ui` per checkout. Both installs are present on this machine, along with Node.js 24 (CI uses 22), cargo-nextest 0.9.128, stable Rust, and `nightly-2026-09-05`. + - Workshop clippy and tests need the gateway sidecar staged for Tauri's `externalBin`: `cargo build --locked -p gateway --no-default-features`, then `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe`, and `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc` afterward. A sidecar from an earlier build today is staged in `crates/workshop/desktop/binaries/` now. +- Focused test command pattern: `cargo nextest run --locked -p --all-features `, such as `-p promptforge-parser` or `-p promptforge-engine`. Drop `--all-features` for `workshop`, `workshop-server`, and `workshop-server-api`, which CI never builds with all features. +- Component test command pattern: `cargo nextest run --locked -p --all-features`, with the same three exceptions. `cargo test -p build-xtask` runs the boundary and structural checks, and `cargo xtask tidy` prints the same report. +- Full-suite test command: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api`. The first run includes `build-xtask`. + - CI also runs `cargo nextest run --locked -p workshop-workspace --all-features` and `cargo nextest run --locked -p workshop-server --features headless`. + - The wording and rulebook scan: `node --test crates/workshop/ui/test/docs-claims.mjs`. It needs no `npm ci` and runs from any directory. + - The UI suites: `npm test --workspaces --if-present` in `crates/workshop` and `npm test` in `crates/gateway/config-ui/ui`. +- Linter command: + - `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, both with `CARGO_BUILD_WARNINGS=deny` (in PowerShell, `$env:CARGO_BUILD_WARNINGS='deny'`). + - Never run a standalone `cargo check --workspace` beside them. The one exception is `cargo check -p gateway --no-default-features`, the headless build shape clippy does not cover. + - UI type checks: `npm run typecheck --workspaces --if-present` in `crates/workshop` and `npm run typecheck` in `crates/gateway/config-ui/ui`. + - Supply chain, run in CI: `cargo deny check`, `cargo audit`, `cargo hakari verify`, and a `cargo tree` check that `ring` stays out of the gateway's normal closure. +- Formatter check command: `cargo fmt --all --check`, with `style_edition = "2024"` in `rustfmt.toml`. The pre-commit hook runs it. +- Docs command: + - Each with `RUSTDOCFLAGS="-D warnings"` (in PowerShell, `$env:RUSTDOCFLAGS='-D warnings'`): `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api`, `cargo doc -p promptforge --no-deps`, `cargo doc -p harness --no-deps`, `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items`, and `cargo doc --locked --no-deps -p workshop-server --document-private-items`. The facade docs build with default features on purpose. + - The facade surface check: `cargo +nightly-2026-09-05 xtask api --check` against `crates/promptforge/public-api.txt`; `--bless` regenerates the listing. The nightly is pinned in `crates/build-xtask/src/api/toolchain.rs`, and its fixtures run with `cargo +nightly-2026-09-05 nextest run --locked -p build-xtask --run-ignored only`. + - The documentation site: `cargo xtask site [--books-only]`, which needs `PROMPTFORGE_DOCS` naming a `promptforge-docs` checkout and an mdBook binary (`MDBOOK`, default `mdbook`). +- Test placement and naming conventions: + - Unit tests mostly sit in a sibling file wired with `#[cfg(test)] #[path = "-tests.rs"] mod tests;`, such as `crates/promptforge-internal/parser/src/build-tests.rs`. A module whose tests reach three files keeps them in a `tests/` subdirectory beside a `tests.rs` that declares them, one file per topic, as in `crates/promptforge-internal/parser/src/tests/` and `crates/promptforge-internal/engine/src/execute/tests/`. Inline `#[cfg(test)] mod tests { }` blocks are also common, in about 150 files across most families, including `parser/src/list.rs` and `engine/src/execute/bindings.rs`. + - Integration tests compile as one binary per crate: from `tests/it/main.rs` in most crates, and from `tests/suite/main.rs` in `harness` and `promptforge`. Each topic is one module file, and shared helpers go in `support.rs`. `crates/promptforge/tests/prompts/` holds prompt fixtures. + - A few crates keep standalone test files directly in `tests/`: `gateway-stt-engine`, `gateway-stt-backend-whisper`, `gateway-cloud-providers`, and `build-workshop`. `promptforge-engine` has one bench, `benches/models_loop.rs`. + - Test names are snake_case sentences stating the behavior, such as `the_workspace_has_no_doctests`. + - Doctests are banned. `build-xtask`'s `no_doctests` check fails any doc comment holding a code block rustdoc would compile, so examples use `text`, `json`, or `toml` fences. + - Nightly-only `build-xtask` fixtures and the native whisper tests are `#[ignore]`d. `.config/nextest.toml` puts the STT crates in a `heavy` test group. +- Directory map: + - `crates/`: all 52 Cargo packages. The top level holds the public and shared layer: the `promptforge` facade, `promptforge-plugin`, `harness`, `harness-gateway-client`, `gateway-api-types`, `gateway-api-discovery`, the `plugin-*` crates (`plugin-web`, `plugin-user-input`, `plugin-mcp`), `shared-error-source`, `shared-loopback`, the `build-*` tooling (`build-ceiling`, `build-llama-cuda`, `build-ui`, `build-user-guide`, `build-workshop`, `build-xtask`), and `workspace-hack`. + - `crates/promptforge-internal/`: the Engine's private crates `types`, `engine`, `lua`, `parser`, `vfs`, and `model-client`. + - `crates/harness-internal/runner`: the Harness's private per-run runner and effect loop, including the package registry and `HostContext`. + - `crates/gateway/`: the gateway's private crates `app`, `cloud-providers`, `config`, `config-ui`, `local`, `logging`, `progress`, `protocol`, `routing`, and `web-search`, plus the `stt/` subsystem (`api`, `engine`, `backend-whisper`, `whisper-ffi`). + - `crates/workshop/`: Workshop's crates `desktop` (package `workshop`), `server`, `server-api`, `agents`, `gateway`, `menu`, `protocol`, `registry`, `run-log`, `status`, `support`, `user-state`, and `workspace`, plus the npm workspace packages `ui`, `look`, and `platform`. + - `crates/shared-ui`: a TypeScript and CSS package the gateway config UI consumes, not a Rust crate. + - `guide/`: mdBook files for the gateway, language, and Workshop books, the shared page chrome, and the landing page style. The chapters live in the separate `promptforge-docs` checkout. + - `prompts/`: five example prompt files. `images/`: README art. + - `tools/`: Node scripts for staging the gateway sidecar and a live TTS check, each with a `.test.mjs`. + - `vibe/`: plan records, recent ones at the top level and older ones filed by month, plus an ignored `scratch/`. + - `.github/workflows/`: `ci.yml`, whose `ci-green` job is the single gate, plus release, nightly, site, and native-library workflows. + - `.config/`: `nextest.toml` and `hakari.toml`. `.cargo/config.toml`: the `xtask` and `workshop` aliases and the Windows `rust-lld` linker with the static CRT. `.githooks/`: pre-commit (format) and pre-push (wording scan, headless check, clippy, deny), with `core.hooksPath` set to it. `.cursor/rules/`: the Workshop architecture and SPA rules. + - Not tracked: `local/`, `target/`, and `target-msrv/` (ignored), and `cabinet/`, which holds only an empty `_trash/`. +- Component boundaries, which `cargo test -p build-xtask` enforces across normal, dev, and build dependencies: + - The Engine (`promptforge*`) depends on no gateway, Workshop, Harness, or Plugin crate. Outside crates reach it only through `promptforge` and `promptforge-plugin`, the only two crates that may enter `promptforge-internal/`. + - Inside the Engine: `types` and `vfs` depend on no Engine crate, `model-client` on `types`, `lua` on `types`, `vfs`, and `model-client`, `parser` on `types` and `lua`, and `engine` on all five. The facade re-exports all six, and `promptforge-plugin` names only `types` and `vfs`. `crates/promptforge/public-api.txt` pins the facade's surface. + - The Harness (`harness*`) names only `promptforge`, `promptforge-plugin`, and `workspace-hack` outside its family. The exception is `harness-gateway-client`, which depends on `plugin-web` to implement its `SearchProvider`. Outside crates reach the Harness only through `harness` and `harness-gateway-client`, and only `harness` may enter `harness-internal/`. The chain runs `harness-gateway-client` on `harness`, `harness` on `harness-runner`, and `harness-runner` on `promptforge` and `promptforge-plugin`. + - Plugin crates depend only on `promptforge-plugin`, `shared-*` crates, `workspace-hack`, and outside libraries. Their only dependents are `harness-gateway-client`, `workshop-server`, and `workshop-agents`. The native `promptforge/subagents` package is not a Plugin crate: its `NATIVE` entry and `Native` object, which offers no tools, are defined in `crates/harness-internal/runner/src/packages.rs`. + - The gateway (`gateway*`) depends on no Engine, Workshop, Harness, or Plugin crate. Outside crates reach it only through `gateway-api-types` and `gateway-api-discovery`. Inside the family, only `gateway-stt` is visible outside `gateway/stt/`. + - Workshop is a Host. It reaches the gateway only through that public pair, the Engine only through `promptforge` and `promptforge-plugin`, and the Harness only through `harness` and `harness-gateway-client`. Its tiers depend one way: the server on features (`agents`, `workspace`, `user-state`, `run-log`), features on services (`menu`, `gateway`, `status`), and services on vocabulary (`protocol`, `registry`, `support`). The desktop app names only `workshop-server-api`, which names only `workshop-server`. + - `shared-*` crates depend on no product crate. `build-*` crates are tooling and may depend into any container. +- Conventions summary: + - Engine, Harness, Host, and Plugin are capitalized with one meaning each. Engine crates say "the caller" and never mention the Host. `docs-claims.mjs` enforces what a scan can see in every `AGENTS.md`, every `## Invariants` crate doc, and every `.cursor/rules` file. + - 25 crate `lib.rs` files open with a `//!` doc carrying a `## Invariants` section, including every Engine-internal crate, `promptforge-plugin`, each `plugin-*` crate, `harness-runner`, and every Workshop library. + - Every crate's build script calls `build_ceiling::check()`, which fails any Rust file over 500 lines. Splits go into private child modules without widening visibility. + - Clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, and `allow_attributes` deny, so suppressions use `#[expect(..., reason = "...")]`. `missing_docs`, `missing_debug_implementations`, and `unreachable_pub` warn, which is fatal under the gate. `unsafe_code` is denied outside its owned boundaries, rustdoc's broken and private intra-doc links deny, and `clippy.toml` bans process-global installers outside binary entry points. + - Source directories stay flat until a group reaches three files. Below that, kebab siblings such as `foo-bar.rs` are wired with `#[path]`. + - Comments explain only non-obvious constraints, and every workaround cites its upstream issue URL. + - Error messages are written for a model to read: concise, naming what was required and what was found. + - Behavior changes ship with tests in the same change. A Cargo feature gates only a real build constraint, never product shape. + - JSON that reaches a recorder round-trips exactly, with sorted keys and finite numbers. + - Workspace dependencies are declared once in the root `Cargo.toml`, with a comment on every non-obvious pin. Every member inherits `workspace-hack`, which `cargo hakari` manages. + - Commits carry a one-line summary, a prose body, a bullet list of changes, optional `Design:` and `Deferred:` lines, and a `Plan: vibe/` trailer. A plan's closing commit is titled `Close plan: `. + + + + +## Execution Instructions + + + +### Step 1: Correct the native and Plugin package docs [completed] + +- Component: Plugin package docs +- Component placement: first of three. It depends on nothing, touches only `harness-runner`, the three `plugin-*` crates, and `AGENTS.md`, and shares no file with the D1-17 work, so the false Host-facing text is corrected before the larger redesign starts. +- Piece: native and Plugin doc pass (D1-1, CH-1). Built sequentially before the refusal-advice piece. The two pieces are independent; the doc pass goes first because D1-1 and CH-1 share one cause (the incomplete doc sweep in `92bff15b4`) and one pass fixes both. +- Changes: + - `crates/harness-internal/runner/src/packages.rs`: `NATIVE`'s `.description(...)` and the `PluginKind::Native` doc comment state present behavior, using the Technical Design's suggested wording. Leave the constraint comment on `NATIVE` unchanged. + - `AGENTS.md` line 12: replace only the Engine-native clause with the Technical Design's suggested wording, keeping every other sentence word for word. + - The `PACKAGE` doc line in `crates/plugin-web/src/web.rs`, `crates/plugin-user-input/src/lib.rs`, and `crates/plugin-mcp/src/lib.rs` becomes "The Plugin's label, which a Host adds to its package registry." No intra-doc link to `PackageRegistry`. +- Tests: none new. `natives_offer_no_tools_prelude_or_needs`, `an_installed_native_adds_no_catalog_tool_and_no_prelude`, and `a_new_registry_holds_the_native_and_lists_every_package_in_name_order` pass unchanged. +- Verify: `cargo nextest run --locked -p harness-runner -p plugin-web -p plugin-user-input -p plugin-mcp --all-features`; `node --test crates/workshop/ui/test/docs-claims.mjs` (if it rejects the suggested wording, adjust the wording; the scan decides); the first `rg` exit check prints only the `host.rs` "before installing it" line, which Step 2 removes. +- Commit: one commit with the five text edits, message per `commit.mdc`, with the `Plan: vibe/` trailer. + + + + + +### Step 2: Make the UnknownPackage advice followable + +- Component: Plugin package docs +- Piece: install refusal advice (D1-10). Follows the doc pass and shares no file with it. +- Changes: + - `crates/harness-internal/runner/src/host.rs`: `InstallError::UnknownPackage`'s `#[error(...)]` text becomes "Plugin package `{package}` is not in the registry; add it with PackageRegistry::add before building the HostContext". Keep the variant, its field, and its doc comment, apart from any doc sentence that repeats the old advice. Add no special case for `promptforge/tasks`; it gets the same message. +- Tests: none new. `install_refuses_a_package_the_registry_does_not_hold` (`crates/harness-internal/runner/tests/it/packages.rs`) still finds the package name and `PackageRegistry::add` in the message. +- Verify: `cargo nextest run --locked -p harness-runner --all-features packages::`; the first `rg` exit check prints nothing. +- Commit: one commit, message per `commit.mdc`, with the `Plan: vibe/` trailer. + + + + + +### Step 3: Resolve jump, call, and fanout targets in place + +- Component: In-place visible sets +- Component placement: second. It depends on nothing and uses only the existing `Section` API. It precedes the shared-body component because both edit `read_visible`, so the parser's field-type change lands on readers that already borrow. +- Piece: target lookups through a borrowed view. Built sequentially before the section-entry piece, because that piece's read callbacks go through `Visible::of` and the view-taking `read_visible`, which this piece adds. +- Changes: + - `crates/promptforge-internal/engine/src/heading_address.rs`: add `pub(crate) struct Visible<'a>` (`Clone`, `Copy`, `Debug`; fields `home`, `caller: Option`, `children`) with `Visible::of(home, caller)`, `Visible::all(slice)`, and `iter()`, which yields the home slice in order with the caller skipped, then the children. `find_visible` and `resolve_visible` take `Visible<'a>` in place of `&'a [Section]` with the same matching, ambiguity detection, and error text; `resolve_visible`'s not-found message lists `visible.iter()` again. Update the module and function docs to describe the view. + - `crates/promptforge-internal/engine/src/execute/walk_target.rs`: `read_visible(heading: &str, visible: Visible<'_>)`. `resolve_jump_target(heading: &str, siblings: &[Section], jumper: usize)` resolves against `Visible::of(siblings, jumper)`, maps the match to `JumpTarget::Child` or `JumpTarget::Sibling`, and keeps its internal-invariant error. `h1_visible_sections` and `section_position` stay. `home_without` and `visible_sections` stay for one more step, used only by frame construction. + - `crates/promptforge-internal/engine/src/execute/scheduler/walk.rs`: `resolve_chain_target` passes `index` to `resolve_jump_target`, and its H1 branch (line 290) resolves against `Visible::all(sections)`. `scheduler/h1.rs` line 148 does the same. + - `crates/promptforge-internal/engine/src/execute/section_context-construct.rs`: interim only. Both read callbacks keep their owned copy and call `read_visible(&heading, Visible::all(&visible))`; Step 4 replaces them. + - Test call sites: `heading_address-tests.rs` passes `Visible::all(§ions)`, including `resolve_visible_rejects_more_than_one_match`, and `duplicate_top_level_section_names_error_loudly` in `execute/tests/exec_flow.rs` passes the jumper's index `0`. +- Tests: none new; existing tests assert the same facts. Covered by `heading_address-tests.rs`, `execute/tests/scheduler/walk/jumps.rs`, `execute/tests/scheduler/fanout/arms.rs` (including `a_jump_inside_a_fanout_arm_drives_a_child_walk` and `a_jump_from_an_arm_to_a_worker_child_walks_the_child_slice`), `execute/tests/tasks-chain-end.rs`, and `duplicate_top_level_section_names_error_loudly`. +- Verify: `cargo nextest run --locked -p promptforge-engine --all-features`; `cargo clippy -p promptforge-engine --all-targets --all-features` with `CARGO_BUILD_WARNINGS=deny`. +- Commit: one commit, message per `commit.mdc`, with the `Plan: vibe/` trailer. + + + + + +### Step 4: Read prompt sections in place at section entry + +- Component: In-place visible sets +- Piece: section-entry reads in place. Follows the target-lookup piece, whose `Visible` and view-taking `read_visible` it uses. +- Changes: + - Move `SlicePath` (the struct, its derives, its doc comment, and its methods `root`, `child`, `resolve`, and `name_at`) from `crates/promptforge-internal/engine/src/execute/scheduler.rs` into `walk_target.rs`, making the struct and all four methods `pub(super)`. `scheduler/chain.rs`, `chain_record.rs`, `h1.rs`, `test_hooks.rs`, and `walk.rs` import it from `crate::execute::walk_target` in place of `super::SlicePath`, and the `SlicePath` intra-doc link in the `scheduler.rs` module doc still resolves. + - `section_context-construct.rs`: `SectionContext::new` takes `home: &SlicePath` and `index: usize` in place of `section: &Section` and `siblings: &[Section]`, finds its section with `home.resolve(ctx.prompt()).get(index)`, and returns `Error::internal("an entered section is absent from its slice")` on a miss. Its read callback captures `Arc::clone(ctx.prompt_arc())`, a `SlicePath` clone, and `index`, and reads through `Visible::of` over the resolved home slice on each call. `new_live_h1`'s callback captures `Arc::clone(ctx.prompt_arc())` and reads through `Visible::all(h1_visible_sections(&prompt))`, replacing the `.to_vec()` at line 168. The callback bound in `section_vm.rs` is unchanged. Update the `#[expect(clippy::too_many_arguments, reason = ...)]` text to name the slice path and index, and the doc comment (lines 35-37) that says the visible set "is built" from `siblings`. + - `scheduler/walk.rs`: `enter_section` passes `&chain.slice` and the entered index to `SectionContext::new`. + - `walk_target.rs`: delete `home_without` and `visible_sections`. +- Tests: none new. Covered by `prompt_reads_resolve_over_the_visible_set` and `prompt_reads_raise_only_for_mistakes_in_the_call` (`execute/tests/suite/exec_flow/prompt_reads.rs`), the fanout collection tests in `exec_flow/fanout_collections.rs` and `exec_flow/fanout_arms.rs`, and the rest of the Engine suite. +- Verify: `cargo nextest run --locked -p promptforge-engine --all-features`; the second `rg` exit check prints nothing; `cargo clippy -p promptforge-engine --all-targets --all-features` with `CARGO_BUILD_WARNINGS=deny`; `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items` with `RUSTDOCFLAGS="-D warnings"`; `cargo test -p build-xtask` (500-line ceiling). +- Commit: one commit, message per `commit.mdc`, with the `Plan: vibe/` trailer. + + + + + +### Step 5: Borrow parser intermediates from the body + +- Component: Shared prompt body +- Component placement: third and last. It depends on the in-place component because both edit `read_visible`, and its Engine reader changes land on the in-place code. +- Piece: borrowed parser intermediates. Built sequentially before the storage piece, because `BodyText::within` needs every intermediate `&str` to be a subslice of the shared body, and this piece makes them so. +- Changes: + - `crates/promptforge-internal/parser/src/build.rs`: `Heading` gains a lifetime, and `content` becomes a `&str` slice of the body in place of `body[start..end].to_string()` (lines 136-146). The markdown trim at line 317 keeps the `&str` instead of calling `.to_owned()`. + - `fence.rs`: `RawBlock::Prose` carries a `&str` in place of a `String`; drop the `.to_string()` after each `pending_prose(...).trim()` at lines 259, 292, and 305. `split_h1` still splits its own `h1_content` copy, so its prose borrows from that copy. + - `list.rs`: `parse_bullet_items` returns `&str` items borrowed from its input (lines 23-29 and 93-136). + - Interim: `Section` and `Block::Prose` still own `String`s, so the places that build them (`build.rs` lines 171 and 311-318, `fence.rs` line 88) copy once with `to_owned()`; Step 6 replaces those copies with `BodyText`. Lua fence sources go to `LuaProgram::compile` as today. + - Inline tests in `list.rs` change only where an expected value's type changed. +- Tests: none new. Every parser test of markdown, items, prose, thematic breaks, fences, and line mapping, including the CRLF cases in `parser/src/tests/h1_fences.rs` and `section_fences.rs`, asserts the same facts. +- Verify: `cargo nextest run --locked -p promptforge-parser --all-features`; `cargo clippy -p promptforge-parser --all-targets --all-features` with `CARGO_BUILD_WARNINGS=deny`. +- Commit: one commit, message per `commit.mdc`, with the `Plan: vibe/` trailer. + + + + + +### Step 6: Store each prompt's text once as BodyText + +- Component: Shared prompt body +- Piece: shared-body storage and its readers. Follows the borrowed-intermediates piece. The parser field changes and the Engine reader changes share one step because changing the `Section` and `Block::Prose` field types breaks the Engine readers at compile time. +- Changes: + - New `crates/promptforge-internal/parser/src/body_text.rs`, declared in `lib.rs` and re-exported beside `Section` and `Block`: `pub struct BodyText { text: Arc, range: Range }` deriving `Clone`, with `pub fn as_str`, `pub(crate) fn within(text: &Arc, part: &str) -> Option` (offset from `str::as_ptr` with `.addr()`, no `unsafe`, `None` unless `part` lies wholly inside `text`), and `pub(crate) fn from_owned(text: String)`. Gate `from_owned` with `#[cfg(any(test, feature = "test-support"))]`, since its only callers are parser tests and the feature-gated `test_support` module, and an ungated crate-private function is dead code in builds without the feature. Implement `PartialEq` and `Eq` on the text read, `Debug` by delegating to `::fmt` on `as_str()`, `AsRef`, `PartialEq`, and `PartialEq<&str>`. + - `parse.rs`: `parse_inner` turns the LF-normalized body from `split_frontmatter` into one `Arc` right after the split. `parse_body` and the section builders receive that `&Arc` beside the `&str` they read. + - `prompt.rs`: `Section.markdown` becomes `BodyText` (`markdown()` still returns `&str`) and `Section.items` becomes `Vec` (`items()` returns `&[BodyText]`); `name` stays `String`. `Block::Prose { text: BodyText }`, with `prose()` still returning `&str`; `Block::Lua` is unchanged. + - `build.rs` (lines 171 and 311-318) and `fence.rs` replace Step 5's interim `to_owned()` with `BodyText::within`. A `None` returns `Error::Internal("parser: section text is not a slice of the prompt body")`, never a panic. + - `fence.rs` `split_h1`: turn the masked `h1_content` copy into its own `Arc` before the split, and build H1 prose through `within` against it (line 88). The empty-prose check (lines 104-109) and `description_text` (lines 110-117) read `as_str()`; `title` and `description_text` stay `String`s. + - `test_support.rs`: `synthetic_section` and `prose_block` build their text with `BodyText::from_owned` and keep their signatures. + - Engine readers: `read_visible` in `walk_target.rs` builds `SectionText` from `markdown().to_owned()` and each item's `as_str().to_owned()`. The `pending_prose` field in `scheduler/chain_record.rs` becomes `Option`; `scheduler/step.rs` stores a clone of the block's `BodyText`, and `start_lua` passes `as_str()` to `install_lazy_prose`. The `items()` check in `scheduler/tasks.rs` adapts to `&[BodyText]`. + - Parser test expected values change shape only, for example `Block::Prose { text: BodyText::from_owned(...) }`. +- Tests: new `crates/promptforge-internal/parser/src/body_text-tests.rs`, wired with `#[cfg(test)] #[path = "body_text-tests.rs"] mod tests;`, holding `a_body_text_reads_back_the_part_it_was_made_from`, `a_string_outside_the_shared_text_is_refused`, and `equal_text_from_different_stretches_compares_equal`. Existing parser and Engine tests, including the H1 prose around a shared fence in `h1_fences.rs`, assert the same facts. +- Verify: `cargo nextest run --locked -p promptforge-parser -p promptforge-engine -p promptforge-lua --all-features`. Then, as the plan's last step, run every Testing Plan exit check: both full nextest runs, both clippy runs with `CARGO_BUILD_WARNINGS=deny` (staging the gateway sidecar for the Workshop run as the Project Survey describes), `cargo fmt --all --check`, both `cargo doc` runs with `RUSTDOCFLAGS="-D warnings"`, `cargo +nightly-2026-09-05 xtask api --check` passing with `crates/promptforge/public-api.txt` unchanged, `cargo test -p build-xtask`, `node --test crates/workshop/ui/test/docs-claims.mjs`, and both `rg` checks printing nothing. Fix any failure before this commit, in the files that own it. +- Commit: one commit, message per `commit.mdc`, with the `Plan: vibe/` trailer. + + + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..745d9ab75 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-10-11-2-native-docs-debt-removal.md From 070b972173bdfb37317f84fc2894cbd35c32abba Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 08:09:12 -0700 Subject: [PATCH 15/34] Make the UnknownPackage advice followable When an install asks for a package the registry does not hold, the error now tells the Host to add the package before building the host context, instead of before installing it. The host context takes the registry by value when it is built and offers no way back to it, so by the time an install is refused there is no registry left to add to. The old advice named a step a Host could no longer take. - `UnknownPackage` now says to add the package with `PackageRegistry::add` before building the `HostContext`. `HostContext::new` takes the `PackageRegistry` by value and keeps it in a private field, so building the context is the last point where `add` can still be called. - `UnknownPackage` keeps its `package` field and its doc comment unchanged. Only the message text changes. Plan: vibe/2026-10-11-2-native-docs-debt-removal.md --- crates/harness-internal/runner/src/host.rs | 2 +- vibe/2026-10-11-2-native-docs-debt-removal.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/harness-internal/runner/src/host.rs b/crates/harness-internal/runner/src/host.rs index cb20e8a1a..7eed2ec34 100644 --- a/crates/harness-internal/runner/src/host.rs +++ b/crates/harness-internal/runner/src/host.rs @@ -243,7 +243,7 @@ pub enum InstallError { }, /// The registry holds no package by the name the install asked for. #[error( - "Plugin package `{package}` is not in the registry; add it with PackageRegistry::add before installing it" + "Plugin package `{package}` is not in the registry; add it with PackageRegistry::add before building the HostContext" )] UnknownPackage { /// The package name the install asked for. diff --git a/vibe/2026-10-11-2-native-docs-debt-removal.md b/vibe/2026-10-11-2-native-docs-debt-removal.md index 6c1b90ed8..7d8ffe5cc 100644 --- a/vibe/2026-10-11-2-native-docs-debt-removal.md +++ b/vibe/2026-10-11-2-native-docs-debt-removal.md @@ -398,7 +398,7 @@ impl BodyText { -### Step 2: Make the UnknownPackage advice followable +### Step 2: Make the UnknownPackage advice followable [completed] - Component: Plugin package docs - Piece: install refusal advice (D1-10). Follows the doc pass and shares no file with it. From 363cca117f207dad1ebbe1c1b8cd380459fd0bb1 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 08:18:08 -0700 Subject: [PATCH 16/34] Resolve jump, call, and fanout targets in place Jump, call, and fanout targets now resolve against a borrowed view of the caller's visible set instead of an owned copy of it. The view reads the home slice in place, skips the caller, and then reads the caller's direct children, so lookups see the same sections in the same order without cloning them. Heading matching, ambiguity detection, and error text are unchanged. Section read callbacks still build their copy and read it through the same view. - `Visible` holds a home slice, an optional caller index, and the caller's children, all private and borrowed. `Visible::of` derives the children from the caller and debug-asserts that the caller index lies inside the home slice, while `Visible::all` covers a whole slice with no caller and no children. - `resolve_jump_target` takes the jumper's index in its sibling slice instead of a section reference. It excludes the jumper by that index rather than by a level-and-name match. - `find_visible` and `resolve_visible` take the view in place of a slice and keep their matching and error text. The not-found message walks the view a second time, in the same order the copied list had. - `resolve_jump_target` returns an internal error when the jumper index falls outside its sibling slice. - `visible_sections` still clones the visible set for the section read callbacks, which wrap that copy in `Visible::all`. The H1 read callback likewise wraps a cloned top-level slice. - `heading_address-tests.rs` exercises the view only through `Visible::all`, so caller exclusion and child ordering in `Visible::of` have no unit test there. Design: new encapsulated-invariant @ crates/promptforge-internal/engine/src/heading_address.rs::Visible Design: extends pure-function @ crates/promptforge-internal/engine/src/heading_address.rs::find_visible deps: Visible,str Design: extends pure-function @ crates/promptforge-internal/engine/src/heading_address.rs::resolve_visible deps: Visible,str Design: extends pure-function @ crates/promptforge-internal/engine/src/execute/walk_target.rs::read_visible deps: Visible,str Design: extends pure-function @ crates/promptforge-internal/engine/src/execute/walk_target.rs::resolve_jump_target deps: Section,str,usize Deferred: Section read callbacks still read an owned copy of their visible set Plan: vibe/2026-10-11-2-native-docs-debt-removal.md --- .../engine/src/execute/scheduler/h1.rs | 4 +- .../engine/src/execute/scheduler/walk.rs | 7 +- .../src/execute/section_context-construct.rs | 5 +- .../engine/src/execute/tests/exec_flow.rs | 2 +- .../engine/src/execute/walk_target.rs | 30 ++++---- .../engine/src/heading_address-tests.rs | 30 +++++--- .../engine/src/heading_address.rs | 74 +++++++++++++++---- vibe/2026-10-11-2-native-docs-debt-removal.md | 2 +- 8 files changed, 106 insertions(+), 48 deletions(-) diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs b/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs index e74e7de61..0dc10d8a9 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs @@ -15,7 +15,7 @@ use promptforge_types::ids::ChainId; use crate::execute::support::GENERIC_COMPLETION; use crate::execute::walk_target::{h1_visible_sections, section_position}; -use crate::heading_address; +use crate::heading_address::{self, Visible}; use crate::{Error, Result}; use super::{ChainIndex, Counters, Scheduler, SlicePath, prompt_origin}; @@ -145,7 +145,7 @@ impl Scheduler { ) -> Result<()> { let prompt = self.prompt(); let sections = h1_visible_sections(&prompt); - let target = heading_address::resolve_visible(heading, sections)?; + let target = heading_address::resolve_visible(heading, Visible::all(sections))?; let start = section_position(sections, target).ok_or(Error::internal( "a resolved H1 jump target is absent from the top-level slice", ))?; diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs b/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs index e63fe2bbc..77e3a0592 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs @@ -17,7 +17,7 @@ use crate::execute::section_context::SectionContext; use crate::execute::walk_target::{ JumpTarget, h1_visible_sections, resolve_jump_target, section_position, }; -use crate::heading_address; +use crate::heading_address::{self, Visible}; use crate::parser::Block; use crate::{Error, Result}; @@ -287,7 +287,7 @@ impl Scheduler { let chain = &self.chains[id.index()]; if chain.h1 { let sections = h1_visible_sections(prompt); - let target = heading_address::resolve_visible(heading, sections)?; + let target = heading_address::resolve_visible(heading, Visible::all(sections))?; let index = section_position(sections, target).ok_or(Error::internal( "a resolved H1 target is absent from the top-level slice", ))?; @@ -299,8 +299,7 @@ impl Scheduler { } let slice = chain.slice.resolve(prompt); let index = chain.index; - let jumper = &slice[index]; - match resolve_jump_target(heading, slice, jumper)? { + match resolve_jump_target(heading, slice, index)? { JumpTarget::Child(child) => Ok(ChainTarget { slice: chain.slice.child(index), index: child, diff --git a/crates/promptforge-internal/engine/src/execute/section_context-construct.rs b/crates/promptforge-internal/engine/src/execute/section_context-construct.rs index 94cf9db7c..e7c27b68f 100644 --- a/crates/promptforge-internal/engine/src/execute/section_context-construct.rs +++ b/crates/promptforge-internal/engine/src/execute/section_context-construct.rs @@ -16,6 +16,7 @@ use crate::Result; use crate::execute::context::RunState; use crate::execute::section_vm::{VmSeed, setup_section_vm}; use crate::execute::walk_target::{h1_visible_sections, read_visible, visible_sections}; +use crate::heading_address::Visible; use crate::lua::SectionVm; use crate::parser::Section; use promptforge_types::event::lifecycle; @@ -89,7 +90,7 @@ impl SectionContext { // suspending calls (`call`, `fanout`, `models.infer`) are the // yield shims the setup half installs. let visible = visible_sections(siblings, section); - let read = move |heading: String| read_visible(&heading, &visible); + let read = move |heading: String| read_visible(&heading, Visible::all(&visible)); // The setup half of the section lifecycle - Engine injection, Engine // globals, the control surface, the shared replay, and the captured // alias bindings - is shared with the H1 pass; only the seed, the @@ -166,7 +167,7 @@ impl SectionContext { ctx.limits().lua_logs().get(), )?; let visible = h1_visible_sections(ctx.prompt()).to_vec(); - let read = move |heading: String| read_visible(&heading, &visible); + let read = move |heading: String| read_visible(&heading, Visible::all(&visible)); // H1's one privilege: `argv` installs writable, so the repair // pattern can assign it; the executor reads the value back at the // freeze (see the scheduler's H1-to-walk handoff). diff --git a/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs b/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs index 96d15c279..a435321da 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/exec_flow.rs @@ -134,7 +134,7 @@ fn duplicate_top_level_section_names_error_loudly() { synthetic_section("Dup", 2, Vec::new()), synthetic_section("Dup", 2, Vec::new()), ]; - let error = super::super::walk_target::resolve_jump_target("## Dup", §ions, §ions[0]) + let error = super::super::walk_target::resolve_jump_target("## Dup", §ions, 0) .expect_err("two visible sections with one name must be ambiguous"); let rendered = error.to_string(); assert!(rendered.contains("ambiguous"), "error was: {rendered}"); diff --git a/crates/promptforge-internal/engine/src/execute/walk_target.rs b/crates/promptforge-internal/engine/src/execute/walk_target.rs index b5d537ddb..80487ddd2 100644 --- a/crates/promptforge-internal/engine/src/execute/walk_target.rs +++ b/crates/promptforge-internal/engine/src/execute/walk_target.rs @@ -10,7 +10,7 @@ //! `prompt.list` reads resolve through them too, so every surface agrees on //! what a heading may name. -use crate::heading_address; +use crate::heading_address::{self, Visible}; use crate::lua::SectionText; use crate::parser::{Prompt, Section}; use crate::{Error, Result}; @@ -50,13 +50,14 @@ pub(super) fn h1_visible_sections(prompt: &Prompt) -> &[Section] { } /// What a prompt read sees of the section `heading` names in a caller's -/// visible set: its own raw Markdown and its pre-parsed list items, or -/// `None` when no visible section has that heading. +/// visible set, read in place through `visible`: its own raw Markdown and +/// its pre-parsed list items, or `None` when no visible section has that +/// heading. /// /// # Errors /// Returns [`Error::Lua`] when the heading is malformed or matches more /// than one visible section (see [`heading_address::find_visible`]). -pub(super) fn read_visible(heading: &str, visible: &[Section]) -> Result> { +pub(super) fn read_visible(heading: &str, visible: Visible<'_>) -> Result> { Ok( heading_address::find_visible(heading, visible)?.map(|section| SectionText { markdown: section.markdown().to_owned(), @@ -74,10 +75,11 @@ pub(super) enum JumpTarget { Child(usize), } -/// Resolves `heading` against the jumper's visible set (its sibling slice -/// minus itself, plus its direct children) and classifies the target: a -/// direct child of the jumper starts a child-level walk; anything else is a -/// sibling within the jumper's own slice. +/// Resolves `heading` against the visible set of `siblings[jumper]` (the +/// sibling slice minus the jumper, plus its direct children), read in place +/// through [`Visible::of`], and classifies the target: a direct child of +/// the jumper starts a child-level walk; anything else is a sibling within +/// the jumper's own slice. /// /// Resolution is an exact `(level, name)` match (see /// [`heading_address::resolve_visible`]): two visible sections sharing an @@ -90,14 +92,16 @@ pub(super) enum JumpTarget { pub(super) fn resolve_jump_target( heading: &str, siblings: &[Section], - jumper: &Section, + jumper: usize, ) -> Result { - let visible = visible_sections(siblings, jumper); - let target = heading_address::resolve_visible(heading, &visible)?; - if let Some(index) = section_position(jumper.children(), target) { + let jumper_section = siblings.get(jumper).ok_or(Error::internal( + "a jumping section is absent from its sibling slice", + ))?; + let target = heading_address::resolve_visible(heading, Visible::of(siblings, jumper))?; + if let Some(index) = section_position(jumper_section.children(), target) { return Ok(JumpTarget::Child(index)); } - // `target` was resolved out of the visible set built from exactly these + // `target` was resolved out of the visible set read from exactly these // two slices, so a miss here is an internal invariant violation, not a // user-facing Lua error. section_position(siblings, target) diff --git a/crates/promptforge-internal/engine/src/heading_address-tests.rs b/crates/promptforge-internal/engine/src/heading_address-tests.rs index 354c34981..3bf6a1240 100644 --- a/crates/promptforge-internal/engine/src/heading_address-tests.rs +++ b/crates/promptforge-internal/engine/src/heading_address-tests.rs @@ -5,21 +5,23 @@ use super::*; #[test] fn resolve_visible_finds_exact_match() { let sections = vec![sibling("Worker", 3), sibling("Topics", 3)]; - let found = resolve_visible("### Worker", §ions).expect("must resolve"); + let found = resolve_visible("### Worker", Visible::all(§ions)).expect("must resolve"); assert_eq!(found.name(), "Worker"); } #[test] fn resolve_visible_missing_heading_lists_available() { let sections = vec![sibling("Worker", 3)]; - let err = resolve_visible("### Missing", §ions).expect_err("missing heading must error"); + let err = resolve_visible("### Missing", Visible::all(§ions)) + .expect_err("missing heading must error"); assert!(err.to_string().contains("### Worker"), "error was: {err}"); } #[test] fn resolve_visible_bare_name_errors() { let sections = vec![sibling("Worker", 3)]; - let err = resolve_visible("Worker", §ions).expect_err("bare name without ### must error"); + let err = resolve_visible("Worker", Visible::all(§ions)) + .expect_err("bare name without ### must error"); assert!(err.to_string().contains("### markers"), "error was: {err}"); } @@ -34,7 +36,7 @@ fn sibling(name: &str, level: u8) -> Section { #[test] fn resolve_visible_requires_whitespace_after_markers() { let sections = vec![sibling("Worker", 3)]; - let err = resolve_visible("###Worker", §ions) + let err = resolve_visible("###Worker", Visible::all(§ions)) .expect_err("no whitespace after markers must error"); assert!(err.to_string().contains("whitespace"), "error was: {err}"); } @@ -42,7 +44,8 @@ fn resolve_visible_requires_whitespace_after_markers() { #[test] fn resolve_visible_marker_only_heading_errors_as_nameless() { let sections = vec![sibling("Worker", 3)]; - let err = resolve_visible("### ", §ions).expect_err("a marker-only heading must error"); + let err = resolve_visible("### ", Visible::all(§ions)) + .expect_err("a marker-only heading must error"); assert!(err.to_string().contains("has no name"), "error was: {err}"); } @@ -50,18 +53,19 @@ fn resolve_visible_marker_only_heading_errors_as_nameless() { fn resolve_visible_requires_exact_level() { let sections = vec![sibling("Worker", 3)]; // Same name, wrong marker level, must not resolve. - let err = resolve_visible("## Worker", §ions) + let err = resolve_visible("## Worker", Visible::all(§ions)) .expect_err("a level mismatch must not resolve by name alone"); assert!(err.to_string().contains("not found"), "error was: {err}"); // The exact address resolves. - let ok = resolve_visible("### Worker", §ions).expect("exact address resolves"); + let ok = + resolve_visible("### Worker", Visible::all(§ions)).expect("exact address resolves"); assert_eq!(ok.name(), "Worker"); } #[test] fn resolve_visible_rejects_more_than_one_match() { let sections = vec![sibling("Worker", 3), sibling("Worker", 3)]; - let err = resolve_visible("### Worker", §ions) + let err = resolve_visible("### Worker", Visible::all(§ions)) .expect_err("two identical siblings must be rejected as ambiguous"); assert!(err.to_string().contains("ambiguous"), "error was: {err}"); } @@ -69,11 +73,13 @@ fn resolve_visible_rejects_more_than_one_match() { #[test] fn find_visible_returns_none_for_an_address_not_in_the_set() { let sections = vec![sibling("Worker", 3)]; - let missing = find_visible("### Missing", §ions).expect("a missing name is not an error"); + let missing = find_visible("### Missing", Visible::all(§ions)) + .expect("a missing name is not an error"); assert!(missing.is_none(), "a missing name must be None"); - let wrong_level = - find_visible("## Worker", §ions).expect("a level mismatch is not an error"); + let wrong_level = find_visible("## Worker", Visible::all(§ions)) + .expect("a level mismatch is not an error"); assert!(wrong_level.is_none(), "a level mismatch must be None"); - let found = find_visible("### Worker", §ions).expect("exact address resolves"); + let found = + find_visible("### Worker", Visible::all(§ions)).expect("exact address resolves"); assert_eq!(found.map(Section::name), Some("Worker")); } diff --git a/crates/promptforge-internal/engine/src/heading_address.rs b/crates/promptforge-internal/engine/src/heading_address.rs index b1b257c88..2d8a30452 100644 --- a/crates/promptforge-internal/engine/src/heading_address.rs +++ b/crates/promptforge-internal/engine/src/heading_address.rs @@ -11,14 +11,64 @@ //! one task per member - every arm up front, before any join, so no arm is //! ordered after a sibling - and waits on the live set while the scheduler's //! admission limit bounds how many arms run at once. This module defines -//! [`find_visible`], the one heading lookup those surfaces share, and -//! [`resolve_visible`], its adapter for surfaces where a miss is an error; -//! the collection enumeration sits in the `promptforge-lua` crate, beside the -//! VM and the coroutine protocol that consume it. +//! [`Visible`], the view of a caller's visible set read in place over the +//! shared prompt, [`find_visible`], the one heading lookup those surfaces +//! share, and [`resolve_visible`], its adapter for surfaces where a miss is +//! an error; the collection enumeration sits in the `promptforge-lua` crate, +//! beside the VM and the coroutine protocol that consume it. use crate::parser::Section; use crate::{Error, Result}; +/// A section's visible set, read in place: its home slice minus itself, +/// plus its direct children. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Visible<'a> { + home: &'a [Section], + caller: Option, + children: &'a [Section], +} + +impl<'a> Visible<'a> { + /// A walked section's set: `home` minus `home[caller]`, plus + /// `home[caller]`'s children. `caller` must index `home`; an + /// out-of-range one yields all of `home` and no children, the same set + /// as [`Visible::all`]. + pub(crate) fn of(home: &'a [Section], caller: usize) -> Visible<'a> { + debug_assert!( + caller < home.len(), + "a walked section lies inside its home slice" + ); + Visible { + home, + caller: Some(caller), + children: home.get(caller).map_or(&[][..], Section::children), + } + } + + /// A whole slice and nothing else: the H1 pass's top-level set, and + /// synthetic sets in tests. + pub(crate) fn all(slice: &'a [Section]) -> Visible<'a> { + Visible { + home: slice, + caller: None, + children: &[], + } + } + + /// The sections in order: the home slice in order with the caller + /// skipped, then the children. + pub(crate) fn iter(self) -> impl Iterator { + let caller = self.caller; + self.home + .iter() + .enumerate() + .filter(move |&(index, _)| Some(index) != caller) + .map(|(_, section)| section) + .chain(self.children) + } +} + /// Parses a section heading like `"### Name"` into an exact `(level, name)` /// address. /// @@ -56,8 +106,9 @@ fn parse_heading_address(heading: &str) -> Result<(usize, String)> { Ok((level, name.to_owned())) } -/// Looks up a heading string like `"### Name"` in a caller's visible -/// sections, returning the single matching section or `None`. +/// Looks up a heading string like `"### Name"` in a caller's visible set, +/// read in place through `visible`, returning the single matching section +/// or `None`. /// /// The heading is parsed into an exact `(level, name)` address; a section /// matches only when BOTH its level and name are equal. A heading outside the @@ -70,10 +121,7 @@ fn parse_heading_address(heading: &str) -> Result<(usize, String)> { /// Returns [`Error::Lua`] when the heading is malformed (see /// [`parse_heading_address`]) or when more than one visible section matches /// the exact address. -pub(crate) fn find_visible<'a>( - heading: &str, - visible: &'a [Section], -) -> Result> { +pub(crate) fn find_visible<'a>(heading: &str, visible: Visible<'a>) -> Result> { let (level, name) = parse_heading_address(heading)?; let mut matches = visible .iter() @@ -95,9 +143,9 @@ pub(crate) fn find_visible<'a>( /// # Errors /// Returns [`Error::Lua`] for every [`find_visible`] error, and when no /// visible section matches the exact address. The not-found message lists the -/// visible sections and nothing else, so the error channel cannot leak the -/// rest of the document's structure. -pub(crate) fn resolve_visible<'a>(heading: &str, visible: &'a [Section]) -> Result<&'a Section> { +/// view's sections in its order and nothing else, so the error channel +/// cannot leak the rest of the document's structure. +pub(crate) fn resolve_visible<'a>(heading: &str, visible: Visible<'a>) -> Result<&'a Section> { find_visible(heading, visible)?.ok_or_else(|| { let available: Vec = visible .iter() diff --git a/vibe/2026-10-11-2-native-docs-debt-removal.md b/vibe/2026-10-11-2-native-docs-debt-removal.md index 7d8ffe5cc..e2b7dbb98 100644 --- a/vibe/2026-10-11-2-native-docs-debt-removal.md +++ b/vibe/2026-10-11-2-native-docs-debt-removal.md @@ -412,7 +412,7 @@ impl BodyText { -### Step 3: Resolve jump, call, and fanout targets in place +### Step 3: Resolve jump, call, and fanout targets in place [completed] - Component: In-place visible sets - Component placement: second. It depends on nothing and uses only the existing `Section` API. It precedes the shared-body component because both edit `read_visible`, so the parser's field-type change lands on readers that already borrow. From 5a617eadc1c9cc1dcb3725f2593017ec19b1dddd Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 08:32:18 -0700 Subject: [PATCH 17/34] Read prompt sections in place at section entry Entering a section no longer copies the sections it can address by heading. Section entry now receives the path to the section's sibling slice and its index there, and each prompt read resolves that slice from the shared prompt tree and reads it in place. The H1 pass reads the top-level sections in place the same way. The type that names a walk position moves beside the other heading lookup helpers so section setup can use it. - `SlicePath` moves out of the scheduler module into `walk_target.rs`, and the struct and its four methods become `pub(super)` so section setup can reach them. The scheduler files import it from there, and the scheduler module doc links to the new path. - `SectionContext::new` takes `home: &SlicePath` and `index: usize` in place of the section and its sibling slice, so its argument count stays at eight. The added lookup pushes the constructor to 80 lines, past the 75-line unit ceiling. - `Visible::of` now backs the walked-section read callback, which captures the shared prompt `Arc`, a clone of the slice path, and the index, and resolves the home slice on every read. - `h1_visible_sections` is read in place by the H1 read callback, which captures the shared prompt instead of a `.to_vec()` copy. - `Error::Internal` is returned when section entry finds no section at `index` in the resolved slice, with the message "an entered section is absent from its slice". The walk already stops when the index is past the slice, so this marks a broken invariant, and no test in this change reaches it. - `visible_sections` and `home_without` are deleted, and with them the cloned `Vec
` each section entry built. Design: replaces newtype @ crates/promptforge-internal/engine/src/execute/walk_target.rs::SlicePath was: crates/promptforge-internal/engine/src/execute/scheduler.rs::SlicePath Design: new oversized-unit @ crates/promptforge-internal/engine/src/execute/section_context-construct.rs::SectionContext::new Design: removes pure-function @ crates/promptforge-internal/engine/src/execute/walk_target.rs::visible_sections deps: Section Design: removes pure-function @ crates/promptforge-internal/engine/src/execute/walk_target.rs::home_without deps: Section Plan: vibe/2026-10-11-2-native-docs-debt-removal.md --- .../engine/src/execute/scheduler.rs | 49 +------------- .../engine/src/execute/scheduler/chain.rs | 3 +- .../src/execute/scheduler/chain_record.rs | 3 +- .../engine/src/execute/scheduler/h1.rs | 4 +- .../src/execute/scheduler/test_hooks.rs | 3 +- .../engine/src/execute/scheduler/walk.rs | 8 +-- .../src/execute/section_context-construct.rs | 42 +++++++----- .../engine/src/execute/walk_target.rs | 67 +++++++++++++------ vibe/2026-10-11-2-native-docs-debt-removal.md | 2 +- 9 files changed, 89 insertions(+), 92 deletions(-) diff --git a/crates/promptforge-internal/engine/src/execute/scheduler.rs b/crates/promptforge-internal/engine/src/execute/scheduler.rs index 727ed797d..a0ed00d51 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler.rs @@ -26,8 +26,8 @@ //! merged: `RunState` is cloned into callbacks, while the scheduler must //! stay unreachable from the callback layer. The prompt tree is shared //! through the context's `Arc`, so a chain names its position in it by -//! path ([`SlicePath`]) rather than by borrow, and the scheduler owns -//! itself outright. +//! path ([`SlicePath`](super::walk_target::SlicePath)) rather than by +//! borrow, and the scheduler owns itself outright. //! //! This file holds the scheduler core: the chain arena, the //! ready queue, the admission queues (spawned tasks wait there until @@ -88,7 +88,7 @@ use promptforge_types::ids::{Provenance, RoundId, TaskId}; use promptforge_vfs::Origin; use promptforge_vfs::detail::ScopeHandle; -use crate::parser::{Block, Prompt, Section}; +use crate::parser::{Block, Prompt}; use crate::{Error, Result}; use promptforge_types::event::{ReplyOrigin, lifecycle}; @@ -100,49 +100,6 @@ use pending::{Continuation, Pending, ToolCallContinuation, VfsContinuation}; use tasks::TaskSlot; pub(in crate::execute) use tool_call::RESERVED_TOOL_NAMES; -/// Where a sibling slice sits in the prompt tree: the index of each -/// ancestor section from the top level down to the slice's parent. The -/// empty path is the top-level slice. A chain names its walk position by -/// path so it borrows nothing from the tree the run shares through its -/// `Arc`; the path resolves to the slice on demand. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct SlicePath(Vec); - -impl SlicePath { - /// The top-level slice. - fn root() -> Self { - Self::default() - } - - /// The slice of `parent`'s children, where `parent` is the section at - /// `index` of this slice. - fn child(&self, index: usize) -> Self { - let mut path = self.0.clone(); - path.push(index); - Self(path) - } - - /// Resolves the path against `prompt`. A path the scheduler built is - /// always in range; an out-of-range index resolves to the empty slice, - /// which the walk treats as exhausted rather than panicking. - fn resolve<'p>(&self, prompt: &'p Prompt) -> &'p [Section] { - let mut slice = promptforge_parser::detail::sections(prompt); - for &index in &self.0 { - slice = slice.get(index).map_or(&[], Section::children); - } - slice - } - - /// The name of the slice's section at `index`, or the prompt's title - /// when `index` is past the slice: a chain's section name before its - /// first entry. - fn name_at<'p>(&self, prompt: &'p Prompt, index: usize) -> &'p str { - self.resolve(prompt) - .get(index) - .map_or(prompt.title(), Section::name) - } -} - /// The most precise prompt-source line known for `blocks`: the first /// compiled chunk's absolute source line, else the prompt's opening line. fn first_chunk_line(blocks: &[Block]) -> u32 { diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/chain.rs b/crates/promptforge-internal/engine/src/execute/scheduler/chain.rs index 82268fd72..3977fedb1 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/chain.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/chain.rs @@ -10,9 +10,10 @@ use crate::execute::context::RunState; use crate::execute::protocol::Answer; use crate::execute::run::EffectId; use crate::execute::support::GENERIC_COMPLETION; +use crate::execute::walk_target::SlicePath; use crate::{Error, Result}; -use super::{Chain, ChainIndex, ChatAnchor, Continuation, Counters, Pending, Scheduler, SlicePath}; +use super::{Chain, ChainIndex, ChatAnchor, Continuation, Counters, Pending, Scheduler}; impl Scheduler { /// Allocates the next child id under `owner`'s chain: the owner's id diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/chain_record.rs b/crates/promptforge-internal/engine/src/execute/scheduler/chain_record.rs index 65453f9df..0f73598f8 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/chain_record.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/chain_record.rs @@ -14,13 +14,14 @@ use crate::execute::context::RunState; use crate::execute::protocol::Answer; use crate::execute::scope::DispatchTarget; use crate::execute::section_context::{SectionContext, TaskSeed}; +use crate::execute::walk_target::SlicePath; use crate::lua::UsageAnchor; use crate::model::{Message, ModelId}; use crate::parser::{Block, Prompt, Section}; use crate::{Error, Result}; use super::await_tasks::AwaitTasks; -use super::{ChainIndex, Counters, SlicePath, SpawnRecord}; +use super::{ChainIndex, Counters, SpawnRecord}; /// What the chain's `chat` rounds taught the context precheck about token /// counts: the provider's numbers for the newest round that reported usage. diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs b/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs index 0dc10d8a9..f8f60cff9 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/h1.rs @@ -14,11 +14,11 @@ use std::sync::Arc; use promptforge_types::ids::ChainId; use crate::execute::support::GENERIC_COMPLETION; -use crate::execute::walk_target::{h1_visible_sections, section_position}; +use crate::execute::walk_target::{SlicePath, h1_visible_sections, section_position}; use crate::heading_address::{self, Visible}; use crate::{Error, Result}; -use super::{ChainIndex, Counters, Scheduler, SlicePath, prompt_origin}; +use super::{ChainIndex, Counters, Scheduler, prompt_origin}; impl Scheduler { /// Starts the H1 pass as the driver loop's first chain: the prompt's diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/test_hooks.rs b/crates/promptforge-internal/engine/src/execute/scheduler/test_hooks.rs index 321da624c..caafc660e 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/test_hooks.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/test_hooks.rs @@ -6,9 +6,10 @@ use promptforge_types::ids::{ChainId, TaskId}; use crate::Result; +use crate::execute::walk_target::SlicePath; pub(crate) use super::tasks::TaskState; -use super::{ChainIndex, Counters, Scheduler, SlicePath}; +use super::{ChainIndex, Counters, Scheduler}; impl Scheduler { /// Shrinks the chain-count bound so a test can drive the diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs b/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs index 77e3a0592..545f153b5 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/walk.rs @@ -15,13 +15,13 @@ use promptforge_types::ids::ChainId; use crate::execute::section_context::SectionContext; use crate::execute::walk_target::{ - JumpTarget, h1_visible_sections, resolve_jump_target, section_position, + JumpTarget, SlicePath, h1_visible_sections, resolve_jump_target, section_position, }; use crate::heading_address::{self, Visible}; use crate::parser::Block; use crate::{Error, Result}; -use super::{Chain, ChainIndex, Counters, Scheduler, SlicePath, prompt_origin}; +use super::{Chain, ChainIndex, Counters, Scheduler, prompt_origin}; use promptforge_vfs::Access; /// A heading resolved against a chain's visible set: the slice the walk or @@ -160,8 +160,8 @@ impl Scheduler { let frame = SectionContext::new( &chain.ctx, chain.access()?, - &slice[index], - slice, + &chain.slice, + index, §ion_id, &task, &chain.var, diff --git a/crates/promptforge-internal/engine/src/execute/section_context-construct.rs b/crates/promptforge-internal/engine/src/execute/section_context-construct.rs index e7c27b68f..5444cb721 100644 --- a/crates/promptforge-internal/engine/src/execute/section_context-construct.rs +++ b/crates/promptforge-internal/engine/src/execute/section_context-construct.rs @@ -12,13 +12,12 @@ use std::sync::Arc; use promptforge_types::ids::{ChainId, TaskId}; -use crate::Result; use crate::execute::context::RunState; use crate::execute::section_vm::{VmSeed, setup_section_vm}; -use crate::execute::walk_target::{h1_visible_sections, read_visible, visible_sections}; +use crate::execute::walk_target::{SlicePath, h1_visible_sections, read_visible}; use crate::heading_address::Visible; use crate::lua::SectionVm; -use crate::parser::Section; +use crate::{Error, Result}; use promptforge_types::event::lifecycle; use promptforge_vfs::Access; @@ -33,9 +32,10 @@ impl SectionContext { /// the rest of the shared setup half (Engine injection, Engine globals, the /// Plugin preludes, the shared replay, the store yield shims). /// - /// `siblings` is the caller's own walk slice, from which the section's - /// visible set (its siblings minus itself, plus its direct children) is - /// built for the `prompt` reads. `section_id` is the + /// `home` is the path of the caller's own walk slice and `index` the + /// section's place in it. Each `prompt` read resolves that slice from + /// the shared prompt and reads the section's visible set (its siblings + /// minus itself, plus its direct children) in place. `section_id` is the /// section's `sys.id`: the entering chain's hierarchical id extended /// by its local entry counter, allocated by the scheduler; `task_id` /// is the entering chain's `sys.taskid`. `var` is the walk's current @@ -44,24 +44,31 @@ impl SectionContext { /// every other entry. /// /// # Errors - /// Returns the [`Error`](crate::Error) of whichever step failed. A VM + /// Returns [`Error::Internal`] when `index` is past the `home` slice, + /// and otherwise the [`Error`] of whichever step failed. A VM /// construction or limits failure propagates bare, before any teardown /// observation exists; a setup failure tears the fresh VM down first, so /// the teardown boundary still fires exactly once on that path. #[expect( clippy::too_many_arguments, - reason = "the walk frame keeps its context, capability, section, visible slice, entry id, task id, var seed, and task seed explicit and linear" + reason = "the walk frame keeps its context, capability, slice path, section index, entry id, task id, var seed, and task seed explicit and linear" )] pub(crate) fn new( ctx: &RunState, access: &Arc, - section: &Section, - siblings: &[Section], + home: &SlicePath, + index: usize, section_id: &str, task_id: &TaskId, var: &serde_json::Value, seed: TaskSeed, ) -> Result { + let section = home + .resolve(ctx.prompt()) + .get(index) + .ok_or(Error::internal( + "an entered section is absent from its slice", + ))?; let mut sys = ctx.sys_json(section_id, task_id, section.name()); // A spawned chain's `sys.index` is the spawn's own value, verbatim; // absent otherwise, so a walked section reading `sys.index` raises @@ -89,8 +96,11 @@ impl SectionContext { // The `prompt` reads resolve over the section's visible set; the // suspending calls (`call`, `fanout`, `models.infer`) are the // yield shims the setup half installs. - let visible = visible_sections(siblings, section); - let read = move |heading: String| read_visible(&heading, Visible::all(&visible)); + let prompt = Arc::clone(ctx.prompt_arc()); + let home = home.clone(); + let read = move |heading: String| { + read_visible(&heading, Visible::of(home.resolve(&prompt), index)) + }; // The setup half of the section lifecycle - Engine injection, Engine // globals, the control surface, the shared replay, and the captured // alias bindings - is shared with the H1 pass; only the seed, the @@ -139,7 +149,7 @@ impl SectionContext { /// as the visible set behind the `prompt` reads. /// /// # Errors - /// Returns the [`Error`](crate::Error) of whichever step failed. A VM + /// Returns the [`Error`] of whichever step failed. A VM /// construction or limits failure propagates bare, before any teardown /// observation exists; a setup failure tears the fresh VM down first, so /// the teardown boundary still fires exactly once on that path. @@ -166,8 +176,10 @@ impl SectionContext { ctx.limits().lua_memory().get(), ctx.limits().lua_logs().get(), )?; - let visible = h1_visible_sections(ctx.prompt()).to_vec(); - let read = move |heading: String| read_visible(&heading, Visible::all(&visible)); + let prompt = Arc::clone(ctx.prompt_arc()); + let read = move |heading: String| { + read_visible(&heading, Visible::all(h1_visible_sections(&prompt))) + }; // H1's one privilege: `argv` installs writable, so the repair // pattern can assign it; the executor reads the value back at the // freeze (see the scheduler's H1-to-walk handoff). diff --git a/crates/promptforge-internal/engine/src/execute/walk_target.rs b/crates/promptforge-internal/engine/src/execute/walk_target.rs index 80487ddd2..dc4e5741a 100644 --- a/crates/promptforge-internal/engine/src/execute/walk_target.rs +++ b/crates/promptforge-internal/engine/src/execute/walk_target.rs @@ -8,13 +8,58 @@ //! structure. The scheduler's chains resolve jumps, `call` targets, and //! fanout workers through these helpers, and the `prompt.section` and //! `prompt.list` reads resolve through them too, so every surface agrees on -//! what a heading may name. +//! what a heading may name. [`SlicePath`] names a walk position in the +//! shared prompt tree, so a chain and a section's `prompt` reads find +//! their slice by path and read it in place. use crate::heading_address::{self, Visible}; use crate::lua::SectionText; use crate::parser::{Prompt, Section}; use crate::{Error, Result}; +/// Where a sibling slice sits in the prompt tree: the index of each +/// ancestor section from the top level down to the slice's parent. The +/// empty path is the top-level slice. A chain names its walk position by +/// path so it borrows nothing from the tree the run shares through its +/// `Arc`; the path resolves to the slice on demand. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct SlicePath(Vec); + +impl SlicePath { + /// The top-level slice. + pub(super) fn root() -> Self { + Self::default() + } + + /// The slice of `parent`'s children, where `parent` is the section at + /// `index` of this slice. + pub(super) fn child(&self, index: usize) -> Self { + let mut path = self.0.clone(); + path.push(index); + Self(path) + } + + /// Resolves the path against `prompt`. A path the scheduler built is + /// always in range; an out-of-range index resolves to the empty slice, + /// which the walk treats as exhausted rather than panicking. + pub(super) fn resolve<'p>(&self, prompt: &'p Prompt) -> &'p [Section] { + let mut slice = promptforge_parser::detail::sections(prompt); + for &index in &self.0 { + slice = slice.get(index).map_or(&[], Section::children); + } + slice + } + + /// The name of the slice's section at `index`, or the prompt's title + /// when `index` is past the slice: a chain's section name before its + /// first entry. + pub(super) fn name_at<'p>(&self, prompt: &'p Prompt, index: usize) -> &'p str { + self.resolve(prompt) + .get(index) + .map_or(prompt.title(), Section::name) + } +} + /// The index of `target` in `slice`, matched on the parser-unique /// `(level, name)` pair; `None` when the slice does not contain it. pub(super) fn section_position(slice: &[Section], target: &Section) -> Option { @@ -23,26 +68,6 @@ pub(super) fn section_position(slice: &[Section], target: &Section) -> Option Vec
{ - let caller_index = section_position(home, caller); - home.iter() - .enumerate() - .filter(|(index, _)| Some(*index) != caller_index) - .map(|(_, section)| section.clone()) - .collect() -} - -/// The sections a running section may address by heading: the caller's own -/// home slice minus the caller itself, plus the caller's direct children. -pub(super) fn visible_sections(home: &[Section], caller: &Section) -> Vec
{ - home_without(home, caller) - .into_iter() - .chain(caller.children().iter().cloned()) - .collect() -} - /// The H1 pass's visible set: the whole top-level slice. H1 is /// section 0, so it excludes nothing and has no children. pub(super) fn h1_visible_sections(prompt: &Prompt) -> &[Section] { diff --git a/vibe/2026-10-11-2-native-docs-debt-removal.md b/vibe/2026-10-11-2-native-docs-debt-removal.md index e2b7dbb98..0d2701d86 100644 --- a/vibe/2026-10-11-2-native-docs-debt-removal.md +++ b/vibe/2026-10-11-2-native-docs-debt-removal.md @@ -431,7 +431,7 @@ impl BodyText { -### Step 4: Read prompt sections in place at section entry +### Step 4: Read prompt sections in place at section entry [completed] - Component: In-place visible sets - Piece: section-entry reads in place. Follows the target-lookup piece, whose `Visible` and view-taking `read_visible` it uses. From 9ac7ee84d0e252a3f589ccede8ce5a853a0a5ba5 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 08:42:16 -0700 Subject: [PATCH 18/34] Borrow parser intermediates from the prompt body Parsing no longer copies each heading's content, each prose segment, and each list item into a new string while it builds the section tree. Those intermediate values now borrow from the text they were cut from, and the one copy left for each is made where the finished section or prose block is built. Parsed sections, their text, and parse errors stay the same, and no test changes. - `Heading` gains a lifetime, and its `content` field is now a `&'a str` slice of the body in place of a copied `String`. `collect_headings` returns headings that borrow from the body it walks, and every function that takes a heading now spells it `Heading<'_>`. - `RawBlock::Prose` carries a `&'a str` in place of a `String`, so `split_section_blocks` returns prose that borrows from the content it splits. Lua fence sources stay owned `String`s. - `parse_bullet_items` returns items borrowed from its prose input instead of copying each one. - `Block::Prose` and a section's items and markdown still own their text, so each is copied once with `to_owned()` where it is built. `list_items` copies the items, and `build_heading_blocks` and `split_h1` copy the prose. - `split_h1` still splits its own copy of the H1 content, with any `lua shared` fence blanked out, so H1 prose borrows from that copy rather than from the body. Design: extends pure-function @ crates/promptforge-internal/parser/src/build.rs::collect_headings deps: str Design: extends pure-function @ crates/promptforge-internal/parser/src/build.rs::check_child_level deps: Heading,u8 Design: extends pure-function @ crates/promptforge-internal/parser/src/build.rs::section_name deps: Heading,u8 Design: extends pure-function @ crates/promptforge-internal/parser/src/build.rs::list_items deps: [Block],str Design: extends pure-function @ crates/promptforge-internal/parser/src/build.rs::check_unique_sibling deps: Heading,[(String,u32)],str,u32 Design: extends pure-function @ crates/promptforge-internal/parser/src/fence.rs::split_section_blocks deps: str,str Design: extends pure-function @ crates/promptforge-internal/parser/src/list.rs::parse_bullet_items deps: str,str Plan: vibe/2026-10-11-2-native-docs-debt-removal.md --- .../promptforge-internal/parser/src/build.rs | 31 +++++++++++-------- .../promptforge-internal/parser/src/fence.rs | 19 +++++++----- .../promptforge-internal/parser/src/list.rs | 4 +-- .../promptforge-internal/parser/src/parse.rs | 6 ++-- vibe/2026-10-11-2-native-docs-debt-removal.md | 2 +- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/crates/promptforge-internal/parser/src/build.rs b/crates/promptforge-internal/parser/src/build.rs index 093cea4e9..df28a6478 100644 --- a/crates/promptforge-internal/parser/src/build.rs +++ b/crates/promptforge-internal/parser/src/build.rs @@ -49,10 +49,10 @@ pub(crate) fn line_add(a: u32, b: u32) -> Result { /// A heading with its title and the prose/Lua that follows it (before the next /// heading of any level). #[derive(Debug, Clone)] -pub(crate) struct Heading { +pub(crate) struct Heading<'a> { pub(crate) level: u8, pub(crate) title: String, - pub(crate) content: String, + pub(crate) content: &'a str, /// 1-based line number within `body` where `content` begins. pub(crate) content_start_line: u32, /// 1-based line number within `body` of the heading line itself. @@ -91,7 +91,7 @@ fn level_num(level: HeadingLevel) -> u8 { /// /// A heading inside a block quote or list item is not a section boundary; its /// text stays in the enclosing heading's content. -pub(crate) fn collect_headings(body: &str) -> Result> { +pub(crate) fn collect_headings(body: &str) -> Result>> { // First pass: find each heading's level, title, and source byte range. struct Raw { level: u8, @@ -143,7 +143,7 @@ pub(crate) fn collect_headings(body: &str) -> Result> { headings.push(Heading { level: raws[i].level, title: raws[i].title.clone(), - content: content.to_string(), + content, content_start_line, source_line, span: raws[i].range.clone(), @@ -154,13 +154,13 @@ pub(crate) fn collect_headings(body: &str) -> Result> { /// Builds one heading's executable blocks while preserving source positions. fn build_heading_blocks( - heading: &Heading, + heading: &Heading<'_>, name: &str, frontmatter_lines: u32, emitter: &Emitter, ) -> Result> { let content_abs_line = line_add(frontmatter_lines, heading.content_start_line)?; - let raw_blocks = split_section_blocks(&heading.content, name)?; + let raw_blocks = split_section_blocks(heading.content, name)?; let has_prose = raw_blocks .iter() .any(|block| matches!(block, RawBlock::Prose(_))); @@ -168,7 +168,9 @@ fn build_heading_blocks( let mut blocks = Vec::with_capacity(total); for (index, raw) in raw_blocks.into_iter().enumerate() { match raw { - RawBlock::Prose(text) => blocks.push(Block::Prose { text }), + RawBlock::Prose(text) => blocks.push(Block::Prose { + text: text.to_owned(), + }), RawBlock::Lua { source, line_offset, @@ -193,7 +195,7 @@ fn build_heading_blocks( /// H2 with no intervening H3, or an H3/H4 top-level section with no parent H2). /// Such a heading has no well-defined parent, so reject it rather than /// silently reparenting it to a shallower ancestor. -fn check_child_level(heading: &Heading, parent_level: u8) -> Result<()> { +fn check_child_level(heading: &Heading<'_>, parent_level: u8) -> Result<()> { let level = heading.level; if level > parent_level + 1 { return Err(Error::parse_at( @@ -212,7 +214,7 @@ fn check_child_level(heading: &Heading, parent_level: u8) -> Result<()> { /// Returns the heading's section name. A section's name is its runtime address /// (jumps, lookups, fanout), so an empty/whitespace heading is unaddressable /// and must be rejected at parse. -fn section_name(heading: &Heading, level: u8) -> Result { +fn section_name(heading: &Heading<'_>, level: u8) -> Result { let name = heading.title.clone(); if name.trim().is_empty() { return Err(Error::parse_at( @@ -240,7 +242,10 @@ fn list_items(blocks: &[Block], name: &str) -> Result> { && let Some(prose) = prose_for_items && is_all_list_markers(prose) { - parse_bullet_items(prose, name) + Ok(parse_bullet_items(prose, name)? + .into_iter() + .map(str::to_owned) + .collect()) } else { Ok(Vec::new()) } @@ -253,7 +258,7 @@ fn list_items(blocks: &[Block], name: &str) -> Result> { fn check_unique_sibling( siblings: &[(String, u32)], name: &str, - heading: &Heading, + heading: &Heading<'_>, heading_abs_line: u32, ) -> Result<()> { if let Some((_, first_line)) = siblings.iter().find(|(n, _)| n == name) { @@ -282,7 +287,7 @@ fn check_unique_sibling( /// [`Error::LuaCompile`] or [`Error::Lua`] from Lua compilation and /// [`Error::Internal`] from invalid or overflowing source-line calculations. pub(crate) fn build_sections( - headings: &[Heading], + headings: &[Heading<'_>], pos: &mut usize, parent_level: u8, frontmatter_lines: u32, @@ -314,7 +319,7 @@ pub(crate) fn build_sections( blocks, children, items, - markdown: trim_leading_blank_lines(&h.content).trim_end().to_owned(), + markdown: trim_leading_blank_lines(h.content).trim_end().to_owned(), }); } Ok(result) diff --git a/crates/promptforge-internal/parser/src/fence.rs b/crates/promptforge-internal/parser/src/fence.rs index dfb1dcff4..d91cec2bf 100644 --- a/crates/promptforge-internal/parser/src/fence.rs +++ b/crates/promptforge-internal/parser/src/fence.rs @@ -17,13 +17,13 @@ use super::{Block, LuaProgram, ParseErrorKind}; use crate::{Error, Result}; /// Uncompiled block produced while scanning section content. -pub(super) enum RawBlock { +pub(super) enum RawBlock<'a> { Lua { source: String, /// Lines before the first line of Lua source within section content. line_offset: u32, }, - Prose(String), + Prose(&'a str), } /// Extracts the optional exact `lua shared` library from H1 and compiles the @@ -85,7 +85,9 @@ pub(super) fn split_h1( let mut blocks = Vec::with_capacity(raw_blocks.len()); for raw in raw_blocks { match raw { - RawBlock::Prose(text) => blocks.push(Block::Prose { text }), + RawBlock::Prose(text) => blocks.push(Block::Prose { + text: text.to_owned(), + }), RawBlock::Lua { source, line_offset, @@ -251,12 +253,15 @@ fn pending_prose<'a>(content: &'a str, segment: Range, rules: &[Range Result> { +pub(super) fn split_section_blocks<'a>( + content: &'a str, + section: &str, +) -> Result>> { let openings = exact_lua_openings(content); let rules = rule_ranges(content); if openings.is_empty() { let pending = pending_prose(content, 0..content.len(), &rules); - return Ok(vec![RawBlock::Prose(pending.trim().to_string())]); + return Ok(vec![RawBlock::Prose(pending.trim())]); } let leading_start = leading_content_start(content); @@ -289,7 +294,7 @@ pub(super) fn split_section_blocks(content: &str, section: &str) -> Result Result Result> { +pub(super) fn parse_bullet_items<'a>(prose: &'a str, section: &str) -> Result> { let mut items = Vec::new(); for line in prose.lines() { let trimmed = line.trim(); @@ -26,7 +26,7 @@ pub(super) fn parse_bullet_items(prose: &str, section: &str) -> Result items.push(content.to_string()), + ListLine::Item(content) => items.push(content), ListLine::EmptyMarker => { return Err(Error::parse( ParseErrorKind::List, diff --git a/crates/promptforge-internal/parser/src/parse.rs b/crates/promptforge-internal/parser/src/parse.rs index 128142ebc..459045a00 100644 --- a/crates/promptforge-internal/parser/src/parse.rs +++ b/crates/promptforge-internal/parser/src/parse.rs @@ -125,7 +125,7 @@ impl Prompt { let title = h1.title.clone(); let h1_content_abs_line = line_add(frontmatter_lines, h1.content_start_line)?; let shared_fences = exact_shared_openings(body); - let h1_shared_fences = exact_shared_openings(&h1.content); + let h1_shared_fences = exact_shared_openings(h1.content); if shared_fences.len() > 1 { return Err(Error::parse( ParseErrorKind::Fence, @@ -140,11 +140,11 @@ impl Prompt { )); } let (replay, h1_blocks, description_text) = - split_h1(&h1.content, &title, h1_content_abs_line, emitter)?; + split_h1(h1.content, &title, h1_content_abs_line, emitter)?; // Everything before the H1 is preface and has no prompt semantics. // Sections are headings after the H1 at level 2 or deeper. - let section_headings: Vec = headings + let section_headings: Vec> = headings .into_iter() .skip(*h1_index + 1) .filter(|h| h.level >= 2) diff --git a/vibe/2026-10-11-2-native-docs-debt-removal.md b/vibe/2026-10-11-2-native-docs-debt-removal.md index 0d2701d86..9e381cd3a 100644 --- a/vibe/2026-10-11-2-native-docs-debt-removal.md +++ b/vibe/2026-10-11-2-native-docs-debt-removal.md @@ -448,7 +448,7 @@ impl BodyText { -### Step 5: Borrow parser intermediates from the body +### Step 5: Borrow parser intermediates from the body [completed] - Component: Shared prompt body - Component placement: third and last. It depends on the in-place component because both edit `read_visible`, and its Engine reader changes land on the in-place code. From cf4a94ba5df4d7f8872dbf055692fa446a0a16d2 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 08:57:31 -0700 Subject: [PATCH 19/34] Store each prompt's body once and read its text in place The parser now keeps one shared, read-only copy of each prompt's body, and each section's Markdown, list items, and prose blocks point into it by offset instead of holding their own copies. Prose under the top-level heading points into its own shared copy instead, since that copy can have the shared library fence blanked out. Prose waiting for the next Lua fence now shares that text instead of copying it. Equality and debug output still work on the text itself, so blocks compare and print as they did before. - `BodyText` holds a shared handle to the text plus a private byte range. `within` builds one only when the given slice lies wholly inside the shared text, located by address arithmetic with no `unsafe`, and `as_str` slices without checking again. - `from_owned` compiles only under test or the `test-support` feature, so other builds can make a `BodyText` only through `within`. - `body_text` returns an internal parser error, not a panic, when section text is not a slice of the shared text. - `build_sections` and `build_heading_blocks` take the shared body alongside the frontmatter line count and emitter, the same three values `parse_body` carries. - `split_h1` moves its masked copy of the H1 content into its own shared text and builds H1 prose against that copy. - `Block::Prose` now holds a `BodyText`, and a section's items come back as `&[BodyText]`; `markdown()` still returns `&str`. - `PartialEq` for `BodyText` compares the text read, not its position, and `Debug` prints the bare string. - `pending_prose` now holds a `BodyText`, so stashing a prose block for the next Lua fence clones a shared handle rather than the prose. - `read_visible` still copies the found section's Markdown and items into owned strings for Lua. - `AsRef` is implemented for `BodyText`, but nothing in this change calls it. Design: new encapsulated-invariant @ crates/promptforge-internal/parser/src/body_text.rs::BodyText boundary: pub Design: new pure-function @ crates/promptforge-internal/parser/src/build.rs::body_text deps: Arc,str Design: new shared-parameter-cluster @ crates/promptforge-internal/parser/src/build.rs::build_heading_blocks deps: Arc,Emitter,Heading,str,u32 Design: new shared-parameter-cluster @ crates/promptforge-internal/parser/src/build.rs::build_sections deps: Arc,Emitter,[Heading],u32,u8,usize Design: new shared-parameter-cluster @ crates/promptforge-internal/parser/src/parse.rs::Prompt::parse_body Design: extends pure-function @ crates/promptforge-internal/parser/src/build.rs::list_items deps: Arc,[Block],str Design: extends oversized-unit @ crates/promptforge-internal/parser/src/fence.rs::split_h1 deps: Emitter,str,str,u32 Design: extends pure-function @ crates/promptforge-internal/parser/src/test_support.rs::synthetic_section deps: Vec,str,u8 boundary: pub Design: extends pure-function @ crates/promptforge-internal/parser/src/test_support.rs::prose_block deps: String boundary: pub Design: extends facade @ crates/promptforge-internal/parser/src/lib.rs boundary: pub Design: extends facade @ crates/promptforge-internal/engine/src/parser.rs Design: extends pure-function @ crates/promptforge-internal/engine/src/execute/walk_target.rs::read_visible deps: Visible,str Plan: vibe/2026-10-11-2-native-docs-debt-removal.md --- .../src/execute/scheduler/chain_record.rs | 4 +- .../engine/src/execute/scheduler/step.rs | 6 +- .../engine/src/execute/walk_target.rs | 6 +- .../promptforge-internal/engine/src/parser.rs | 2 +- .../parser/src/body_text-tests.rs | 62 ++++++++++++++ .../parser/src/body_text.rs | 81 +++++++++++++++++++ .../promptforge-internal/parser/src/build.rs | 37 ++++++--- .../promptforge-internal/parser/src/fence.rs | 12 ++- crates/promptforge-internal/parser/src/lib.rs | 2 + .../promptforge-internal/parser/src/parse.rs | 15 +++- .../promptforge-internal/parser/src/prompt.rs | 12 +-- .../parser/src/test_support.rs | 8 +- .../promptforge-internal/parser/src/tests.rs | 2 +- .../parser/src/tests/breaks.rs | 8 +- vibe/2026-10-11-2-native-docs-debt-removal.md | 2 +- 15 files changed, 221 insertions(+), 38 deletions(-) create mode 100644 crates/promptforge-internal/parser/src/body_text-tests.rs create mode 100644 crates/promptforge-internal/parser/src/body_text.rs diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/chain_record.rs b/crates/promptforge-internal/engine/src/execute/scheduler/chain_record.rs index 0f73598f8..29df6510f 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/chain_record.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/chain_record.rs @@ -17,7 +17,7 @@ use crate::execute::section_context::{SectionContext, TaskSeed}; use crate::execute::walk_target::SlicePath; use crate::lua::UsageAnchor; use crate::model::{Message, ModelId}; -use crate::parser::{Block, Prompt, Section}; +use crate::parser::{Block, BodyText, Prompt, Section}; use crate::{Error, Result}; use super::await_tasks::AwaitTasks; @@ -183,7 +183,7 @@ pub(super) struct Chain { /// consumes, installed as that block's lazy `prose` template when the /// coroutine starts. Cleared at every section entry; an unconsumed /// buffer drops with the section, never evaluated. - pub(super) pending_prose: Option, + pub(super) pending_prose: Option, /// The walk's `var` table: seeds each section's VM at entry; the /// section's final `var` is read back before teardown and replaces the /// slot. A call chain's slot seeds from the caller's snapshot and diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/step.rs b/crates/promptforge-internal/engine/src/execute/scheduler/step.rs index 994b7fdcb..e6a12412d 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/step.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/step.rs @@ -12,7 +12,7 @@ use mlua::Thread; use crate::execute::protocol::{Answer, YieldParse}; use crate::lua::{CoroStep, LuaBlockResult}; -use crate::parser::Block; +use crate::parser::{Block, BodyText}; use crate::{Error, Result}; use promptforge_types::event::lifecycle; @@ -156,7 +156,9 @@ impl Scheduler { .frame .as_ref() .ok_or(Error::internal("a live chain holds its frame"))?; - if let Err(error) = frame.install_lazy_prose(&chain.ctx, pending.as_deref().unwrap_or("")) { + if let Err(error) = + frame.install_lazy_prose(&chain.ctx, pending.as_ref().map_or("", BodyText::as_str)) + { emitter.report(&name, lifecycle::LUA_CHUNK_FAILED); return Err(error); } diff --git a/crates/promptforge-internal/engine/src/execute/walk_target.rs b/crates/promptforge-internal/engine/src/execute/walk_target.rs index dc4e5741a..b7c2675c6 100644 --- a/crates/promptforge-internal/engine/src/execute/walk_target.rs +++ b/crates/promptforge-internal/engine/src/execute/walk_target.rs @@ -86,7 +86,11 @@ pub(super) fn read_visible(heading: &str, visible: Visible<'_>) -> Result
{ let Value::Table(table) = collection else { return Err(Error::Lua( - "fanout's second parameter is a collection; for a list section use prompt.list(heading), which returns nil when no visible section has that heading".to_owned(), + "tasks.fanout's second parameter is a collection; for a list section use prompt.list(heading), which returns nil when no visible section has that heading".to_owned(), )); }; let members = lua.create_table().map_err(Error::lua)?; diff --git a/crates/promptforge-internal/lua/src/coro.rs b/crates/promptforge-internal/lua/src/coro.rs index fcb98ccf7..ed8747233 100644 --- a/crates/promptforge-internal/lua/src/coro.rs +++ b/crates/promptforge-internal/lua/src/coro.rs @@ -2,10 +2,10 @@ //! suspending Engine calls. //! //! Yield cannot cross the C boundary, so `models.infer`, `call`, -//! `tools.call`, the `tasks` namespace, and `fanout` are Lua shims (source -//! in the `__impl_*.lua` files beside this one) that `coroutine.yield` a -//! request table and interpret the resume values as the `(ok, result)` -//! envelope; coroutine driving itself +//! `tools.call`, the `tasks` namespace, and `tasks.fanout` are Lua shims +//! (source in the `__impl_*.lua` files beside this one) that +//! `coroutine.yield` a request table and interpret the resume values as the +//! `(ok, result)` envelope; coroutine driving itself //! (`Thread::create`/`resume`) is pure Rust in the scheduler. The source is //! pulled in with `include_str!` so chunk line 1 is file line 1, compiled //! once through the usual [`LuaProgram`] machinery, and loaded per VM. The @@ -134,13 +134,13 @@ static FANOUT_PROGRAM: LazyLock> = /// `yield` and `var_snapshot` captures plus the prelude's returned failure /// helpers, and installed as the `tasks` global; `fanout` is a third, run /// over the same captures plus the collection enumerator and the item -/// renderer, and installed as the `fanout` global. The `models.loop` shim -/// is stashed in the registry for [`install_section_loop_shim`], and the -/// chunk's `handle_methods` table, a model handle's `infer` and `loop`, for -/// [`handle_method`]. `max_tool_iterations`, the run's resolved round cap, -/// and the `compactors` table go to -/// [`loop_begin`](crate::models_loop::loop_begin), which runs the loop's -/// rules behind the shim's trampoline. +/// renderer, and installed on the `tasks` table as `tasks.fanout`. The +/// `models.loop` shim is stashed in the registry for +/// [`install_section_loop_shim`], and the chunk's `handle_methods` table, a +/// model handle's `infer` and `loop`, for [`handle_method`]. +/// `max_tool_iterations`, the run's resolved round cap, and the +/// `compactors` table go to [`loop_begin`](crate::models_loop::loop_begin), +/// which runs the loop's rules behind the shim's trampoline. /// /// Three further captures give the chunk the structured error shape: /// `error_value(kind, fields)` builds the `{ kind, message, ... }` table @@ -229,7 +229,7 @@ pub(crate) fn install_shim_prelude( .load(lua)? .call((yield_fn.clone(), var_snapshot.clone(), helpers.clone())) .map_err(Error::lua)?; - globals.raw_set("tasks", tasks).map_err(Error::lua)?; + globals.raw_set("tasks", &tasks).map_err(Error::lua)?; // The enumerator answers `(members)` or `(nil, message)` so the shim // raises the message as its own call error, exactly as the other // author-argument failures surface at the call site. @@ -261,7 +261,7 @@ pub(crate) fn install_shim_prelude( render_item, )) .map_err(Error::lua)?; - globals.raw_set("fanout", fanout).map_err(Error::lua)?; + tasks.raw_set("fanout", fanout).map_err(Error::lua)?; stash_shims(lua, &shims)?; globals .raw_set("coroutine", Value::Nil) diff --git a/crates/promptforge-internal/lua/src/globals.rs b/crates/promptforge-internal/lua/src/globals.rs index aab86c150..bb015a419 100644 --- a/crates/promptforge-internal/lua/src/globals.rs +++ b/crates/promptforge-internal/lua/src/globals.rs @@ -152,12 +152,11 @@ impl fmt::Display for Reserved { /// A global section setup installs must be listed here: the Engine's /// section setup tests compare this list against a set-up VM's globals in /// both directions. -pub const RESERVED_NAMES: [(&str, Reserved); 61] = [ +pub const RESERVED_NAMES: [(&str, Reserved); 60] = [ ("args", Reserved::EngineGlobal), ("argv", Reserved::EngineGlobal), ("call", Reserved::EngineGlobal), ("compactors", Reserved::EngineGlobal), - ("fanout", Reserved::EngineGlobal), ("item", Reserved::EngineGlobal), ("jump", Reserved::EngineGlobal), ("log", Reserved::EngineGlobal), diff --git a/crates/promptforge-internal/lua/src/protocol.rs b/crates/promptforge-internal/lua/src/protocol.rs index 0c2b37d95..d68c65f75 100644 --- a/crates/promptforge-internal/lua/src/protocol.rs +++ b/crates/promptforge-internal/lua/src/protocol.rs @@ -2,7 +2,7 @@ //! yield/resume boundary between section Lua and the scheduler driver. //! //! A suspending Engine call (`models.infer(prompt)`, `h:infer(prompt)`, `call`, -//! `tasks.spawn`, `fanout`, `tools.call`, `store.*`, and +//! `tasks.spawn`, `tasks.fanout`, `tools.call`, `store.*`, and //! the `chat` and `tool_call` rounds the `models.loop` shim yields on the //! author's behalf) is a Lua-side shim that yields a request table; the driver validates the //! yield into a [`Request`], dispatches it, and resumes the coroutine with diff --git a/crates/promptforge-internal/lua/src/protocol/render.rs b/crates/promptforge-internal/lua/src/protocol/render.rs index 837192bf9..03fb9495e 100644 --- a/crates/promptforge-internal/lua/src/protocol/render.rs +++ b/crates/promptforge-internal/lua/src/protocol/render.rs @@ -63,7 +63,8 @@ fn store_value(lua: &Lua, outcome: VfsOutcome) -> mlua::Result { /// failure rendered as the error table the shim hands back unraised, so /// `join` reports it without raising. The failure is also returned as /// the typed error to retain: a shim that re-raises the member's failure at -/// once (`fanout` on a fatal arm) surfaces the member's own typed error. +/// once (`tasks.fanout` on a fatal arm) surfaces the member's own typed +/// error. fn delivery_values( lua: &Lua, delivery: TaskDelivery, diff --git a/crates/promptforge-internal/lua/src/protocol/request.rs b/crates/promptforge-internal/lua/src/protocol/request.rs index 94043d5de..ff9a30204 100644 --- a/crates/promptforge-internal/lua/src/protocol/request.rs +++ b/crates/promptforge-internal/lua/src/protocol/request.rs @@ -70,8 +70,8 @@ pub enum Request { /// Whether the spawn is a `fanout` arm. Shim-produced: the `fanout` /// shim says `true`, `tasks.spawn` leaves it absent. The depth-cap /// refusal is named after the author-facing call that tripped it - /// (`fanout` or `call`), so the typed error uses the wording the - /// author reads and no shim re-match is needed. + /// (`tasks.fanout` or `call`), so the typed error uses the wording + /// the author reads and no shim re-match is needed. fanout: bool, }, /// The wait shims' internal timeout timer: a leaf request whose work diff --git a/crates/promptforge-internal/lua/src/sys.rs b/crates/promptforge-internal/lua/src/sys.rs index 33676070b..8d9e16ad7 100644 --- a/crates/promptforge-internal/lua/src/sys.rs +++ b/crates/promptforge-internal/lua/src/sys.rs @@ -242,8 +242,8 @@ pub(crate) fn guarded_var(lua: &Lua, initial: Option<&Json>) -> Result -### Step 2: Move fanout into the tasks table as tasks.fanout +### Step 2: Move fanout into the tasks table as tasks.fanout [completed] - Component: `none` - Work items: `fanout`, `fanout-docs`, and `verify`, detailed in the Technical Design's Step 2 list. The install change, the `RESERVED_NAMES` entry, the call-site renames, and the error-text asserts land in one `promptforge` commit, because the tests fail with any one of them alone. From 83d40b1b932844c170b9b0ba73d8f4eaec2b7ef9 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 12:07:51 -0700 Subject: [PATCH 23/34] Close plan: parallel-tool-calls Plan: vibe/2026-10-11-3-parallel-tool-calls.md --- vibe/2026-10-11-3-parallel-tool-calls.md | 14 +++++++------- vibe/ACTIVE | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/2026-10-11-3-parallel-tool-calls.md b/vibe/2026-10-11-3-parallel-tool-calls.md index 3a2aabe2b..74910c7a3 100644 --- a/vibe/2026-10-11-3-parallel-tool-calls.md +++ b/vibe/2026-10-11-3-parallel-tool-calls.md @@ -4,25 +4,25 @@ overview: Inside `models.loop`, send a model round's remaining tool calls to the todos: - id: machine content: "Step 1, Lua crate: Request::ToolCalls over the existing ToolCallRecord, Answer::ToolCallsResult of texts, their parse (reusing parse_tool_call per entry) and render; the machine sends every remaining call together as tool_calls, even one, and a call alone only when the Engine stopped at it" - status: pending + status: completed - id: scheduler content: "Step 1, Engine: dispatch_tool_calls issues the leading calls that are neither task built-ins nor local tools, each tagged with its slot; a preparation failure is a filled Err slot that stops issuing; apply fills slots and answers once in call order; abort_subtree orphans every pending effect, sorted by id" - status: pending + status: completed - id: batch-tests content: "Step 1: rewrite the eight tests that pin or answer the single-call request after a model reply (seven in machine-tests.rs, one in engine lua/tests/shims.rs), one new machine test plus two one-line extensions, four hand-stepped Engine tests in parallel_tool_calls.rs, one Harness test over HOLD" - status: pending + status: completed - id: fanout content: "Step 2: set fanout on the tasks table instead of as a global (two lines in coro.rs); drop fanout from RESERVED_NAMES (61 to 60); four error texts; rename every call site in 25 test files and 5 fixtures; update three exact asserts and tighten three loose ones" - status: pending + status: completed - id: fanout-docs content: "Step 2: the doc and comment lines that name the call or the global; language spec edits in ../promptforge-design committed there" - status: pending + status: completed - id: verify content: "Each step: both nextest runs, both clippy runs with warnings denied, fmt, docs and facade docs, xtask api --check, build-xtask, docs-claims" - status: pending + status: completed - id: close content: "Step 3: mark the vibe/ copy completed and commit Close plan: parallel-tool-calls; nothing is pushed" - status: pending + status: completed isProject: false --- diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 222fc2357..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-10-11-3-parallel-tool-calls.md From 3121b18c751793155ec00f525c7892f64105ae7c Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 13:12:49 -0700 Subject: [PATCH 24/34] Keep Workshop waits answerable when calls run in parallel Now that one model round can run several tool calls at once, the Workshop agent panel can face more than one open operator question, and other calls can finish while a question waits. The session service now keeps every open question in the order the server announced it and always offers the oldest, so answering or cancelling one moves on to the next instead of stranding the rest. A sibling call's result no longer switches the panel into its working state while a question is open, so the composer keeps offering Send rather than Stop. After a reconnect the list rebuilds from the server's resend, oldest first. - `openWaits` holds the open wait tokens in announcement order and is now the source of the pin. `pendingInputToken` returns its oldest entry, `pinOldestWait` republishes that entry through `setPinnedToken` after every add, answer, and cancel, and `pinnedToken` only remembers the last published value. - `this.openWaits.includes(token)` guards each new announcement, so a token the server repeats is never queued twice. - `respond` answers the oldest open wait and spends only that token, then turns `generating` on only when no wait remains open. A failed send still keeps every wait open. - `wire.onInputCancelled` removes the cancelled token wherever it sits, so cancelling a newer wait leaves the older one pinned. - `tool_call_update` turns `generating` on only when `answersModelCall` matches and no wait is open. This relies on each wait closing before its own call's result arrives. - `acknowledge` empties the list on every acknowledgment and relies on the server resending unresolved waits right after, in creation order, so the oldest comes back first. - `crates/workshop/ui/test/agent-session-service.mjs` now cancels each open wait in the existing model-ask checks before the next result or turn arrives, and keeps their assertions. A cancel leaves `generating` off, so only the result can turn it on and the checks still test the flip. - `crates/workshop/ui/test/agent-session-view.mjs` checks that the composer action stays `send` when a sibling call's result lands while the ask waits. Design: extends god-object @ crates/workshop/ui/src/services/agent-session.ts::AgentSessionService boundary: pub Design: extends temporal-coupling @ crates/workshop/ui/src/services/agent-session.ts::AgentSessionService::acknowledge Design: new temporal-coupling @ crates/workshop/ui/src/services/agent-session.ts::AgentSessionService::foldEvent Design: new clone-block @ crates/workshop/ui/test/agent-session-view.mjs Deferred: The published guide's move from fanout to tasks.fanout waits for a separate change. Deferred: The published guide's removal of list_from_section calls waits for a separate change. Repairs: generating stays off while a wait is open @ crates/workshop/ui/src/services/agent-session.ts::AgentSessionService::foldEvent - a sibling call's result no longer turns generating on Repairs: every open wait stays reachable oldest first @ crates/workshop/ui/src/services/agent-session.ts::AgentSessionService::pendingInputToken - a newer wait no longer replaces an older open one Plan: vibe/2026-10-11-4-workshop-waits.md --- .../workshop/ui/src/services/agent-session.ts | 75 ++++-- .../ui/test/agent-session-service.mjs | 92 +++++++ .../workshop/ui/test/agent-session-view.mjs | 22 ++ vibe/2026-10-11-4-workshop-waits.md | 233 ++++++++++++++++++ vibe/ACTIVE | 1 + 5 files changed, 397 insertions(+), 26 deletions(-) create mode 100644 vibe/2026-10-11-4-workshop-waits.md create mode 100644 vibe/ACTIVE diff --git a/crates/workshop/ui/src/services/agent-session.ts b/crates/workshop/ui/src/services/agent-session.ts index 500a9ff8d..552df8b04 100644 --- a/crates/workshop/ui/src/services/agent-session.ts +++ b/crates/workshop/ui/src/services/agent-session.ts @@ -205,6 +205,12 @@ export class AgentSessionService extends Disposable { * streaming under an idle composer. */ private stopped = false; + /** + * The open wait tokens in the order the server announced them. A + * round's calls run in parallel, so several waits can be open at once; + * the oldest is the pin the operator answers next. + */ + private openWaits: string[] = []; private pinnedToken: string | null = null; private working = false; private dropped = false; @@ -263,7 +269,11 @@ export class AgentSessionService extends Disposable { // The relaunched agent is waiting: the cancel grace window is over. this.stopped = false; this.setGenerating(false); - this.setPinnedToken(token); + // A reattach resend repeats tokens that may already be open. + if (!this.openWaits.includes(token)) { + this.openWaits.push(token); + } + this.pinOldestWait(); if (closed) { this._onDidChangeTranscript.fire(); } @@ -271,10 +281,10 @@ export class AgentSessionService extends Disposable { ); this._register( wire.onInputCancelled((token) => { - // Only the announced wait dies; a newer pin stays. - if (this.pinnedToken === token) { - this.setPinnedToken(null); - } + // Only the announced wait dies, wherever it sits; the oldest + // wait still open keeps or takes the pin. + this.openWaits = this.openWaits.filter((open) => open !== token); + this.pinOldestWait(); }), ); this._register(wire.onError((message) => this.foldError(message, null))); @@ -298,13 +308,14 @@ export class AgentSessionService extends Disposable { /** The pending wait's token, or null while the agent is working. */ get pendingInputToken(): string | null { - return this.pinnedToken; + return this.openWaits[0] ?? null; } /** * True while the agent is working on the operator's last message: on - * after a send (and after an answered tool call), off when the reply - * lands, a wait opens, an error arrives, or a cancel is sent. + * after a send or an answered tool call that leaves no wait open, off + * when the reply lands, a wait opens, an error arrives, or a cancel is + * sent. */ get generating(): boolean { return this.working; @@ -328,17 +339,18 @@ export class AgentSessionService extends Disposable { } /** - * Answers the pending wait with the operator's text, byte-exact. The - * text never enters the transcript here: the server records it and the - * durable `user_message` event renders it, so the view shows exactly - * what the log holds. False when no wait is pinned or the socket is - * down; a failed send keeps the pin (the wait is still open - * server-side) and folds a local error item, because a downed socket - * is the one failure the server can never report. A sent answer starts - * the turn's work, so `generating` turns on. + * Answers the pinned wait - the oldest open one - with the operator's + * text, byte-exact. The text never enters the transcript here: the + * server records it and the durable `user_message` event renders it, + * so the view shows exactly what the log holds. False when no wait is + * open or the socket is down; a failed send keeps every open wait (each + * is still open server-side) and folds a local error item, because a + * downed socket is the one failure the server can never report. A sent + * answer spends its token and pins the next oldest wait; `generating` + * turns on only once no wait remains open. */ respond(text: string): boolean { - const token = this.pinnedToken; + const token = this.pendingInputToken; if (token === null) { return false; } @@ -349,8 +361,11 @@ export class AgentSessionService extends Disposable { // The answer restarts the turn's work, so a stop no longer applies. this.stopped = false; // The token is single-use; the response just spent it. - this.setPinnedToken(null); - this.setGenerating(true); + this.openWaits = this.openWaits.filter((open) => open !== token); + this.pinOldestWait(); + if (this.openWaits.length === 0) { + this.setGenerating(true); + } return true; } @@ -378,9 +393,10 @@ export class AgentSessionService extends Disposable { * Folds a session acknowledgment. A different session id means a * different event log replaying from index zero, so the transcript * resets with it; a same-session reattach keeps the transcript (the - * socket's cursor already deduplicates the replay). Wait pinning - * resets on every acknowledgment: the server resends unresolved waits - * right after, so a stale prompt vanishes by its token's absence. The + * socket's cursor already deduplicates the replay). The open waits + * reset on every acknowledgment: the server resends unresolved waits + * right after, in creation order, so the list rebuilds oldest first + * and a stale prompt vanishes by its token's absence. The * acknowledgment also ends a reconnect. */ private acknowledge(frame: AgentSessionFrame): void { @@ -394,6 +410,7 @@ export class AgentSessionService extends Disposable { this.setGenerating(false); this._onDidChangeTranscript.fire(); } + this.openWaits = []; this.setPinnedToken(null); this.setReconnecting(false); this._onDidChangeSession.fire(frame); @@ -455,11 +472,12 @@ export class AgentSessionService extends Disposable { } case "tool_call_update": { // An answered model call means the agent has its result and is - // working again - this is also what ends the wait of a model's - // own ask. Checked before the result lands so the scan reads - // only the calls. + // working again, unless a wait is still open: a round's calls run + // in parallel, so a sibling's result does not end the operator's + // turn. Checked before the result lands so the scan reads only + // the calls. const toolCallId = event.tool_call_id ?? null; - if (toolCallId !== null && this.answersModelCall(toolCallId)) { + if (toolCallId !== null && this.openWaits.length === 0 && this.answersModelCall(toolCallId)) { this.setGenerating(true); } this.transcript.push({ @@ -629,6 +647,11 @@ export class AgentSessionService extends Disposable { return -1; } + /** Pins the oldest open wait, or clears the pin when none is open. */ + private pinOldestWait(): void { + this.setPinnedToken(this.openWaits[0] ?? null); + } + /** Pins or clears the wait token, firing only on a real change. */ private setPinnedToken(token: string | null): void { if (this.pinnedToken === token) { diff --git a/crates/workshop/ui/test/agent-session-service.mjs b/crates/workshop/ui/test/agent-session-service.mjs index 56306ed7a..ed8c344c4 100644 --- a/crates/workshop/ui/test/agent-session-service.mjs +++ b/crates/workshop/ui/test/agent-session-service.mjs @@ -293,6 +293,69 @@ await assertNoLeaks(lifecycle, () => { service.dispose(); } + // --- Open waits pin the oldest and advance on each answer ----------------- + + { + const wire = makeWire(); + const service = new AgentSessionService(wire); + wire.fire.inputRequired("tok1"); + wire.fire.inputRequired("tok2"); + check("two open waits pin the oldest", service.pendingInputToken === "tok1"); + service.respond("first"); + check("answering the oldest wait pins the next", service.pendingInputToken === "tok2"); + check("a wait still open keeps generating off", service.generating === false); + service.respond("second"); + check("answering the last open wait unpins", service.pendingInputToken === null); + check("answering the last open wait turns generating on", service.generating === true); + check( + "each answer spent its own token", + isDeepStrictEqual(wire.responses, [ + ["tok1", "first"], + ["tok2", "second"], + ]), + ); + service.dispose(); + } + + // --- A cancelled wait leaves wherever it sits in the open waits ------------ + + { + const openTwo = () => { + const wire = makeWire(); + const service = new AgentSessionService(wire); + wire.fire.inputRequired("tok1"); + wire.fire.inputRequired("tok2"); + return { wire, service }; + }; + const oldest = openTwo(); + oldest.wire.fire.inputCancelled("tok1"); + check("cancelling the oldest wait pins the next", oldest.service.pendingInputToken === "tok2"); + oldest.service.dispose(); + const newer = openTwo(); + newer.wire.fire.inputCancelled("tok2"); + check("cancelling a newer wait keeps the oldest pinned", newer.service.pendingInputToken === "tok1"); + newer.service.dispose(); + } + + // --- A reattach pins the oldest unresolved wait ---------------------------- + + { + const wire = makeWire(); + const service = new AgentSessionService(wire); + wire.fire.session("s1"); + wire.fire.inputRequired("tok1"); + wire.fire.inputRequired("tok2"); + wire.fire.session("s1"); + wire.fire.inputRequired("tok1"); + wire.fire.inputRequired("tok2"); + check("the reattach resend pins the oldest unresolved wait", service.pendingInputToken === "tok1"); + service.respond("first"); + check("answering after the resend pins the newer wait", service.pendingInputToken === "tok2"); + service.respond("second"); + check("two answers drain the resent waits", service.pendingInputToken === null); + service.dispose(); + } + // --- A failed respond keeps the pin and folds a local error --------------- { @@ -497,6 +560,10 @@ await assertNoLeaks(lifecycle, () => { }); wire.fire.inputRequired("tok1"); check("an open wait keeps generating off", service.generating === false); + // Each wait closes before its result lands, as it does in the shipped + // UI; a cancel, not a respond, so generating is still off and only the + // result can turn it on. + wire.fire.inputCancelled("tok1"); wire.fire.event("tool_call_update", "operator answer", { tool_call_id: "c1" }); check( "a tool_call_update answering a model batch call turns generating on", @@ -504,6 +571,7 @@ await assertNoLeaks(lifecycle, () => { ); wire.fire.inputRequired("tok2"); check("input_required turns generating off", service.generating === false); + wire.fire.inputCancelled("tok2"); wire.fire.event("user_message", "second turn"); wire.fire.event("tool_call_update", "stray", { tool_call_id: "c1" }); check( @@ -526,6 +594,30 @@ await assertNoLeaks(lifecycle, () => { service.dispose(); } + // --- A sibling call's result does not end an open wait -------------------- + + { + const wire = makeWire(); + const service = new AgentSessionService(wire); + wire.fire.session("s1"); + wire.fire.event("user_message", "parallel round"); + wire.fire.event( + "tool_call", + JSON.stringify([ + { id: "c1", name: "ask", arguments: {}, tool: "user-input/ask" }, + { id: "c2", name: "fetch", arguments: {}, tool: "web/fetch" }, + ]), + { reply: 0 }, + ); + wire.fire.inputRequired("tok1"); + wire.fire.event("tool_call_update", "page body", { tool_call_id: "c2" }); + check("a sibling call's result while a wait is open leaves generating off", service.generating === false); + check("a sibling call's result leaves the wait pinned", service.pendingInputToken === "tok1"); + service.respond("answer"); + check("answering the open wait after a sibling's result turns generating on", service.generating === true); + service.dispose(); + } + // --- reconnecting: disconnect on, next acknowledgment off ------------------ { diff --git a/crates/workshop/ui/test/agent-session-view.mjs b/crates/workshop/ui/test/agent-session-view.mjs index 7dad90a6a..c52771514 100644 --- a/crates/workshop/ui/test/agent-session-view.mjs +++ b/crates/workshop/ui/test/agent-session-view.mjs @@ -1343,6 +1343,28 @@ await assertNoLeaks(lifecycle, async () => { dispose(); } + { + // A sibling call finishing while the round's ask waits must not offer Stop. + const { wire, service, send, dispose } = setup(); + wire.fire.session("s-parallel"); + wire.fire.event("user_message", "go"); + wire.fire.event( + "tool_call", + JSON.stringify([ + { id: "c1", name: "ask", arguments: {}, tool: "user-input/ask" }, + { id: "c2", name: "fetch", arguments: {}, tool: "web/fetch" }, + ]), + { reply: 0 }, + ); + wire.fire.inputRequired("tok1"); + wire.fire.event("tool_call_update", "page body", { tool_call_id: "c2" }); + check( + "a sibling call's result while the ask waits keeps the action on send", + service.generating === false && send.getAttribute("data-action") === "send", + ); + dispose(); + } + // --- The mode follows the toolbar's chip onto the action button ------------------------------ { diff --git a/vibe/2026-10-11-4-workshop-waits.md b/vibe/2026-10-11-4-workshop-waits.md new file mode 100644 index 000000000..5dac62baa --- /dev/null +++ b/vibe/2026-10-11-4-workshop-waits.md @@ -0,0 +1,233 @@ +--- +name: Workshop waits under parallel calls +overview: "Remove the debt that parallel tool calls exposed in the Workshop agent session service: an open operator question must keep the composer answerable when a sibling call lands, and every open question must stay reachable, oldest first. The guide's stale fanout examples are recorded but left out at the user's direction." +todos: + - id: ui-waits + content: "D1-1 and C1-1: track open waits oldest first in agent-session.ts, gate the tool_call_update generating flip on no open wait, and add the service and view regression tests" + status: pending + - id: verify + content: Run npm test and typecheck in crates/workshop and the docs-claims scan; confirm each new test fails at 83d40b1b9 + status: pending +isProject: false +--- + +# Keep Workshop waits correct when a round's calls run in parallel + + + +## Product Requirements + +This plan fixes two Workshop UI defects that parallel tool calls made reachable in the ordinary one-loop agent shape. Both live in one service file and are fixed together with regression tests. The debt the target itself introduced, a published guide that still teaches the removed `fanout` global, is recorded but left out of this plan at the user's direction. + +- Scope and target work: + - Target: commits `1ce5831d0` (Run one round's catalog tool calls in parallel), `a490542e9` (Move fanout into the tasks table as tasks.fanout), and `83d40b1b9` (Close plan: parallel-tool-calls), all with `Plan: vibe/2026-10-11-3-parallel-tool-calls.md`. + - Baseline `e62bd6d77`, the merge base of `master` with `origin/master`. Endpoint and disposition `83d40b1b9`, with a clean worktree. + - Removal scope: D1-1 and C1-1, which the target exposed in Workshop UI code it did not touch, brought into scope by the user. +- Cleanup goals: + - While an operator question from the model is open, the composer stays answerable, whatever the round's other calls do. + - When the model asks two or more questions in one round, every one stays reachable, and the composer answers them oldest first. +- Non-goals: + - No Engine, Harness, Workshop server, or wire-format change. + - No edit to `../promptforge-docs`. + - No change to what Stop means while a question is open (D1-12). +- Success criteria: + - Each new regression test fails at `83d40b1b9` and passes after the change. + - Every existing Workshop UI test, the UI type checks, and the wording scan pass. Four existing checks in the model-ask block keep their assertions, but their setup clears a wait the shipped UI could never leave open (Testing Plan). + +## Functional Specification + +The inventory separates the debt the target added, which is excluded by user decision, from the two exposed defects this plan fixes. + +### Debt Inventory + +- Debt added: + - D1-2, introduced by `a490542e9`: the published language guide still teaches the removed `fanout` global. + - Evidence: at the endpoint `fanout` lives only on the `tasks` table (`crates/promptforge-internal/lua/src/coro.rs` line 264) and `RESERVED_NAMES` has 60 entries (`crates/promptforge-internal/lua/src/globals.rs`). `../promptforge-docs` at `44ff253` has 32 bare `fanout(` calls across chapters 01, 02, 07, 08, 09, 14, and 17, and no `tasks.fanout`. `.github/workflows/site.yml` publishes that checkout beside this repository. + - Relationship: the plan's claim that "No other crate, `prompts/` file, or guide calls `fanout`" covered only the in-repo `guide/`, which holds no chapters. + - Impact: once the target is pushed, every guide example of `fanout` fails at run time with a call to a nil global. + - Reversal cost: low, documentation only. Target state: the guide calls `tasks.fanout(worker, collection)`. + - Disposition: excluded by user decision. See Deferred and Out of Scope. +- Exposed debt, brought into scope by the user: + - D1-1: a sibling call's result turns "generating" back on while the model's own question is open. + - Evidence: `1ce5831d0` issues every leading catalog call in one step (`crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs`, `dispatch_tool_calls`), so `user-input/ask` and a sibling such as `web/fetch` are in flight together, and each sibling's `ToolResult` fires when it lands. In `crates/workshop/ui/src/services/agent-session.ts`, the `tool_call_update` case (lines 456-464) turns generating on for any result that `answersModelCall` (lines 601-615) matches, without looking at the pinned wait. The `generating` getter's doc (lines 299-311) says it is off while a wait is open. + - Impact: the composer action becomes Stop (`crates/workshop/ui/src/parts/agent/agent-session-view.ts` lines 279-291), Enter does nothing (`crates/workshop/ui/src/parts/chatbox/chat-box.ts` lines 522-526), and the operator must press Stop to reach Send. If another sibling is still running, that Stop drops it, the batch answers with the cancelled error, and the round's records, including the operator's answer, are discarded. + - Reachability: an operator-authored agent that offers `user-input/ask` to the model, a supported shape. The built-in chat agent does not. + - Reversal cost: low, local to the UI service. + - C1-1: two model questions in one round strand the first. + - Evidence: two `user-input/ask` calls in one round both issue at once. Each opens its own wait and sends `WaitFrame::Required` (`crates/workshop/agents/src/input-tool.rs` lines 89-103), and the server registry holds many waits in creation order (`crates/workshop/agents/src/input.rs` lines 81-87 and 120-131). The service keeps one pinned token, and `onInputRequired` overwrites it (`agent-session.ts` lines 259-279 and 632-648). After the operator answers, `respond` clears the pin (lines 340-375), and no frame re-announces the first wait until a reattach (`crates/workshop/server/src/agents/socket.rs` `resend_unresolved`). + - Impact: the run stays parked on the unanswered first question, and the composer has no token to answer it. `pendingInputToken` is documented as "The pending wait's token, or null while the agent is working", yet it returns null while a wait is open. + - Reversal cost: low, local to the UI service. +- Cheap fixes: none. +- Unrelated pre-existing debt: D1-17, the guide still calls `list_from_section`, which `d24da1d75` removed before the baseline. Excluded with D1-2. +- Rejected candidates: 15. + - Residual but acceptable, 9 (D1-3 to D1-8, D1-11, D1-12, D1-14): exhaustive protocol and dispatch arms, a private correlated parameter cluster, batch slots whose lifecycle every path enforces, reset lists updated in both places, a rename complete inside the repository, sibling store identities recorded as a known trade-off, Stop beside a call that survives stops as a semantic choice, and one stale doc phrase. + - Weak or speculative, 4 (D1-9, D1-10, D1-13, D1-15): duplicated routing checks with no reaching path, no contract that bounds calls per round, effect ids that shift only in multi-chain runs with no replayer, and a test-only single-call route. + - False, 1 (D1-16): teardown's orphans keep the run's scope open on purpose. + + + + +## Technical Design + +The agent session service tracks every open operator wait in the order the server announced it, instead of one pinned token, and stops treating a sibling call's result as proof the agent is working while a wait is open. The view, server, and wire format stay as they are. + +- In `crates/workshop/ui/src/services/agent-session.ts`: + - Open waits: keep the open wait tokens in announcement order. `pendingInputToken` returns the oldest open token, or null when none is open, which matches its doc. + - `onInputRequired(token)`: add the token unless it is already open, since a reattach resend repeats tokens. Keep today's closing of thinking, clearing of `stopped`, and turning generating off, then pin the oldest open token. + - `respond`, on success: remove the spent token, pin the next oldest, and turn generating on only when no wait remains open. A failed respond keeps every token, as today. + - `onInputCancelled(token)`: remove that token wherever it sits, and pin the next oldest. + - The attach acknowledgement: clear every open token before the server's `resend_unresolved` re-announces the unresolved waits in creation order, so the oldest is pinned again. + - `tool_call_update` (lines 456-464): turn generating on only when `answersModelCall(id)` holds and no wait is open. + - `setPinnedToken` (lines 632-648) stays the one place that publishes the pin, fed from the oldest open token, so the view's subscription is unchanged. +- Unchanged: `agent-session-view.ts` and `chat-box.ts`, which already read `generating` and `pendingInputToken`, the Workshop server and its frames, and the Engine and Harness. + + + + +## Testing Plan + +Each defect gets a regression test that fails at `83d40b1b9`. The existing pin, model-ask, and Stop tests are the regression suite for the paths this plan keeps. + +- D1-1, in `crates/workshop/ui/test/agent-session-service.mjs`, beside the model-ask block (lines 490-506), using `makeWire` and `AgentSessionService` as that block does: + - New test: fire `tool_call` for `[c1 user-input/ask, c2 web/fetch]`, then `inputRequired("tok1")`, then `tool_call_update` for `c2`. Expect `generating === false` and `pendingInputToken === "tok1"`. Then `respond` turns generating on. + - Fails before the change, because the update turns generating on. +- D1-1, in `crates/workshop/ui/test/agent-session-view.mjs`, beside the Stop tests (lines 1060-1085): under the same sequence, the composer's `data-action` stays `send`. +- C1-1, in `agent-session-service.mjs`, beside the pin tests (lines 266-294): + - Two open waits pin the oldest and advance: `inputRequired("tok1")` and `inputRequired("tok2")` leave `pendingInputToken === "tok1"`. Responding to `tok1` pins `tok2` with generating still off, and responding to `tok2` leaves null with generating on. Fails before the change, which pins `tok2` first and null after one respond. + - Cancelling a wait removes it wherever it sits: `tok1`, `tok2`, then `inputCancelled("tok1")` leaves `tok2`, and in a second fixture `inputCancelled("tok2")` leaves `tok1`. Only the second case fails before the change, which already pins `tok2` by overwrite. + - A reattach pins the oldest unresolved wait: `tok1`, `tok2`, the attach acknowledgement, then both `inputRequired` frames again leave `tok1`, with no duplicate token. +- Existing tests that must pass unchanged: "input_required pins", "a spent token unpins", "input_cancelled unpins", "an open wait keeps generating off", and the view's Stop tests (lines 1303-1342). +- Existing checks whose setup changes, in the model-ask block of `agent-session-service.mjs` (lines 490-525): "a tool_call_update answering a model batch call turns generating on", "a result for the current turn's call turns generating on", "an answered call turns it back on", and "a same-session acknowledgment leaves generating alone". Their setup leaves `tok1` and then `tok2` open when the tool results land, a state the shipped UI cannot reach. Each wait is cleared before its result lands, without turning `generating` on (for example with `inputCancelled`, not `respond`), so every check still sees a tool result turn `generating` from off to on. Their assertions stay as they are. +- Exit checks, in `crates/workshop`: `npm test --workspaces --if-present` and `npm run typecheck --workspaces --if-present`. From the repository root: `node --test crates/workshop/ui/test/docs-claims.mjs`. + + + + +## Decision Record + +The fix stays inside the UI service, mirroring the server's existing wait registry. The user brought the two exposed defects into scope and kept the separate docs repository out of it. + +- Reversible decisions and consequences: + - Track open waits in the service in announcement order and pin the oldest. Consequence: the composer answers questions in the order the model asked them, which matches the server registry and its resend order. + - Gate the `tool_call_update` generating flip on no open wait. Consequence: with no wait open, a landed sibling still turns generating on, as today. + - Edit the setup of four existing model-ask checks, not the gate. Raised when the coding sub-agent found that those checks leave a wait open when a tool result lands. Evidence: wait frames carry only a token, with no call id (`crates/workshop/agents/src/input.rs`, `crates/workshop/ui/src/services/agent-socket.ts` lines 249-253). In a single client, `respond` clears the pin before the ask's result arrives. The shipped desktop UI gives each agent panel its own socket and session, and uses attach only to reconnect (`crates/workshop/ui/src/parts/agent/agent-panel.ts`, `crates/workshop/ui/src/services/agent-socket.ts`). So no shipped flow lands a result under a wait this client still holds open. Falsifier: a shipped flow that attaches two sockets to one conversation at once. There, a peer whose question was answered elsewhere would keep `generating` off until the next reply, and the precise fix would carry the ask's call id on the wait frame, a wire change. + - Rejected: exempt an ask's own result from the gate. Two asks in one round would then turn `generating` on while the second question is still open. +- User-resolved choices: + - Scope expanded to D1-1 and C1-1. User selected "D1-1 and C1-1: fix the two Workshop UI bugs in agent-session.ts". + - `../promptforge-docs` is not edited. User: "leave promptforge-docs ALONE". This governs over the user's selection of the D1-17 guide pass, which edits the same repository, so D1-2 and D1-17 are out of scope. +- Rejected alternatives, assumptions, and risks: + - The server resends the remaining waits after each accepted response. Rejected: it adds a second resend trigger and server traffic, and the UI must still handle two open waits on attach. + - Turn generating on only when every call of the batch has a result. Rejected: it needs per-batch bookkeeping, and with no wait open, a landed sibling does mean the agent is working. + - Run calls that survive stops alone in the Engine. Rejected: the Engine would need a Host concept, and it would undo the rule that a catalog call's route never depends on its round. + - Assumption, inferred from code and not observed: the ask's wait usually opens before a network sibling lands. The fix holds in either order. + - Risk: D1-12 remains. A deliberate Stop while a question is open still drops running siblings and discards the round. Fixing D1-1 removes the forced Stop that made this routine. + +### Deferred and Out of Scope + +- D1-2, the guide's 32 bare `fanout(` calls in `../promptforge-docs`. Excluded by the user. Revisit before pushing `a490542e9`, because `.github/workflows/site.yml` publishes the guide beside the Engine, or whenever that repository is next updated. +- D1-17, the guide's `list_from_section` calls. Excluded with D1-2, with the same revisit condition. +- A durable guard against guide drift after author-facing renames. Declined by the user. Revisit at the next author-facing rename. +- D1-9, one shared route classifier for `runs_alone` and `prepare_tool_call`. Revisit when the subagents plan edits tool-call routing. +- D1-13, effect ids that shift in multi-chain runs. Revisit when a replayer is built. +- D1-10, a cap on calls in flight per round. Revisit if provider rate limits appear. +- D1-11's `ToolContext` doc sentence on sibling concurrency, and D1-14's stale `tool_call` phrase in `crates/promptforge-internal/lua/src/protocol.rs`. Revisit at the next edit of those files. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build --locked -p gateway`. + - `crates/gateway/app` is the workspace's only default member, so plain `cargo build` builds the same thing. The desktop app builds with `cargo workshop`, an alias for `run -p build-workshop --` in `.cargo/config.toml`. + - The Workshop UI bundle builds with `npm run build --workspace ui` in `crates/workshop`, which runs esbuild through `ui/build.mjs`. The `workshop-server` and `gateway-config-ui` build scripts run the same bundling, so workspace-wide cargo commands need Node.js plus the npm installs in `crates/workshop` and `crates/gateway/config-ui/ui`. Both `node_modules` trees are present on this machine, along with Node.js 24.19 (CI uses 22), cargo-nextest 0.9.128, stable Rust, and `nightly-2026-09-05`. + - Workshop clippy and tests need the gateway sidecar staged for Tauri's `externalBin`: `cargo build --locked -p gateway --no-default-features`, then `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe`, and `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc` afterward. `crates/workshop/desktop/binaries/` is empty now. The UI test suites do not need the sidecar. +- Focused test command pattern: + - Workshop UI: `node test/.mjs` from `crates/workshop/ui`, such as `node test/agent-session-service.mjs` or `node test/agent-session-view.mjs`. Each file is a standalone script and the smallest runnable unit, with no per-assertion filter. Both of those pass at `83d40b1b9`. The `look` and `platform` packages run their files the same way from their own directories. + - Rust: `cargo nextest run --locked -p --all-features `. Drop `--all-features` for `workshop`, `workshop-server`, and `workshop-server-api`, which CI never builds with all features. +- Component test command pattern: + - Workshop UI: `npm test --workspace ui` in `crates/workshop`, which runs `node --test "test/**/*.mjs" "src/**/*.test.mjs"`. The second glob matches nothing today, and the first also picks up `test/helpers/`. `--workspace @workshop/look` and `--workspace @workshop/platform` run the other two packages. + - Rust: `cargo nextest run --locked -p --all-features`, with the same three exceptions. `cargo test -p build-xtask` runs the boundary and structural checks. +- Full-suite test command: + - Workshop UI: `npm test --workspaces --if-present` in `crates/workshop`, plus `npm test` in `crates/gateway/config-ui/ui`. + - The wording and rulebook scan: `node --test crates/workshop/ui/test/docs-claims.mjs` from the repository root. It needs no `npm ci` and runs from any directory. + - Rust: `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` with the sidecar staged. CI also runs `-p workshop-workspace --all-features`, `-p workshop-server --features headless`, and five launch-lease tests by name. +- Linter command: + - Workshop UI: no ESLint or other JavaScript linter is configured, so the type check is the gate: `npm run typecheck --workspaces --if-present` in `crates/workshop`, which runs `tsc --noEmit` in `ui`, `look`, and `platform` (TypeScript 7, `strict`, `noUncheckedIndexedAccess`, `verbatimModuleSyntax`). All three pass at `83d40b1b9`. The `ui` tsconfig covers only `src`, so the `.mjs` tests are not type-checked. The gateway config UI runs `npm run typecheck` in `crates/gateway/config-ui/ui`. + - Rust: `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, both with `CARGO_BUILD_WARNINGS=deny` (in PowerShell, `$env:CARGO_BUILD_WARNINGS='deny'`). Never run a standalone `cargo check --workspace` beside them; the one exception is `cargo check -p gateway --no-default-features`. + - Supply chain, run in CI: `cargo deny check`, `cargo audit`, `cargo hakari verify`, and a `cargo tree` check that `ring` stays out of the gateway's normal closure. + - The pre-push hook in `.githooks/` (set as `core.hooksPath`) runs the wording scan, the headless gateway check, the non-Workshop clippy run, and `cargo deny` when installed. +- Formatter check command: `cargo fmt --all --check`, with `style_edition = "2024"` in `rustfmt.toml`, which the pre-commit hook runs. No formatter is configured for the TypeScript sources or `.mjs` tests, so new code matches the surrounding style by hand. +- Docs command: + - Each with `RUSTDOCFLAGS="-D warnings"` (in PowerShell, `$env:RUSTDOCFLAGS='-D warnings'`): `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api`, `cargo doc -p promptforge --no-deps`, `cargo doc -p harness --no-deps`, `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items`, and `cargo doc --locked --no-deps -p workshop-server --document-private-items`. + - The facade surface check: `cargo +nightly-2026-09-05 xtask api --check` against `crates/promptforge/public-api.txt`, with the nightly pinned in `crates/build-xtask/src/api/toolchain.rs`. + - The Workshop UI has no generated docs. Its module and test-file header comments are the documentation. +- Test placement and naming conventions: + - Workshop UI tests live flat in `crates/workshop/ui/test/` as one `.mjs` per subject (113 files), with shared modules such as `leak-check.mjs` and `boot.mjs` in `test/helpers/`. `look/test/` (9 files) and `platform/test/` (10 files) use the same flat placement, and some of their files assert with `node:assert/strict` instead. + - A UI test file opens with a comment stating what it covers, usually ending `Run: node test/.mjs` (65 files). It bundles the TypeScript under test with esbuild from an inline `stdin` entry resolved from the `ui` directory, writes the bundle to the OS temp directory, and imports it. DOM tests run under jsdom (73 files) with `.css` loaded as empty. + - Assertions go through a local `check(name, condition)` into a `failures` list (85 files), not `node:test` cases. The file prints `: all assertions passed`, or lists every failing name and exits 1. Check names are lowercase sentences stating the behavior, such as "a spent token unpins" and "input_required pins its token". + - Cases are scoped `{ }` blocks under `// --- Title ---` banners. Each builds its own fixtures, such as `makeWire()` and a fresh `AgentSessionService` in the agent-session tests, and disposes what it built. `assertNoLeaks` from `test/helpers/leak-check.mjs` checks disposal. + - Rust unit tests mostly sit in a sibling file wired with `#[cfg(test)] #[path = "-tests.rs"] mod tests;` (about 176 files). A module whose tests reach three files keeps them in a `tests/` subdirectory beside it. Inline `mod tests { }` blocks remain in 141 files. + - Rust integration tests compile as one binary per crate, from `tests/it/main.rs` in 18 crates and `tests/suite/main.rs` in `harness` and `promptforge`, one module per topic with helpers in `support.rs`. Test names are snake_case sentences stating the behavior. Doctests are banned by `build-xtask`'s `no_doctests` check. `.config/nextest.toml` puts `gateway-stt` and `gateway-stt-backend-whisper` in a `heavy` test group. +- Directory map: + - `crates/`: all 52 Cargo packages. The top level holds the public and shared layer: the `promptforge` facade, `promptforge-plugin`, `harness`, `harness-gateway-client`, `gateway-api-types`, `gateway-api-discovery`, the `plugin-*` crates, the `shared-*` crates, the `build-*` tooling, and `workspace-hack`. + - `crates/promptforge-internal/`: the Engine's private crates `types`, `engine`, `lua`, `parser`, `vfs`, and `model-client`. `crates/harness-internal/runner`: the Harness's private per-run runner. + - `crates/gateway/`: the gateway's private crates, plus the `stt/` subsystem. + - `crates/workshop/`: Workshop's Rust crates `desktop` (package `workshop`), `server`, `server-api`, `agents`, `gateway`, `menu`, `protocol`, `registry`, `run-log`, `status`, `support`, `user-state`, and `workspace`, plus the npm workspace root `package.json` holding the packages `ui`, `look`, and `platform`. + - `crates/workshop/ui/src/`: `base/` (paths), `services/` (30 service modules, including `agent-session.ts`, `agent-socket.ts`, and `protocol.ts`), `parts/` (14 feature directories, including `agent/` and `chatbox/`, plus `workbench.contributions.ts`), `tokens/` (component CSS tokens), and `main.ts`, the entry. `ui/test/` holds the UI tests. + - `crates/workshop/look/`: the shared look (palette, size and semantic tokens, icons, and behavioral components). `crates/workshop/platform/`: the shared UI mechanics (lifecycle, events, registries, `WorkshopPart`). + - `crates/shared-ui`: a TypeScript and CSS package the gateway config UI uses, excluded from the Cargo workspace. + - `guide/`: the `books`, `chrome`, and `landing` pieces of the mdBook site; the chapters live in the separate `promptforge-docs` checkout. `prompts/`: five example prompts. `tools/`: Node scripts for staging the gateway sidecar and a live TTS check, each with a `.test.mjs`. `images/`: README art. + - `vibe/`: plan records, recent ones at the top level and older ones filed by month (`2026-07` through `2026-10`), plus `scratch/`. + - `.github/workflows/`: `ci.yml`, whose `ci-green` job is the single gate (its `ui` job runs the UI typecheck, build, and tests), plus release, nightly, site, Miri, native-library, and installer smoke workflows. `.config/`: `nextest.toml` and `hakari.toml`. `.githooks/`: pre-commit and pre-push. `.cursor/rules/`: `workshop-architecture.mdc` and `workshop-spa.mdc`. + - Not tracked: `local/`, `target/`, `target-msrv/`, and `cabinet/`, which holds only an empty `_trash/`. `master` is at `83d40b1b9` (`Close plan: parallel-tool-calls`), three commits ahead of `origin/master`, with a clean worktree. +- Component boundaries: + - Rust, enforced by `cargo test -p build-xtask` across normal, dev, and build dependencies: the Engine (`promptforge*`) depends on no gateway, Workshop, Harness, or Plugin crate, and outside crates reach it only through `promptforge` and `promptforge-plugin`. The Harness chain runs `harness-gateway-client` on `harness` on `harness-runner` on the Engine pair. Plugin crates depend only on `promptforge-plugin`, `shared-*`, `workspace-hack`, and outside libraries. The gateway depends on no Engine, Workshop, Harness, or Plugin crate and is reached only through `gateway-api-types` and `gateway-api-discovery`. + - Workshop is a Host. Its Rust tiers depend one way: the server on features (`agents`, `workspace`, `user-state`, `run-log`), features on services (`menu`, `gateway`, `status`), and services on vocabulary (`protocol`, `registry`, `support`). The desktop app names only `workshop-server-api`, which names only `workshop-server`. + - Workshop UI layers: `base/`, `services/`, and `parts/` may all import `@workshop/platform`, which never imports the app. Services own no views; parts are feature directories whose panels extend `WorkshopPart`. A lazily loaded part has an `index.ts` chunk entry and never imports the eager entry bundle (`main.ts` and the `*.contribution.ts` modules). `look/test/boundary.mjs` and `platform/test/boundary.mjs` guard the shared packages. + - The agent panel's path: `services/agent-socket.ts` is the wire the service consumes, `services/agent-session.ts` (`AgentSessionService`) folds frames into transcript items, the pinned wait token, and `generating`, and `parts/agent/agent-session-view.ts` with `parts/chatbox/chat-box.ts` render from those. The tests drive the service through a scripted wire with the same surface. +- Conventions summary: + - Engine, Harness, Host, and Plugin are capitalized with one meaning each, and Engine crates say "the caller". `docs-claims.mjs` enforces the wording in every `AGENTS.md`, every `## Invariants` crate doc, and every `.cursor/rules` file. + - The Workshop UI copies Cursor's IDE for look, layout, wording, and behavior (`crates/workshop/ui/AGENTS.md`). Files and directories are kebab-case, CSS sits beside its TypeScript with the `.ws-` class prefix and only token values, every persisted value goes through the `ui-storage` adapter to the server, and panels, menus, commands, services, and keybindings self-register through the `@workshop/platform` registries. + - Error and status messages are written for a model to read: concise and naming what was required and what was found. Comments explain only non-obvious constraints, and every workaround cites its upstream issue URL. + - Behavior changes ship with tests in the same change, and refactors preserve existing behavior tests. + - Rust: clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny, and suppressions use `#[expect(..., reason = "...")]`. Every crate build script (51 of them) calls `build_ceiling::check()`, which fails any Rust file over 500 lines. 25 crate `lib.rs` files open with a `## Invariants` doc section. Source directories stay flat until a group reaches three files, with kebab siblings wired by `#[path]` below that. + - Commits have a one-line summary, a prose body, a bullet list of changes, optional `Design:` lines, and a `Plan: vibe/` trailer. A plan's closing commit is titled `Close plan: ` and holds only the trailer. + + + + +## Execution Instructions + + + +### Step 1: Track open waits oldest first and gate the generating flip [completed] + +- Component: none +- Covers: D1-1 and C1-1, both cleanup goals, the whole Technical Design, and the whole Testing Plan. D1-1's "no wait open" check reads the open-wait list that C1-1 introduces, so both defects land in this one step and one commit. +- Starting point: `master` at `83d40b1b9` with a clean worktree. Run commands from `crates/workshop/ui` unless a command says otherwise. +- Tests first, in `crates/workshop/ui/test/agent-session-service.mjs`. Write each case as a scoped `{ }` block under a `// --- Title ---` banner that builds its own `makeWire()` and `AgentSessionService`, disposes them, and asserts through the file's `check(name, condition)` with lowercase behavior names: + - Beside the model-ask block (lines 490-506), the D1-1 case: `tool_call` for `[c1 user-input/ask, c2 web/fetch]`, then `inputRequired("tok1")`, then `tool_call_update` for `c2`. Check `generating === false` and `pendingInputToken === "tok1"`, then check that `respond` turns `generating` on. + - Beside the pin tests (lines 266-294), three C1-1 cases: + - Two open waits pin the oldest and advance: `inputRequired("tok1")` then `inputRequired("tok2")` leaves `pendingInputToken === "tok1"`. Responding pins `tok2` with `generating` still off, and responding again leaves null with `generating` on. + - Cancelling a wait removes it wherever it sits: with `tok1` and `tok2` open, `inputCancelled("tok1")` leaves `tok2`, as the Testing Plan says. In a second fixture in the same block, `inputCancelled("tok2")` leaves `tok1`. The second assertion is what makes this block fail at `83d40b1b9`, because today's handler already pins `tok2` by overwrite, so the first assertion passes there. + - A reattach pins the oldest unresolved wait: `tok1`, `tok2`, the attach acknowledgement (a session frame with the same session id), then `inputRequired("tok1")` and `inputRequired("tok2")` again leave `tok1`. Prove there is no duplicate token by responding twice: the first respond pins `tok2`, the second leaves null. +- Tests first, in `crates/workshop/ui/test/agent-session-view.mjs`, beside the Stop tests (lines 1060-1085): drive the D1-1 sequence and check the composer's `data-action` stays `send`. +- Confirm the failures before touching source: run `node test/agent-session-service.mjs` and `node test/agent-session-view.mjs`. Every new block must report at least one failing check, and every existing check must still pass. Stop and re-plan if a new block passes or an existing check fails. +- Source, in `crates/workshop/ui/src/services/agent-session.ts`: + - Add a private list of open wait tokens in announcement order as the source of truth for the pin. `pendingInputToken` returns the oldest open token, or null when none is open, which matches its doc. + - The `wire.onInputRequired` handler (lines 259-270): append the token unless it is already open, keep closing thinking, clearing `stopped`, and turning `generating` off, then pin the oldest open token. + - `respond` (lines 340-355): on a successful send, remove the spent token, pin the next oldest, and call `setGenerating(true)` only when no wait remains open. A failed send keeps every token and folds the connection error, as today. + - The `wire.onInputCancelled` handler (lines 273-278): remove the cancelled token wherever it sits, then pin the oldest remaining token or null. + - `acknowledge` (lines 386-400): clear every open token where it now calls `setPinnedToken(null)`, so the server's `resend_unresolved` rebuilds the list in creation order. + - The `tool_call_update` case in `foldEvent` (lines 456-464): turn `generating` on only when `answersModelCall(toolCallId)` holds and no wait is open. + - `setPinnedToken` (lines 633-639) stays the only place that publishes the pin, always fed from the oldest open token, so the view's subscription is unchanged. + - Update the comments that describe a single pin - on the `onInputCancelled` handler, `respond`, `acknowledge`, and the `tool_call_update` case - so they describe the open-wait list and the gate. +- Leave unchanged: `agent-session-view.ts`, `chat-box.ts`, the Workshop server and its frames, the Engine, the Harness, and `../promptforge-docs`. Stop while a question is open keeps today's meaning (D1-12). D1-2 and D1-17 stay deferred. +- Verify: rerun the two focused files. Then in `crates/workshop` run `npm test --workspaces --if-present` and `npm run typecheck --workspaces --if-present`, and from the repository root run `node --test crates/workshop/ui/test/docs-claims.mjs`. All new checks pass, and the existing checks named in the Testing Plan ("input_required pins", "a spent token unpins", "input_cancelled unpins", "an open wait keeps generating off", and the view's Stop tests at lines 1303-1342) pass without edits. +- Setup edit to four existing checks in the model-ask block of `agent-session-service.mjs` (lines 490-525): "a tool_call_update answering a model batch call turns generating on", "a result for the current turn's call turns generating on", "an answered call turns it back on", and "a same-session acknowledgment leaves generating alone". Clear `tok1` before `c1`'s `tool_call_update`, and `tok2` before the next turn's `user_message`, in a way that leaves `generating` off (for example `inputCancelled`, not `respond`, since `respond` already turns `generating` on and would make the checks vacuous). Keep every assertion as it is, and confirm each still fails if the `tool_call_update` case stops turning `generating` on. +- Commit: one commit holding the source change and both test files, in the repository format: a one-line summary, a prose body, a bullet list of changes, `Design:` lines for the two reversible decisions (oldest-first open waits, and the generating flip gated on no open wait), and a `Plan: vibe/2026-10-11-4-workshop-waits.md` trailer. Nothing is pushed. + + + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..b44f79283 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-10-11-4-workshop-waits.md From fa59d5f41176dc2e58a2b237713b88b1775b5d5b Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 13:42:23 -0700 Subject: [PATCH 25/34] Close plan: workshop-waits Plan: vibe/2026-10-11-4-workshop-waits.md --- vibe/2026-10-11-4-workshop-waits.md | 4 ++-- vibe/ACTIVE | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/2026-10-11-4-workshop-waits.md b/vibe/2026-10-11-4-workshop-waits.md index 5dac62baa..b5b573d8b 100644 --- a/vibe/2026-10-11-4-workshop-waits.md +++ b/vibe/2026-10-11-4-workshop-waits.md @@ -4,10 +4,10 @@ overview: "Remove the debt that parallel tool calls exposed in the Workshop agen todos: - id: ui-waits content: "D1-1 and C1-1: track open waits oldest first in agent-session.ts, gate the tool_call_update generating flip on no open wait, and add the service and view regression tests" - status: pending + status: completed - id: verify content: Run npm test and typecheck in crates/workshop and the docs-claims scan; confirm each new test fails at 83d40b1b9 - status: pending + status: completed isProject: false --- diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index b44f79283..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-10-11-4-workshop-waits.md From 496d15ada64b049aca409d81f3ca9bd80225fc2d Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 15:07:52 -0700 Subject: [PATCH 26/34] Decide a tool call's route once and type its call site The scheduler now decides where a tool call is answered in one place, so a call sent alone and a call sent in a batch are no longer routed by two copies of the same checks. Catalog preparation moves into its own function that the batch calls directly, which retires the internal error the batch raised when a catalog call came back answered instead of issued. The three loose options that said who asked for a call and where its answer goes become one private enum, so a batch slot can exist only beside a model call id. Script calls, lone model calls, and batched model calls keep the same call ids, turns, caller origins, and answer paths. - `tool_route` decides each call's `ToolRoute` as `Builtin`, `Local`, or `Catalog`, checking reserved names before local tools. Both `prepare_tool_call` and `dispatch_tool_calls` match on it. - `prepare_catalog_call` holds the catalog work split out of `prepare_tool_call`: the binding lookup, the attempt count, the access fork, the origin, and the issue. It returns `Result<()>`, and the batch calls it with a `CallSite::Batch`. - `CallSite` replaces the `call_id`, `turn`, and `slot` options with `Script`, `Model`, and `Batch` variants, and `ToolCallContinuation` stores it in place of its `call_id` and `slot` fields. Only `Batch` carries a slot, and `Script` carries no call id. - `section_call_counts` returns the tool set snapshot and the section's counts for the local and catalog arms, and `call_turn` resolves a site's turn for every arm. - `CallSite::Batch` is the only site whose answer fills a batch slot, and `Model` and `Batch` both take the model-issued answer branch under their call id. A `Script` site still issues with `ToolCaller::Script`, and `Model` and `Batch` sites with `ToolCaller::Model`. - `tool_route` now runs before the tool set snapshot and the counts install. When several of these lookups fail at once, a frame or local-tool lookup error surfaces first. - `runs_alone` is deleted, and with it the batch's internal error "a batched catalog call is issued as an effect". - `ToolRoute` stays private to its module and `CallSite` to the scheduler, and no test file is touched. Design: new parameter-object @ crates/promptforge-internal/engine/src/execute/scheduler/pending.rs::CallSite Design: removes oversized-unit @ crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs::prepare_tool_call Plan: vibe/2026-10-11-5-easy-fixes.md --- .../engine/src/execute/scheduler.rs | 2 +- .../engine/src/execute/scheduler/apply.rs | 20 +- .../engine/src/execute/scheduler/pending.rs | 50 ++- .../engine/src/execute/scheduler/tool_call.rs | 308 +++++++------ vibe/2026-10-11-5-easy-fixes.md | 425 ++++++++++++++++++ vibe/ACTIVE | 1 + 6 files changed, 661 insertions(+), 145 deletions(-) create mode 100644 vibe/2026-10-11-5-easy-fixes.md create mode 100644 vibe/ACTIVE diff --git a/crates/promptforge-internal/engine/src/execute/scheduler.rs b/crates/promptforge-internal/engine/src/execute/scheduler.rs index a0ed00d51..27515cfa9 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler.rs @@ -96,7 +96,7 @@ use super::context::RunState; use super::protocol::Answer; use super::run::{Effect, EffectAnswer, EffectId, Round}; use chain_record::{Chain, ChatAnchor}; -use pending::{Continuation, Pending, ToolCallContinuation, VfsContinuation}; +use pending::{CallSite, Continuation, Pending, ToolCallContinuation, VfsContinuation}; use tasks::TaskSlot; pub(in crate::execute) use tool_call::RESERVED_TOOL_NAMES; diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/apply.rs b/crates/promptforge-internal/engine/src/execute/scheduler/apply.rs index d269d1774..2b72abb73 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/apply.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/apply.rs @@ -29,7 +29,8 @@ use crate::{Error, Result}; use super::dispatch::classify_vfs_failure; use super::tasks::{TaskBacking, TaskState}; use super::{ - ChainIndex, Continuation, EffectAnswer, EffectId, Pending, Scheduler, ToolCallContinuation, + CallSite, ChainIndex, Continuation, EffectAnswer, EffectId, Pending, Scheduler, + ToolCallContinuation, }; /// The cancelled answer for a chain parked on `resume`'s kind of effect: @@ -80,7 +81,10 @@ impl Scheduler { let slot = match &resume { Continuation::ToolCall(call) => { self.end_tool_call(chain, call.exec); - call.slot + match call.site { + CallSite::Batch { slot, .. } => Some(slot), + CallSite::Script { .. } | CallSite::Model { .. } => None, + } } _ => None, }; @@ -177,11 +181,11 @@ impl Scheduler { let emitter = chain.ctx.emitter(); let section = chain.section_name(); let nonce = chain.ctx.nonce(); - match &call.call_id { - // Model-issued: the content always resumes, plain - it is the - // tool record's text for the next round, never classified by - // output kind. - Some(call_id) => { + match &call.site { + // Model-issued, alone or batched: the content always resumes, + // plain - it is the tool record's text for the next round, + // never classified by output kind. + CallSite::Model { call_id, .. } | CallSite::Batch { call_id, .. } => { let report = ModelReport { script: call.report, call_id: call_id.clone(), @@ -198,7 +202,7 @@ impl Scheduler { .map(|outcome| ToolCallOutcome::Plain(outcome.into_content())) .map_err(Error::from) } - None => match prepare_dispatch( + CallSite::Script { .. } => match prepare_dispatch( &call.binding, result, None, diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/pending.rs b/crates/promptforge-internal/engine/src/execute/scheduler/pending.rs index e4d26217c..649978c00 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/pending.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/pending.rs @@ -54,23 +54,57 @@ pub(super) struct VfsContinuation { /// What a bound `tool_call`'s answer is applied with: the binding the call /// resolved to (its alias, output kind, and trust rules), the coordinates -/// the `ToolResult` reports under, the model's call id when the model -/// issued the call, and the call's identity the answer joins. +/// the `ToolResult` reports under, the call's site, and the call's +/// identity the answer joins. pub(super) struct ToolCallContinuation { /// The binding the alias resolved to at dispatch. pub(super) binding: ToolBinding, /// The turn the call fired in. pub(super) report: ScriptReport, - /// The model-issued call id, or `None` for a script call. - pub(super) call_id: Option, + /// Who asked for the call, and where its answer goes. + pub(super) site: CallSite, /// The call's own identity, forked from the chain's access at /// dispatch. The answer, or the call's abort, ends it and joins it /// back into the chain. pub(super) exec: ExecId, - /// The call's position in the batch the chain is parked on, whose - /// slot the answer fills; `None` for a call sent alone, which the - /// answer resumes directly. - pub(super) slot: Option, +} + +/// Who asked for one tool call, and where its answer goes. Only a +/// model-issued call carries a call id, and only a batched one a slot. +pub(super) enum CallSite { + /// A script's `tools.call`, under the turn its request named, if any. + /// The answer resumes the chain directly, classified by the binding's + /// declared output kind, and reports under no call id. + Script { turn: Option }, + /// A model-issued call sent alone, under the turn of the round that + /// requested it, if the request named one. The answer resumes the + /// chain directly as content under the model's call id. + Model { call_id: String, turn: Option }, + /// A model-issued call sent in a `tool_calls` batch, under the round's + /// turn. The answer fills `slot` of the batch the chain is parked on. + Batch { + call_id: String, + turn: u32, + slot: usize, + }, +} + +impl CallSite { + /// The model's call id, or `None` for a script call. + pub(super) fn call_id(&self) -> Option<&str> { + match self { + Self::Script { .. } => None, + Self::Model { call_id, .. } | Self::Batch { call_id, .. } => Some(call_id.as_str()), + } + } + + /// The turn the request named, if any. + pub(super) fn turn(&self) -> Option { + match self { + Self::Script { turn } | Self::Model { turn, .. } => *turn, + Self::Batch { turn, .. } => Some(*turn), + } + } } /// One in-flight leaf effect's pending entry: the chain parked on it and diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs b/crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs index 40dacc2be..d344ccbb7 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs @@ -3,32 +3,32 @@ //! model's `call_id`), plus the `local_tool_done` arm that closes a local //! tool call. //! -//! Three things resolve on the driver thread before any leaf work: the -//! four reserved model built-in names (`task`, `task_cancel`, `task_status`, -//! `await_tasks`) are recognized before any lookup - a model-issued call -//! to the first three is answered by the `builtins` module over the task -//! arena, `await_tasks` by its own module (answered at once or parked on -//! the chain's model tasks), and a script call to any of them answers as -//! unbound; a local Lua tool is answered with its handler; a catalog tool +//! Each call's route is decided once, by `tool_route`, on the driver +//! thread before any leaf work. The four reserved model built-in names +//! (`task`, `task_cancel`, `task_status`, `await_tasks`) are recognized +//! before any lookup - a model-issued call to the first three is answered +//! by the `builtins` module over the task arena, `await_tasks` by its own +//! module (answered at once or parked on the chain's model tasks), and a +//! script call to any of them answers as unbound; a local Lua tool is +//! answered with its handler; anything else is a catalog tool, which //! resolves against the run's offering (a model call by wire name, a //! script call by canonical id, which also reaches a catalog tool left -//! out of the offering), its attempt is counted, and the call is -//! issued as a `ToolCall` effect, carrying an identity forked from the -//! chain's access, whose answer the driver applies through -//! the shared dispatch body. `call_id: Some` always resumes a bound tool -//! with content - a tool's own failure becomes untrusted failure text - -//! and `ToolResult` fires under the id and the turn of the round that -//! requested the call; `call_id: None` keeps the raise-at-call-site -//! behavior, and its `ToolResult` fires under no id and the turn at -//! dispatch. +//! out of the offering), has its attempt counted, and is issued as a +//! `ToolCall` effect, carrying an identity forked from the chain's +//! access, whose answer the driver applies through the shared dispatch +//! body. The call's `CallSite` names who asked. A model-issued call, +//! alone or batched, always resumes a bound tool with content - a tool's +//! own failure becomes untrusted failure text - and `ToolResult` fires +//! under the id and the turn of the round that requested the call; a +//! script call keeps the raise-at-call-site behavior, and its +//! `ToolResult` fires under no id and the turn at dispatch. //! //! The loop sends a round's unanswered calls together as one //! `tool_calls` request. The arm walks them in the model's order and //! issues each catalog call at once into a slot of the chain's batch, -//! stopping at the first call that must run alone: a reserved built-in -//! name or a local tool, the two checks `prepare_tool_call` makes before -//! any catalog lookup. A call that fails to prepare fills its slot with -//! the error and stops the batch too. Each landed answer fills its call's +//! stopping at the first call whose route is a reserved built-in name or +//! a local tool. A call that fails to route or to prepare fills its slot +//! with the error and stops the batch too. Each landed answer fills its call's //! slot, and the chain is answered once no slot is open: every text in //! call order, or the first failure in call order. Fewer texts than //! calls tells the loop where the batch stopped, and the loop sends that @@ -51,7 +51,7 @@ use std::sync::atomic::Ordering; use crate::execute::protocol::{Answer, LocalToolOutcome, ToolCallOutcome, ToolCallRecord}; use crate::execute::run::{Effect, ToolCallOrigin, ToolCaller}; use crate::execute::section_context::LocalCall; -use crate::lua::{ScriptReport, current_tool_bindings}; +use crate::lua::{ScriptReport, ToolCallCounts, ToolSet, current_tool_bindings}; use crate::{Error, Result}; use promptforge_types::event::lifecycle; use promptforge_types::tools::{OutputTrust, ToolId}; @@ -59,7 +59,7 @@ use promptforge_vfs::detail::{access_id, access_spawn}; use super::builtins::is_task_builtin; use super::dispatch::unbound_tool_call; -use super::{ChainIndex, Continuation, Scheduler, ToolCallContinuation, prompt_origin}; +use super::{CallSite, ChainIndex, Continuation, Scheduler, ToolCallContinuation, prompt_origin}; /// The model built-in names the `tasks` namespace answers from this arm, /// recognized before any lookup so no catalog or local tool can shadow @@ -74,9 +74,7 @@ pub(in crate::execute) const RESERVED_TOOL_NAMES: [&str; 4] = /// (a local Lua tool's handler, a task built-in), such an answer plus the /// task chain a `task` built-in started, enqueued behind the caller, or /// the chain parked in the model's `await_tasks` on its live tasks, -/// answered when one ends or its timer fires. A batched call resolves -/// only as `Issued`: the batch stops before any call that would resolve -/// another way. +/// answered when one ends or its timer fires. pub(super) enum ToolCallDispatch { Issued, Answered(Answer), @@ -84,6 +82,20 @@ pub(super) enum ToolCallDispatch { Parked, } +/// Where one tool call is answered, decided before any catalog lookup so +/// a lone call and a batched one route alike. +enum ToolRoute { + /// A reserved model built-in name: a model-issued task built-in is + /// answered over the task arena, and anything else is unbound. + Builtin, + /// A local Lua tool of the chain's section VM, answered with its + /// handler. + Local, + /// A catalog tool, resolved against the run's offering and issued as + /// an effect. + Catalog, +} + impl Scheduler { /// Dispatches a `tool_call` request. An issued bound call parks the /// chain in the pending table; a local Lua tool's handler resumes the @@ -99,7 +111,11 @@ impl Scheduler { call_id: Option, turn: Option, ) { - match self.prepare_tool_call(id, alias, args, call_id, turn, None) { + let site = match call_id { + Some(call_id) => CallSite::Model { call_id, turn }, + None => CallSite::Script { turn }, + }; + match self.prepare_tool_call(id, alias, args, site) { // An issued call is parked on the pending table; a parked // wait was recorded on the chain, and a member's end or the // timer's firing answers it. @@ -120,12 +136,12 @@ impl Scheduler { } /// Dispatches a `tool_calls` request: the unanswered calls of one - /// model round, in the model's order. Each call before the first that - /// must run alone is prepared under the round's `turn` and its own - /// slot, so every catalog call issues in this step. A call that fails - /// to prepare fills its slot with the error and stops the batch. The - /// slots park on the chain, which is answered at once when none is - /// open. + /// model round, in the model's order. Each call before the first + /// whose route is not the catalog is issued as a `CallSite::Batch` + /// under the round's `turn` and its own slot, so every catalog call + /// issues in this step. A call that fails to route or to prepare + /// fills its slot with the error and stops the batch. The slots park + /// on the chain, which is answered at once when none is open. pub(super) fn dispatch_tool_calls( &mut self, id: ChainIndex, @@ -134,23 +150,18 @@ impl Scheduler { ) { let mut batch = Vec::with_capacity(calls.len()); for (slot, call) in calls.into_iter().enumerate() { - let issued = match self.runs_alone(id, &call.name) { - Ok(true) => break, - Ok(false) => self - .prepare_tool_call( - id, - &call.name, - call.arguments, - Some(call.id), - Some(turn), - Some(slot), - ) - .and_then(|dispatch| match dispatch { - ToolCallDispatch::Issued => Ok(()), - _ => Err(Error::internal( - "a batched catalog call is issued as an effect", - )), - }), + let issued = match self.tool_route(id, &call.name) { + Ok(ToolRoute::Builtin | ToolRoute::Local) => break, + Ok(ToolRoute::Catalog) => self.prepare_catalog_call( + id, + &call.name, + call.arguments, + CallSite::Batch { + call_id: call.id, + turn, + slot, + }, + ), Err(error) => Err(error), }; match issued { @@ -165,18 +176,45 @@ impl Scheduler { self.answer_full_batch(id); } - /// Whether the call named `name` must run alone rather than in a - /// batch: a reserved built-in name, or a local tool of the chain's - /// section VM. - fn runs_alone(&self, id: ChainIndex, name: &str) -> Result { - if RESERVED_TOOL_NAMES.contains(&name) { - return Ok(true); + /// Decides the route of the call named `alias`, the one decision a + /// lone call and a batch share. The reservation wins over every + /// lookup: a bound or local tool registered under a reserved name is + /// never reachable. A local tool of the chain's section VM wins over + /// the catalog. + fn tool_route(&self, id: ChainIndex, alias: &str) -> Result { + if RESERVED_TOOL_NAMES.contains(&alias) { + return Ok(ToolRoute::Builtin); } let frame = self.chains[id.index()] .frame .as_ref() .ok_or(Error::internal("a live chain holds its frame"))?; - Ok(frame.vm()?.has_local_tool(name)?) + if frame.vm()?.has_local_tool(alias)? { + return Ok(ToolRoute::Local); + } + Ok(ToolRoute::Catalog) + } + + /// The turn every report of a call at `site` carries: the requesting + /// round's turn when the site names one, else the chain's turn + /// counter at dispatch. + fn call_turn(&self, id: ChainIndex, site: &CallSite) -> u32 { + site.turn() + .unwrap_or_else(|| self.chains[id.index()].ctx.turns().load(Ordering::Relaxed)) + } + + /// The chain's tool set and its section's tool-call counts, which the + /// section's first call installs. + fn section_call_counts(&mut self, id: ChainIndex) -> Result<(ToolSet, ToolCallCounts)> { + let chain = &mut self.chains[id.index()]; + let tool_set = chain.ctx.tool_set_snapshot()?; + let frame = chain + .frame + .as_mut() + .ok_or(Error::internal("a live chain holds its frame"))?; + let effective = current_tool_bindings(&tool_set, &frame.vm()?.tool_runtime)?; + let counts = frame.script_call_counts(&chain.ctx, &effective)?; + Ok((tool_set, counts)) } /// Fills `slot` of the batch the chain is parked on with a landed @@ -227,82 +265,98 @@ impl Scheduler { self.answer_inline(id, Answer::ToolCallsResult(answer)); } - /// The fallible half of tool-call dispatch: the reserved-name check - /// (a model-issued task built-in answered over the arena, a script - /// call to a reserved name unbound), the one-time counts install, the - /// local tool's handler answer, then the catalog tool resolved: a - /// model call's wire name against the run's offering, a script call's - /// canonical id against the offering and then the full catalog, the - /// id then standing as the alias of a tool the run could not offer - /// (the section's effective scope shapes what the model is offered, - /// and the author's own script is not the model, so the scope does - /// not gate it); then the attempt counted under the tool's id, the - /// call's identity forked from the chain's access, and the issued - /// effect, whose origin names the model as caller when a `call_id` is - /// present and the script otherwise. The - /// answer's rules - the model-issued body under a `call_id`, else the - /// script body classified by the binding's declared output kind - are - /// the continuation's, applied when the answer lands. + /// The fallible half of tool-call dispatch, by the call's route: a + /// reserved name (a model-issued task built-in answered over the + /// arena, a script call to a reserved name unbound), a local tool (the + /// one-time counts install, the attempt counted, then the handler's + /// answer), or a catalog tool, which `prepare_catalog_call` issues. /// - /// Every report the call makes carries `turn`, the turn of the model - /// round that requested it, so a local handler earlier in the same - /// batch that runs model rounds cannot shift it. A script call, or a - /// model-issued one yielded without a turn, reports the counter's value - /// at dispatch. `slot` is the call's position in a `tool_calls` batch, - /// kept on the continuation so the answer fills that slot. + /// Every report the call makes carries the turn of the model round + /// that requested it, so a local handler earlier in the same batch + /// that runs model rounds cannot shift it. A script call, or a + /// model-issued one yielded without a turn, reports the counter's + /// value at dispatch. fn prepare_tool_call( &mut self, id: ChainIndex, alias: &str, args: serde_json::Value, - call_id: Option, - turn: Option, - slot: Option, + site: CallSite, ) -> Result { - let chain = &self.chains[id.index()]; - let tool_set = chain.ctx.tool_set_snapshot()?; - let turn = turn.unwrap_or_else(|| chain.ctx.turns().load(Ordering::Relaxed)); - // The reservation wins over every lookup: a bound or local tool - // registered under one of these names is never reachable here. The - // built-ins serve the model; the author's own script reaches the - // arena through the `tasks` namespace, so a script call stays - // unbound. - if RESERVED_TOOL_NAMES.contains(&alias) { - if let Some(call_id) = call_id.as_deref().filter(|_| is_task_builtin(alias)) { - return self.answer_task_builtin(id, alias, &args, call_id, turn); + match self.tool_route(id, alias)? { + // The built-ins serve the model; the author's own script + // reaches the arena through the `tasks` namespace, so a script + // call stays unbound. + ToolRoute::Builtin => { + let tool_set = self.chains[id.index()].ctx.tool_set_snapshot()?; + if let Some(call_id) = site.call_id().filter(|_| is_task_builtin(alias)) { + let turn = self.call_turn(id, &site); + return self.answer_task_builtin(id, alias, &args, call_id, turn); + } + Err(unbound_tool_call(&tool_set, alias)) + } + ToolRoute::Local => { + let report = ScriptReport { + turn: self.call_turn(id, &site), + }; + let (_, counts) = self.section_call_counts(id)?; + // The attempt counts at dispatch, before the handler runs, so + // a handler that then fails still counts. + counts.ensure(alias)?; + counts.increment(alias)?; + self.chains[id.index()] + .frame + .as_mut() + .ok_or(Error::internal("a live chain holds its frame"))? + .push_local_call(LocalCall { + alias: alias.to_owned(), + call_id: site.call_id().map(str::to_owned), + report, + }); + Ok(ToolCallDispatch::Answered(Answer::ToolCallResult(Ok( + ToolCallOutcome::Local { + alias: alias.to_owned(), + args, + }, + )))) + } + ToolRoute::Catalog => { + self.prepare_catalog_call(id, alias, args, site)?; + Ok(ToolCallDispatch::Issued) } - return Err(unbound_tool_call(&tool_set, alias)); - } - let chain = &mut self.chains[id.index()]; - let ctx = chain.ctx.clone(); - let report = ScriptReport { turn }; - let frame = chain - .frame - .as_mut() - .ok_or(Error::internal("a live chain holds its frame"))?; - let effective = current_tool_bindings(&tool_set, &frame.vm()?.tool_runtime)?; - let counts = frame.script_call_counts(&ctx, &effective)?; - if frame.vm()?.has_local_tool(alias)? { - // The attempt counts at dispatch, before the handler runs, so - // a handler that then fails still counts. - counts.ensure(alias)?; - counts.increment(alias)?; - frame.push_local_call(LocalCall { - alias: alias.to_owned(), - call_id, - report, - }); - return Ok(ToolCallDispatch::Answered(Answer::ToolCallResult(Ok( - ToolCallOutcome::Local { - alias: alias.to_owned(), - args, - }, - )))); } + } + + /// Issues a catalog call: the one-time counts install, then the tool + /// resolved - a model call's wire name against the run's offering, a + /// script call's canonical id against the offering and then the full + /// catalog, the id then standing as the alias of a tool the run could + /// not offer (the section's effective scope shapes what the model is + /// offered, and the author's own script is not the model, so the scope + /// does not gate it); then the attempt counted under the tool's id, + /// the call's identity forked from the chain's access, and the issued + /// effect, whose origin names the model as caller for a `Model` or + /// `Batch` site and the script for a `Script` site. The answer's + /// rules - the model-issued body under the call id, else the script + /// body classified by the binding's declared output kind - and a + /// `Batch` site's slot are the continuation's, applied when the answer + /// lands. + fn prepare_catalog_call( + &mut self, + id: ChainIndex, + alias: &str, + args: serde_json::Value, + site: CallSite, + ) -> Result<()> { + let report = ScriptReport { + turn: self.call_turn(id, &site), + }; + let (tool_set, counts) = self.section_call_counts(id)?; + let ctx = &self.chains[id.index()].ctx; // A model reaches only the wire names a round advertised; a script // names a catalog tool by its canonical id, and only a script falls // back to a catalog tool the run could not offer. - let binding = if call_id.is_some() { + let binding = if site.call_id().is_some() { tool_set.wire_binding(alias) } else if ToolId::parse(alias).is_ok() { tool_set @@ -335,12 +389,11 @@ impl Scheduler { .map_err(Error::store)?; let exec = access_id(&access); let origin = ToolCallOrigin { - execution: ctx.execution().to_owned(), + execution: chain.ctx.execution().to_owned(), section: chain.section_name().to_owned(), - caller: if call_id.is_some() { - ToolCaller::Model - } else { - ToolCaller::Script + caller: match &site { + CallSite::Script { .. } => ToolCaller::Script, + CallSite::Model { .. } | CallSite::Batch { .. } => ToolCaller::Model, }, }; let effect = Effect::ToolCall { @@ -353,12 +406,11 @@ impl Scheduler { let resume = Continuation::ToolCall(ToolCallContinuation { binding, report, - call_id, + site, exec, - slot, }); self.issue(id, effect, resume); - Ok(ToolCallDispatch::Issued) + Ok(()) } /// Dispatches a `local_tool_done` request: closes the chain's diff --git a/vibe/2026-10-11-5-easy-fixes.md b/vibe/2026-10-11-5-easy-fixes.md new file mode 100644 index 000000000..d65c32eff --- /dev/null +++ b/vibe/2026-10-11-5-easy-fixes.md @@ -0,0 +1,425 @@ +--- +name: Easy fixes after parallel calls +overview: "Land the small fixes found after the parallel-tool-calls run: two private routing refactors in the Engine scheduler, four doc and test-file fixes in promptforge, the tasks.fanout spec entry in promptforge-design, and four fixes to the vibe coder and debt collector in tools-public." +todos: + - id: route + content: "Step 1: ToolRoute and tool_route shared by prepare_tool_call and the batch; prepare_catalog_call; delete runs_alone and the batch fallback arm" + status: pending + - id: callsite + content: "Step 1: private CallSite enum through prepare_tool_call, ToolCallContinuation, and apply.rs; ToolCaller unchanged" + status: pending + - id: text + content: "Step 2: ToolContext sibling sentence, vibe record effect-id correction, protocol.rs line, parse_tool_call.rs split" + status: pending + - id: spec + content: "Step 2: tasks.fanout spec entry and review example describe today's behavior, committed alone in promptforge-design" + status: pending + - id: tools + content: "Step 2: vibe-coder.md decomposition fixes and debt-collector.md test check, one tools-public commit per file" + status: pending + - id: verify + content: "Each step: Testing Plan checks plus both nextest runs, both clippy runs, fmt, docs, xtask api --check, build-xtask, docs-claims" + status: pending +isProject: false +--- + +# Easy fixes after the parallel tool calls run + +### Context for a fresh reader + +- Repositories: + - `c:\Users\Vinnie\cursor\promptforge` (git, branch `master`). Paths are relative to it unless they start with `../`. + - `../promptforge-design` (git, HEAD `a34adaa`). It holds unrelated changes under `research/` (one modified file, one untracked file) that must never be staged. + - `../tools-public` (git, clean). +- Baseline: `promptforge` commit `fa59d5f41` (Close plan: workshop-waits). Every line number here was verified there; search by name if they have moved. +- Shell: PowerShell on Windows. Set an environment variable for one command with `$env:NAME='value'; `. +- Sources of the fixes: a debt analysis over the parallel-tool-calls commits (`1ce5831d0`, `a490542e9`), whose candidate IDs D1-5, D1-9, D1-11, D1-13, and D1-14 appear below, and the run ledgers of the parallel-tool-calls and workshop-waits plans. +- Terms: + - *route*: what a model's tool call is, as the Engine scheduler decides it: a task built-in (`task`, `task_cancel`, `task_status`, `await_tasks`), a local Lua tool (`tools.offer_local`), or a catalog call (a Plugin tool, issued as an `Effect::ToolCall`). + - *batch*: the leading catalog calls of a model round's unanswered calls, issued together. + - *vibe coder* and *debt collector*: planning tools in `../tools-public/planning/`. The vibe coder runs a plan as tested commits through sub-agents; its coding, review, and fix sub-agents read only the plan's implementation contract, project survey, and their own step. +- Plan record and commits: + - When execution starts, copy this plan to `vibe/` as `2026-10-11-5-easy-fixes.md`, the next free number for that day (the latest record is `vibe/2026-10-11-4-workshop-waits.md`). Mark each step `[completed]` in that copy as it lands. + - Each `promptforge` commit message has a one-line summary, a body, `Design:` lines, and a `Plan: vibe/2026-10-11-5-easy-fixes.md` trailer. Close with a commit `Close plan: easy-fixes` with the same trailer. + - Edits in `../promptforge-design` and `../tools-public` are committed in those repositories on their own. The coding sub-agent leaves them unstaged. The session commits each named file alone after the coding sub-agent returns and before the step's `promptforge` commit is finalized, and amends that commit after any later round that changes the file. + - Nothing is pushed. + + + +## Product Requirements + +This plan lands the small fixes found after the parallel-tool-calls run. In the Engine scheduler, one routing decision replaces two copies of it, and an enum replaces three correlated options. Four doc and test-file fixes go into `promptforge`, the `tasks.fanout` spec entry is brought up to today's behavior, and four gaps in the vibe coder and debt collector are closed. No author-visible behavior changes. + +- Problem and users: + - Engine maintainers: the scheduler's batch decides a call's route with a copy of the checks `prepare_tool_call` makes. If the two ever disagreed, the batch's fallback would run after `prepare_tool_call` had already pushed a local call or started a task (D1-9). `prepare_tool_call` also takes `call_id`, `turn`, and `slot` as three independent options, though a slot without a call id is meaningless (D1-5). + - Plugin authors: the `ToolContext` docs don't say that calls from one model round run concurrently with each other (D1-11). + - Readers of the plan record and the spec: + - `vibe/2026-10-11-3-parallel-tool-calls.md` claims a batch keeps today's effect ids, which holds only in a run with one live chain (D1-13). + - `crates/promptforge-internal/lua/src/protocol.rs` still names only the `chat` and `tool_call` rounds (D1-14). + - The spec's `tasks.fanout` entry describes an old target contract instead of what ships. + - The next test author: `crates/promptforge-internal/lua/src/protocol/tests/parse.rs` is at 499 lines, one short of the build's 500-line ceiling. + - Operators of the planning tools: in both recent runs, the session had to copy test cases and error texts into steps by hand, because the vibe coder's sub-agents can't read them. Decomposition also assigned the plan's seed copy to a step, and the verify instructions name a step's "Tests line" that nothing defines. Separately, the debt collector planned a regression test that already passed at its endpoint. +- Goals: + - One route decision shared by the batch and `prepare_tool_call`, and a batch that prepares only catalog calls, so it has no fallback arm. + - A private enum that makes a slot without a call id unrepresentable. + - The four `promptforge` text fixes, the spec entry, and the four tool fixes described in the Functional Specification. +- Non-goals: + - Any change to `../promptforge-docs`. User: "I dont care about promptforge-docs". + - A cap on tool calls in flight, what Stop means while a question is open, a call id on wait frames, and the test-only `tools.call_as_model` route. + - Any change to the public `ToolCaller` enum (`crates/promptforge-internal/types/src/tools/origin.rs` lines 21-30) or to `crates/promptforge/public-api.txt`. +- Success criteria: + - Step 1 changes no behavior: every existing test passes unchanged, and `rg -n "a batched catalog call is issued as an effect" crates` prints nothing. + - The parse-test split keeps the same tests: `promptforge-lua`'s test count is unchanged, and no file in `crates/` exceeds 500 lines. + - Each text fix reads as the Functional Specification states. + - Every exit gate in the Testing Plan passes. +- Constraints: + - The Engine stays sans-IO and deterministic. Clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, and dead code is fatal under the gate. + - Every `.rs` file stays under 500 lines, and `build-xtask`'s `no_doctests` check bans rustdoc-compilable code fences in doc comments. +- Open questions: none. + +## Functional Specification + +Nothing a prompt author or Host can observe changes. The fixes are a private refactor, documentation, a test-file move, the spec entry, and tool instructions. + +- Actors and workflows: + - Engine: a model round's batch asks one route function per call and stops at the first call that is not a catalog call, exactly as today. + - Planning tools: decomposition writes steps that carry everything their sub-agents need, including a Tests line. The debt collector checks each planned regression test against the endpoint code before writing it into a plan. +- Inputs and outputs: unchanged for prompts, Plugins, and Hosts. +- States and validation: + - Script calls, lone model calls, and batched model calls keep today's call ids, turns, events, effects, and `ToolCaller` origins. +- Errors and recovery: + - The batch's internal error for a non-issued outcome goes away with its arm. Every other error is unchanged, including `UnboundToolCall` for a name the round didn't advertise. +- Security and privacy behavior: unchanged. Each call still forks its own access and ends it when answered. +- Acceptance criteria: the success criteria, each pinned by a check in the Testing Plan. + + + + +## Technical Design + +Step 1 moves the route decision into one function, splits catalog preparation out of `prepare_tool_call`, and replaces three correlated options with a private enum. Step 2 is text: doc comments, a test-file move, a corrected plan-record sentence, the spec entry, and the two tool files. Nothing public changes. + +### Step 1: one route decision and a call-site enum + +- In [crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs](crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs) (D1-9): + - A private `ToolRoute { Builtin, Local, Catalog }` and a `tool_route(&self, id, alias) -> Result` that makes today's checks in today's order: a name in `RESERVED_TOOL_NAMES` is `Builtin`, a name the live frame's VM has as a local tool is `Local`, and anything else is `Catalog`. It borrows `&self` only, as `runs_alone` (lines 171-180) does today. + - `prepare_tool_call` (lines 253-361) matches on `tool_route` first and hands its catalog arm to a new `prepare_catalog_call`. That function holds today's catalog work (binding, counts, the access fork, `ToolCaller`, `issue`) and returns the issued call, or an error. + - `dispatch_tool_calls` (lines 129-166) asks `tool_route` per call, stops at the first call that is not `Catalog`, and calls `prepare_catalog_call` for the rest. `runs_alone` and the fallback arm that turns a non-`Issued` outcome into the internal error "a batched catalog call is issued as an effect" are deleted. +- A private call-site enum (D1-5), declared beside `ToolCallContinuation` in [crates/promptforge-internal/engine/src/execute/scheduler/pending.rs](crates/promptforge-internal/engine/src/execute/scheduler/pending.rs) (lines 59-74): + +```rust +pub(super) enum CallSite { + Script { turn: Option }, + Model { call_id: String, turn: Option }, + Batch { call_id: String, turn: u32, slot: usize }, +} +``` + + - `prepare_tool_call` and `prepare_catalog_call` take a `CallSite` in place of `call_id: Option`, `turn: Option`, and `slot: Option`. + - `dispatch_tool_call` (lines 94-105) builds `Model` from a request with a call id and `Script` from one without. + - `dispatch_tool_calls` builds `Batch`. + - Turn resolution keeps today's rule: the given turn, else the chain's counter. The parser accepts a script request that carries a turn (`crates/promptforge-internal/lua/src/protocol/parse.rs` reads `call_id` and `turn` independently), so `Script` keeps that turn and behavior is unchanged. + - `ToolCallContinuation` stores the `CallSite` in place of its `call_id` and `slot` fields. `apply.rs` matches on it: `Script` takes `accept_tool_call`'s script branch, `Model` and `Batch` its model branch (lines 168-208), and `Batch` sends the final answer to `fill_batch_slot` (lines 80-120). + - `ToolCaller` (`crates/promptforge-internal/types/src/tools/origin.rs` lines 21-30, public and `#[non_exhaustive]`) is unchanged. `Script` maps to `ToolCaller::Script`, and `Model` and `Batch` map to `ToolCaller::Model`. +- Unchanged: `dispatch.rs`, the Lua crate, the Harness, events, effects, turns, `public-api.txt`, and the four task built-ins. + +### Step 2: text, a test-file move, the spec, and the tools + +- [crates/promptforge-plugin/src/context.rs](crates/promptforge-plugin/src/context.rs) (D1-11): in the `ToolContext` doc (lines 14-22), after "...and the call's operations before the chain's next step.", add one sentence: calls a model asks for in one round fork their identities from the chain together, so they are concurrent with each other, and one call's write to a path conflicts with a sibling's read or write of it, as between fanout arms. The VFS tests `two_writes_by_two_identities_conflict` (`crates/promptforge-internal/vfs/src/handle/tests/claims.rs` line 62) and `a_sibling_read_of_a_siblings_write_conflicts` (`handle/tests/happens_before.rs` line 156) already pin the rule. +- [vibe/2026-10-11-3-parallel-tool-calls.md](vibe/2026-10-11-3-parallel-tool-calls.md) line 339 (D1-13): replace "and a batch keeps today's effect ids and issue order." with "and a batch keeps today's issue order. Its effect ids match today's only in a run with one live chain; with several, ids depend on how the chains interleave." Line 340, the falsifier, stays. +- [crates/promptforge-internal/lua/src/protocol.rs](crates/promptforge-internal/lua/src/protocol.rs) lines 4-7 (D1-14): "the `chat` and `tool_call` rounds the `models.loop` shim yields" becomes "the `chat`, `tool_calls`, and `tool_call` rounds the `models.loop` shim yields". +- Parse-test split: move the `tool_call` tests (`crates/promptforge-internal/lua/src/protocol/tests/parse.rs` lines 93-287) into a new `crates/promptforge-internal/lua/src/protocol/tests/parse_tool_call.rs`, declared in `crates/promptforge-internal/lua/src/protocol/tests.rs` beside `mod parse;` (lines 105-109), as `parse_chat` and `parse_tasks` already are. It is a pure move: same test names and bodies, with only the imports the new file needs. +- Spec, `../promptforge-design/design/architect-promptforge-prompting-language.md`. User: "make the tasks.fanout spec entry describe today's behavior". + - The `tasks.fanout(worker, collection)` entry (lines 414-418) states today's contract: + - `worker` is a section heading string, such as `"### Worker"`, run once per member. + - Each arm sees its member as `item` and its 1-based position as `sys.index`. + - The collection is any Lua table: array members first, then hash pairs sorted by key as `{ key, value }`. A non-table raises an error that points at `prompt.list(heading)`, and an empty collection raises. + - It returns the arms' results in member order as read-only tables with `text`, `ok`, `item`, and `exhausted`, and `tostring` of a result is its `text`. + - An arm that exhausts its tool loop leaves a stub result and the rest continue. Any other arm failure cancels the live arms and re-raises. + - Recursion is capped at 8. + - The entry's `**Target contract:**` paragraph (the `call_async`, `args` `{ index, value }`, and `await_all` contract) is removed, and its status line becomes `**Implementation status:** Active.`, the form its neighbors use (line 388). + - The "Deferred fanout review" example (lines 476-505) becomes "Fanout review". Its prose no longer calls the helper a wrapper over `call_async` and `await_all`, and its body reads `item` and each result's `text` in place of `args.value` and joined results. Any anchor in the spec that links the old example title is updated. +- [../tools-public/planning/vibe-coder.md](../tools-public/planning/vibe-coder.md), inside ``: + - The step template (lines 250-260) gains `- Tests: commands Verify runs as written, or none` after `- Component:`. This defines the Tests line that `` already names (line 397). + - After "Each step names concrete artifacts - files, modules, functions, structs - without a full implementation." (line 262), add: the coding, review, and fix sub-agents read only the implementation contract, the project survey, and their step, so copy into each step every test case, error text, and acceptance detail it needs from the product, verification, and decision contracts. + - Add: never assign the plan's seed copy, `vibe/ACTIVE`, or the close commit to a step, since the Run Sequence owns them. + - Append one attribution line after the last one (line 574), separated by one blank line: `*2026-10-11 - Claude Opus 5.5 (Cursor agent). Steps carry their tests and a Tests line; seeding and the close stay out of steps.*` +- [../tools-public/planning/debt-collector.md](../tools-public/planning/debt-collector.md), in `## Removal Plan`: + - After "Give each fix of incorrect behavior a regression test that fails before the change; a test that already passes proves the finding false, so keep the test and skip the change." (line 178), add: before writing a regression test into the plan, read the endpoint code it exercises and name the assertion that fails there; an assertion the endpoint already satisfies is not that test. + - Append one attribution line after the last one (line 255), separated by one blank line: `*2026-10-11 - Claude Opus 5.5 (Cursor agent). Planned regression tests are checked against the endpoint code.*` + + + + +## Testing Plan + +Step 1 is a refactor with no failing-test-first shape: existing suites, exhaustive matches, and the dead-code lint are its proof. Step 2 is checked by test counts, the build ceiling, docs builds, and text searches. + +- Unit and integration, Step 1: these existing suites pass unchanged under `cargo nextest run --locked -p promptforge-engine --all-features` and `-p harness-runner --all-features`: + - `engine/src/execute/tests/parallel_tool_calls.rs`, which pins both stop checks, the empty answer, the unbound-call stop, and the cancel orphaning + - `batch_turn.rs`, `tool_call_arm.rs`, `full_id_calls.rs`, `tool_call_access.rs`, `effects.rs`, `offering.rs`, and `models_loop.rs` + - the Harness's `harness-stop.rs`, including `two_calls_in_one_round_are_in_flight_at_once_and_a_stop_drops_both` +- Regression, Step 1: + - `rg -n "a batched catalog call is issued as an effect|fn runs_alone" crates` prints nothing. + - Clippy with warnings denied shows no dead code. + - `crates/promptforge/public-api.txt` is unchanged, since `xtask api --check` passes. +- Regression, Step 2: + - `promptforge-lua`'s test count under `cargo nextest run --locked -p promptforge-lua --all-features` is the same before and after the split, and the moved test names appear under `protocol::tests::parse_tool_call`. + - `cargo test -p build-xtask` passes, including the 500-line ceiling and `no_doctests`. + - `rg -n "keeps today's effect ids" vibe/2026-10-11-3-parallel-tool-calls.md` prints nothing. + - `rg -n "the .chat. and .tool_call. rounds" crates/promptforge-internal/lua/src/protocol.rs` prints nothing. + - In the spec, the `tasks.fanout` entry and the fanout review example contain no `**Target contract:**`, `args.value`, or `call_async`. + - In `../tools-public`, each new sentence appears once (`rg -n -F` on a distinctive phrase), and each file's diff touches only the lines named in the Technical Design. + - `git -C ../promptforge-design status --porcelain` still lists the two unrelated `research/` changes, unstaged. +- Exit criteria, run at the end of each step: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` with the gateway sidecar staged as the Project Survey says + - both clippy runs with `CARGO_BUILD_WARNINGS=deny` + - `cargo fmt --all --check` + - `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` and `cargo doc -p promptforge --no-deps`, both with `RUSTDOCFLAGS="-D warnings"` + - `cargo +nightly-2026-09-05 xtask api --check` + - `cargo test -p build-xtask` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + + + + +## Decision Record + +The user chose the scope and how the vibe coder carries test details. The refactor's shape and the doc wording are planning defaults. + +- Decisions: + - Scope: the `promptforge` doc and test-file fixes, the two routing refactors, the spec entry, and the four tool fixes. User selected all four groups. + - Decomposition copies test cases, error texts, and acceptance details into steps. User chose "Decomposition copies into each step every test case, error text, and acceptance detail the step needs from sections those agents can't read". + - The spec entry describes today's behavior and drops its target paragraph. User: "make the tasks.fanout spec entry describe today's behavior". + - Defaults chosen in planning, open to override: + - The batch prepares only catalog calls, through `prepare_catalog_call`, so the fallback arm is deleted rather than kept behind a shared check. A shared check alone would leave an arm that runs after side effects. + - `CallSite` is private to the scheduler, and the public `ToolCaller` stays as it is. + - `CallSite::Script` carries the request's optional turn, because the parser accepts a turn on a script request with no call id; a bare `Script` would drop it. + - The `ToolContext` sentence cites the two VFS tests that already pin sibling conflicts, so no new test is added. + - A closed plan record is corrected in place, as `8cade0de4` (Correct the whisper plans' rules surveys) and `0938ee342` (Correct the decomposition records) did. +- Rejected alternatives: + - The vibe coder's coding, review, and fix sub-agents read the verification and product contracts too. Reason: the user chose copying, which keeps each sub-agent's reading to its step. + - New variants on `ToolCaller` for batched calls. Reason: it is public and `#[non_exhaustive]`, so the change would reach Plugins for no benefit. + - A Harness test that two calls in one round writing one path fail. Reason: the VFS tests already pin the rule, and the doc sentence states only that rule. +- Assumptions, risks, and notes: + - This plan's own run reads `vibe-coder.md` while Step 2 edits it. The edits are additions that fit the current flow, so later dispatches in Step 2 still work. + - Step 1 changes no observable behavior. If any existing test needs an edit, the refactor changed behavior, and the step stops for re-planning. + - `../promptforge-design` holds unrelated `research/` changes. Only the spec file is staged there. + +### Deferred and Out of Scope + +- `../promptforge-docs`, including the guide's `fanout(` and `list_from_section` examples. Out of scope by the user. +- A cap on tool calls in flight. Revisit if fanout arms with parallel calls hit provider limits. +- What Stop means while a question is open, where a deliberate Stop discards the round's answer. Revisit with the Stop semantics. +- A call id on wait frames, which a client attached beside another would need. Revisit if Workshop ships two clients on one conversation. +- The test-only `tools.call_as_model` route that drives a single model-issued catalog call (D1-15). Revisit when that hook is next edited. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build --locked -p gateway`. + - `crates/gateway/app` is the workspace's only default member, so plain `cargo build` builds the same thing. `.cargo/config.toml` aliases `cargo workshop` to `run -p build-workshop --` (the desktop app) and `cargo xtask` to `run -p build-xtask --`, and links Windows builds with `rust-lld` and the static CRT. + - Workspace-wide cargo commands run the `workshop-server` and `gateway-config-ui` build scripts, which bundle UI with esbuild, so they need Node.js and the npm installs in `crates/workshop` and `crates/gateway/config-ui/ui`. Both `node_modules` trees are present on this machine, along with Node.js 24.19 (CI uses 22), stable cargo 1.99, cargo-nextest 0.9.128, and the `nightly-2026-09-05` toolchain. + - Workshop clippy and tests need the gateway sidecar staged for Tauri's `externalBin`, as CI's `check-workshop` job does it: `cargo build --locked -p gateway --no-default-features`, then `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe`, and `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc` afterward. The staging directory `crates/workshop/desktop/binaries/` is gitignored and empty now. A `target/debug/promptforge-gateway.exe` already exists, but rebuild it with `--no-default-features` before staging, as CI does. + - The shell is PowerShell. Set an environment variable for one command as `$env:NAME='value'; `. +- Focused test command pattern: `cargo nextest run --locked -p --all-features `. + - For this plan's crates: `-p promptforge-engine` with a filter such as `parallel_tool_calls`, `-p promptforge-lua` with `protocol::tests::parse`, `-p harness-runner` with `harness::stop`, and `-p promptforge-plugin`. `--all-features` turns on each crate's `test-support` feature; `harness-runner` has no features, so the flag is harmless there. + - Drop `--all-features` for `workshop`, `workshop-server`, and `workshop-server-api`, which CI never builds with all features. + - `cargo nextest list --locked -p --all-features --message-format oneline` prints one ` ` line per test. At `fa59d5f41` it lists 404 tests for `promptforge-lua`, 29 of them under `protocol::tests::parse::`. + - Workshop UI: `node test/.mjs` from `crates/workshop/ui`. Each file is a standalone script and the smallest runnable unit. +- Component test command pattern: + - Rust: `cargo nextest run --locked -p --all-features`, with the same three Workshop exceptions. The Testing Plan's Step 1 runs are `-p promptforge-engine --all-features` and `-p harness-runner --all-features`. + - Structural checks: `cargo test -p build-xtask`. It covers the product and container boundaries, the Engine manifest guard, the `test-support` leak guard, `no_doctests`, retired symbols, the unsafe allowlist, the `## Invariants` marker, lint inheritance, and the wiring of every build script to the line ceiling. + - Workshop UI: `npm test --workspace ui` in `crates/workshop`, which runs `node --test "test/**/*.mjs" "src/**/*.test.mjs"`. `--workspace look` and `--workspace platform` run the other two packages. +- Full-suite test command: + - Rust: `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` with the sidecar staged. CI's Windows job also runs `-p workshop-workspace --all-features`, `-p workshop-server --features headless`, and five gateway launch-lease tests by name. + - The wording and rulebook scan: `node --test crates/workshop/ui/test/docs-claims.mjs`. It needs no `npm ci` and runs from any directory. + - Workshop UI: `npm test --workspaces --if-present` in `crates/workshop`, plus `npm test` in `crates/gateway/config-ui/ui`. + - `.config/nextest.toml` reports a test as slow after 60 s and kills it after three periods, and puts `gateway-stt` and `gateway-stt-backend-whisper` in a `heavy` group. +- Linter command: + - Rust: `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and, with the sidecar staged, `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`. Run both with `$env:CARGO_BUILD_WARNINGS='deny'`, which makes every rustc warning fatal, dead code included. Never run a standalone `cargo check --workspace` beside them; the one exception is `cargo check -p gateway --no-default-features`. + - The lints come from `[workspace.lints]` in `Cargo.toml`: clippy `all` and `pedantic` deny, `unwrap_used`, `expect_used`, and `allow_attributes` deny, and `missing_docs`, `unreachable_pub`, and `missing_debug_implementations` warn (fatal under the gate). `clippy.toml` allows `unwrap` and `expect` in tests. Rustdoc's `broken_intra_doc_links` and `private_intra_doc_links` deny. + - Workshop UI: no ESLint, Prettier, or Biome config exists, so the type check is the gate: `npm run typecheck --workspaces --if-present` in `crates/workshop`, which runs `tsc --noEmit` in `ui`, `look`, and `platform`. The gateway config UI runs `npm run typecheck` in `crates/gateway/config-ui/ui`. + - CI also runs `cargo deny check`, `cargo audit`, `cargo hakari verify`, and a `cargo tree` check that `ring` stays out of the gateway's normal closure. + - Git hooks: `core.hooksPath` is `.githooks`. `pre-commit` runs `cargo fmt --all --check`. `pre-push` runs the wording scan, the headless gateway check, the non-Workshop clippy run, and `cargo deny` when installed. Both are bash scripts. +- Formatter check command: `cargo fmt --all --check`, with `style_edition = "2024"` in `rustfmt.toml`. No formatter is configured for TypeScript, `.mjs`, or Markdown, so new text matches the surrounding style by hand. +- Docs command: + - Each with `$env:RUSTDOCFLAGS='-D warnings'`: `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` and `cargo doc -p promptforge --no-deps`, the plan's two. CI also runs `cargo doc -p harness --no-deps`, `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items`, and `cargo doc --locked --no-deps -p workshop-server --document-private-items`. The Engine private-items run is the one that renders the scheduler's private doc comments, such as those on a new `ToolRoute` or `CallSite`. + - The facade surface check: `cargo +nightly-2026-09-05 xtask api --check` against `crates/promptforge/public-api.txt`. The nightly is pinned in `crates/build-xtask/src/api/toolchain.rs` and installed here. CI also runs `cargo nextest run --locked -p build-xtask --run-ignored only` on that nightly. + - Doctests are banned by `build-xtask`'s `no_doctests` check, so doc comments hold no rustdoc-compilable code fences. + - The Workshop UI has no generated docs. +- Test placement and naming conventions: + - Rust unit tests sit in a sibling file wired with `#[cfg(test)] #[path = "-tests.rs"] mod tests;` (176 files) or in an inline `mod tests { }` (141 files). When a module's tests reach three files, they move to a `tests/` subdirectory, with a `tests.rs` beside it that declares one `mod ;` per file. + - Engine scheduler tests follow that layout: `crates/promptforge-internal/engine/src/execute/tests.rs` declares one module per topic from `execute/tests/` (60 files), among them `parallel_tool_calls.rs`, `batch_turn.rs`, `tool_call_arm.rs`, `full_id_calls.rs`, `tool_call_access.rs`, `effects.rs`, `offering.rs`, and `models_loop.rs`. A topic that outgrows one file gets kebab children wired with `#[path]`, such as `models_loop-arguments.rs` declared in `models_loop.rs` as `mod arguments;`. The Engine's top-level `tests/` holds only fixture prompts under `tests/prompts/`. + - Lua protocol tests: `crates/promptforge-internal/lua/src/protocol/tests.rs` (109 lines) holds shared helpers such as `lua_table` and declares `mod answer;`, `mod answer_chat;`, `mod parse;`, `mod parse_chat;`, and `mod parse_tasks;` at lines 105-109, each a file in `protocol/tests/`. Test paths read `protocol::tests::::`. + - Integration tests compile as one binary per crate, from `tests/it/main.rs` (18 crates) or `tests/suite/main.rs` (`harness` and `promptforge`), with one module per topic and, in five of them, helpers in `support.rs`. In `harness-runner`, `tests/it/harness.rs` wires `harness-errors.rs`, `harness-stop.rs`, and `harness-stop-timing.rs` as `#[path]` children named `errors`, `stop`, and `stop_timing`. `two_calls_in_one_round_are_in_flight_at_once_and_a_stop_drops_both` is at `harness-stop.rs` line 203. + - Test names are snake_case sentences stating the behavior, such as `tool_call_with_a_call_id_parses_it_as_a_model_issued_call`. Seven integration test roots, including `harness-runner`'s and `promptforge-plugin`'s, open with a crate-level `#![expect(clippy::expect_used, clippy::unwrap_used, reason = "...")]`. + - Every `.rs` file under a crate's `src`, `tests`, `benches`, and `examples`, plus its `build.rs`, is capped at 500 physical lines by `build_ceiling::check()`, which 51 crate build scripts call (all but `workspace-hack`). It fails only a file over 500, so a file of exactly 500 lines passes. The largest today are `crates/workshop/workspace/src/workspace/backing.rs` (500) and `crates/promptforge-internal/lua/src/protocol/tests/parse.rs` (499). + - Workshop UI tests are flat standalone scripts, `crates/workshop/ui/test/.mjs` (113 files), with shared modules in `test/helpers/`. `look/test/` and `platform/test/` use the same layout. +- Directory map: + - `crates/`: 52 Cargo packages. The top level holds the public and shared layer: the `promptforge` facade, `promptforge-plugin`, `harness`, `harness-gateway-client`, `gateway-api-types`, `gateway-api-discovery`, `plugin-mcp`, `plugin-user-input`, `plugin-web`, `shared-error-source`, `shared-loopback`, the `build-*` tools (`build-ceiling`, `build-llama-cuda`, `build-ui`, `build-user-guide`, `build-workshop`, `build-xtask`), and `workspace-hack` (cargo-hakari). + - `crates/promptforge-internal/`: the Engine's private crates `types`, `engine`, `lua`, `parser`, `vfs`, and `model-client`, packaged as `promptforge-`. `crates/harness-internal/runner`: the package `harness-runner`. + - The Engine scheduler: `crates/promptforge-internal/engine/src/execute/scheduler.rs` (321 lines) declares 20 child modules in `scheduler/`, all private except `#[cfg(test)] pub(crate) mod test_hooks`. This plan touches `tool_call.rs` (419 lines), `pending.rs` (84), and `apply.rs` (277), whose shared items are `pub(super)`; `dispatch.rs` (359) stays unchanged. + - The Lua protocol: `crates/promptforge-internal/lua/src/protocol.rs` (33 lines) with the children `answer`, `parse` (plus `parse/chat.rs`, `parse/store.rs`, and `parse/tasks.rs`), `render`, `request`, and `tests`. + - `crates/promptforge-plugin/src/context.rs` (83 lines) holds `ToolContext`. `ToolCaller` lives in `crates/promptforge-internal/types/src/tools/origin.rs`, doc comment at line 21 through the closing brace at line 30. It is public as `promptforge::effect::ToolCaller` in `crates/promptforge/public-api.txt`. + - `crates/gateway/`: the gateway's private crates plus the `stt/` subsystem. `crates/workshop/`: Workshop's Rust crates plus the npm workspace with the packages `ui`, `look`, and `platform`. `crates/shared-ui`: a TypeScript and CSS package, excluded from the Cargo workspace. + - `vibe/`: plan records. Recent ones sit at the top level (24 files, the latest `2026-10-11-4-workshop-waits.md`), older ones in `2026-07/` through `2026-10/`, plus `scratch/`. No `vibe/ACTIVE` exists now. + - `guide/` (`books`, `chrome`, `landing`), `prompts/` (5 example prompts), `tools/` (the sidecar stager and a live TTS check, each with a `.test.mjs`), `images/`, `.github/workflows/` (`ci.yml`, whose `ci-green` job is the single gate, plus release, nightly, site, Miri, native-library, and installer smoke workflows), `.config/` (`nextest.toml`, `hakari.toml`), `.githooks/`, and `.cursor/rules/` (`workshop-architecture.mdc`, `workshop-spa.mdc`). Nested `AGENTS.md` files sit in 21 directories under `crates/`, including `engine`, `lua`, `types`, and `vfs` under `crates/promptforge-internal/`. + - Not tracked: `local/`, `target/`, and `target-msrv/` are gitignored, and `cabinet/` holds only an empty `_trash/`. + - Git: `master` is at `fa59d5f41` (`Close plan: workshop-waits`), five commits ahead of `origin/master`, with a clean worktree. + - Sibling repositories this plan edits, both Markdown with no build, test, lint, or format tooling over the edited files: + - `../promptforge-design` at `a34adaa`. The spec is `design/architect-promptforge-prompting-language.md` (572 lines). `research/2026-10-10-subagent-tool-naming.md` (modified) and `research/2026-10-10-subagent-tool-catalog.md` (untracked) are unrelated and must stay unstaged. + - `../tools-public` at `6a038da`, clean. `planning/` holds `vibe-coder.md` and `debt-collector.md`. Its one crate, `crates/png2jpg`, is unrelated. +- Component boundaries: + - Enforced by `cargo test -p build-xtask` across normal, dev, build, and target-specific dependencies: + - `promptforge` and `promptforge-*` crates depend on no gateway, Workshop, Harness, or `plugin-*` crate, and crates outside the family reach it only through `promptforge` and `promptforge-plugin`. The Engine's manifests name no `tokio`, `tokio-util`, `async-trait`, or `reqwest` outside `[dev-dependencies]`, except optional entries only `test-support` enables, and no non-dev table may enable a `test-support` feature. + - `harness` and `harness-*` crates name only `promptforge`, `promptforge-plugin`, and `workspace-hack` outside their family, and other crates reach the family only through `harness` and `harness-gateway-client`. `harness-runner` sits in the private `crates/harness-internal/` container, which only `harness` may name. + - `plugin-*` crates depend only on `promptforge-plugin`, `shared-*` crates, `workspace-hack`, and outside libraries. Gateway crates depend on no Engine, Workshop, or Harness crate, and Workshop reaches the gateway only through `gateway-api-types` and `gateway-api-discovery`. + - Inside the Engine, per the crate `AGENTS.md` files: the scheduler decides each tool call's route and issues catalog calls as `Effect::ToolCall`, the Harness performs them, and `promptforge-lua`'s `prepare_dispatch` (or `prepare_model_dispatch` for model-issued calls) applies counts, trust, the nonce wrap, and the `ToolResult` report when the answer lands. Store views derive only from the chain's access inside the Engine. Every `Event` is report-only. + - Visibility: the ceiling's split rule forbids widening anything to `pub(crate)` or `pub` to make a split compile. `ToolCaller` is public and `#[non_exhaustive]`, so a change to it reaches `public-api.txt` and `xtask api --check`. + - Workshop is a Host. Its Rust tiers depend one way, the server on features, features on services, and services on vocabulary, and the desktop app names only `workshop-server-api`. +- Conventions summary: + - Engine, Harness, Host, and Plugin are capitalized with one meaning each. Engine crates call their driver "the caller" and never mention the Host. `docs-claims.mjs` enforces the wording in every `AGENTS.md`, every `## Invariants` crate doc, and every `.cursor/rules` file. + - Comments explain only a non-obvious constraint, ordering requirement, or workaround, and every workaround cites its upstream issue URL. Error and status messages are written for a model to read: concise, naming what was required and what was found. + - Behavior changes ship with tests in the same change, and refactors keep existing behavior tests unchanged. + - Suppressions use `#[expect(, reason = "...")]` (102 files). `#[allow(...)]` is denied and appears only inside test fixture strings. + - Source directories stay flat until a group reaches three files. Below that, children are kebab siblings wired as `#[path = "foo-bar.rs"] mod bar;`. A split-out child opens with a `//!` line naming its one concern. 25 crate `lib.rs` files open with a `## Invariants` doc section. + - JSON that reaches a recorder or replay comparison round-trips exactly, with sorted keys and finite numbers. + - Commits have a one-line summary, a prose body, a bullet list of changes, optional `Design:`, `Deferred:`, and `Repairs:` lines, and a `Plan: vibe/` trailer. A plan's closing commit is titled `Close plan: ` and holds only the trailer. + + + + +## Execution Instructions + +Two steps, each one `promptforge` commit, run in order. They are independent; Step 1 lands first so the refactor's verification runs before any tool file changes. Step 2 also produces one commit in `../promptforge-design` and two in `../tools-public`. The plan's seed copy in `vibe/`, `vibe/ACTIVE`, and the `Close plan: easy-fixes` commit belong to the Run Sequence, not to a step. + + + +### Step 1: Decide a tool call's route once and type its call site [completed] + +- Component: `none` +- Tests: Verify runs every command below as written, in this order, at every scope. Run them from the `promptforge` repository root. A check that "prints nothing" passes when the command prints no output, even though `rg` then exits 1. + - `cargo nextest run --locked -p promptforge-engine --all-features` passes with no test file edited. + - `cargo nextest run --locked -p harness-runner --all-features` passes with no test file edited. + - `rg -n "a batched catalog call is issued as an effect|fn runs_alone" crates` prints nothing. + - `git diff --exit-code fa59d5f41 -- crates/promptforge/public-api.txt crates/promptforge-internal/types/src/tools/origin.rs` prints nothing. + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items` passes. It renders the private docs on `ToolRoute` and `CallSite`. + - Exit criteria: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` + - Stage the gateway sidecar: `cargo build --locked -p gateway --no-default-features`, then `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe`. Run `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` and `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, then always run `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc`, even when a command failed. + - `cargo fmt --all --check` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` and `$env:RUSTDOCFLAGS='-D warnings'; cargo doc -p promptforge --no-deps` + - `cargo +nightly-2026-09-05 xtask api --check` + - `cargo test -p build-xtask` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + - At `FULL` scope this list, plus the survey's build command, is the whole gate. The survey's "CI also runs" extras and the Workshop UI npm suites are not part of it. +- Shape: a refactor with no failing-test-first shape. The existing suites, exhaustive matches, and the dead-code lint are its proof, so write no new test. +- Why (D1-9, D1-5): the batch in `dispatch_tool_calls` decides a call's route with `runs_alone`, a copy of the checks `prepare_tool_call` makes. If the two disagreed, the batch's fallback arm would run after `prepare_tool_call` had already pushed a local call or started a task. `prepare_tool_call` also takes `call_id`, `turn`, and `slot` as three independent options, though a slot without a call id is meaningless. This step is a private refactor and changes no behavior. +- `crates/promptforge-internal/engine/src/execute/scheduler/tool_call.rs` (419 lines at baseline): + - Add a private `enum ToolRoute { Builtin, Local, Catalog }` and `fn tool_route(&self, id: ChainIndex, alias: &str) -> Result`. It makes `runs_alone`'s checks (lines 171-180) in the same order: a name in `RESERVED_TOOL_NAMES` is `Builtin`, a name the live frame's VM has as a local tool is `Local`, and anything else is `Catalog`. A missing frame stays the internal error "a live chain holds its frame". It borrows `&self` only. + - `prepare_tool_call` (lines 253-361) matches on `tool_route` first. The `Builtin` arm is today's lines 270-275: a task built-in under a call id is answered by `answer_task_builtin`, and anything else is `unbound_tool_call`. The `Local` arm is today's lines 285-301: count the attempt, then `push_local_call`. The `Catalog` arm calls the new `prepare_catalog_call` and maps its success to `ToolCallDispatch::Issued`. + - `prepare_catalog_call` holds today's catalog work from lines 302-361: the binding lookup (`wire_binding` under a call id, else the canonical-id lookup), `unbound_tool_call` when nothing binds, the counts, the access fork, the `ToolCaller`, the `Effect::ToolCall`, and `issue`. It computes the effective bindings and counts itself (`current_tool_bindings`, `script_call_counts`) so the batch can call it directly, and returns `Result<()>`, `Ok` once the call is issued. + - `dispatch_tool_calls` (lines 129-166) asks `tool_route` per call and breaks at the first call whose route is not `Catalog`, exactly where today's `Ok(true)` breaks. A `tool_route` error or a `prepare_catalog_call` error fills that call's slot and stops the batch, as today. Each issued call pushes `None`, and the batch still ends with `answer_full_batch`. + - Delete `runs_alone` and the arm that turns a non-`Issued` outcome into the internal error "a batched catalog call is issued as an effect". + - Update the module doc (lines 1-35) and the docs on `dispatch_tool_calls` and `prepare_tool_call` that mention `call_id: Some`, `call_id: None`, `slot`, or "must run alone", so they describe the route and the `CallSite`. Doc comments hold no rustdoc-compilable code fences. +- `crates/promptforge-internal/engine/src/execute/scheduler/pending.rs` (84 lines): beside `ToolCallContinuation` (lines 59-74), declare the private call-site enum with a doc comment on it and each variant: + + ```rust + pub(super) enum CallSite { + Script { turn: Option }, + Model { call_id: String, turn: Option }, + Batch { call_id: String, turn: u32, slot: usize }, + } + ``` + + - `Script` holds a turn because the parser accepts a script request that sets one: `crates/promptforge-internal/lua/src/protocol/parse.rs` lines 399-400 read `call_id` and `turn` independently. Keeping that turn keeps behavior unchanged. + - `ToolCallContinuation` stores a `CallSite` in place of its `call_id` and `slot` fields. `binding`, `report`, and `exec` stay. +- The `CallSite` through the scheduler: + - `prepare_tool_call` and `prepare_catalog_call` take a `CallSite` in place of `call_id: Option`, `turn: Option`, and `slot: Option`. + - `dispatch_tool_call` (lines 94-105) keeps its signature, since `dispatch.rs` calls it unchanged. It builds `CallSite::Model` from a request with a call id and `CallSite::Script` from one without, each with the request's turn. `dispatch_tool_calls` builds `CallSite::Batch` with the round's turn and the call's slot. + - Turn resolution keeps today's rule (line 264): the given turn, else the chain's turn counter at dispatch. + - The task built-in path and `LocalCall` keep their own optional call id, read from the `CallSite`. + - `Script` maps to `ToolCaller::Script`, and `Model` and `Batch` map to `ToolCaller::Model`. `ToolCaller` lives in `crates/promptforge-internal/types/src/tools/origin.rs` lines 21-30, is public and `#[non_exhaustive]`, and stays unchanged. +- `crates/promptforge-internal/engine/src/execute/scheduler/apply.rs` (277 lines) matches on the continuation's `CallSite`. The slot lookup (lines 80-84) takes a slot only from `Batch`, whose final answer goes to `fill_batch_slot` (line 119); `Script` and `Model` resume the chain directly as today. In `accept_tool_call` (lines 168-208, matching `call.call_id` at line 180 today), `Script` takes the script branch, and `Model` and `Batch` take the model branch under their call id. +- Acceptance: + - Script calls, lone model calls, and batched model calls keep today's call ids, turns, events, effects, and `ToolCaller` origins. Each call still forks its own access and ends it when answered. + - Every error except the deleted internal one is unchanged, including `UnboundToolCall` for a name the round didn't advertise. + - These suites pass unchanged: `crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls.rs` (both stop checks, the empty answer, the unbound-call stop, and the cancel orphaning); `batch_turn.rs`, `tool_call_arm.rs`, `full_id_calls.rs`, `tool_call_access.rs`, `effects.rs`, `offering.rs`, and `models_loop.rs` beside it; and `crates/harness-internal/runner/tests/it/harness-stop.rs`, including `two_calls_in_one_round_are_in_flight_at_once_and_a_stop_drops_both` (line 203). + - If any existing test needs an edit, the refactor changed behavior: stop and return for re-planning. + - Clippy with warnings denied reports no dead code, and every touched file stays at or under 500 lines. +- Out of scope: `dispatch.rs`, the Lua crate, the Harness, events, effects, turns, `crates/promptforge/public-api.txt`, the four task built-ins (`task`, `task_cancel`, `task_status`, `await_tasks`), the test-only `tools.call_as_model` route, and `../promptforge-docs`. +- Commit: one `promptforge` commit, which the session makes, suggested subject `Decide a tool call's route once and type its call site`, with a prose body, a bullet list of changes, `Design:` lines, and the trailer `Plan: vibe/2026-10-11-5-easy-fixes.md`. + + + + + +### Step 2: Document sibling calls, split the tool_call parse tests, and fix the spec and tools + +- Component: `none` +- Tests: Verify runs every command below as written, in this order, at every scope. Run them from the `promptforge` repository root. A check that "prints nothing" passes when the command prints no output, even though `rg` then exits 1. + - `cargo nextest list --locked -p promptforge-lua --all-features --message-format oneline` lists 404 tests, the same count as at baseline `fa59d5f41` (recorded by the survey, 29 of them under `protocol::tests::parse::`). Its `protocol::tests::parse::` and `protocol::tests::parse_tool_call::` lines together number 29, with at least one under each. + - `cargo nextest run --locked -p promptforge-lua --all-features` and `cargo nextest run --locked -p promptforge-plugin --all-features` pass. + - `cargo test -p build-xtask` passes, including the 500-line ceiling and `no_doctests`. + - `rg -n "keeps today's effect ids" vibe/2026-10-11-3-parallel-tool-calls.md` prints nothing. Scope it to that file: this plan's own copy in `vibe/` quotes the phrase. + - `rg -n "the .chat. and .tool_call. rounds" crates/promptforge-internal/lua/src/protocol.rs` prints nothing. + - This PowerShell command prints nothing. It prints `missing heading` when a bounding heading is absent, and otherwise any line of the `tasks.fanout` entry or the "Fanout review" example that still holds `**Target`, `args.value`, or `call_async`: + + ```powershell + $s = Get-Content ../promptforge-design/design/architect-promptforge-prompting-language.md; $at = { param($p) ($s | Select-String -Pattern $p).LineNumber }; $a = & $at '^### .tasks\.fanout\(worker, collection\).$'; $b = & $at '^### Deferred .pfanout\('; $c = & $at '^### Fanout review$'; $d = & $at '^### Deferred compact model loop$'; if (-not ($a -and $b -and $c -and $d)) { 'missing heading' } else { $s[($a-1)..($b-2)] + $s[($c-1)..($d-2)] | Select-String -Pattern '\*\*Target|args\.value|call_async' } + ``` + + - `rg -n -i "deferred.fanout.review" ../promptforge-design/design/architect-promptforge-prompting-language.md` prints nothing. + - Each of these prints `1`: `rg -c -F "Tests: commands Verify runs as written" ../tools-public/planning/vibe-coder.md`, `rg -c -F "copy into each step every test case" ../tools-public/planning/vibe-coder.md`, `rg -c -F "assign the plan's seed copy" ../tools-public/planning/vibe-coder.md`, `rg -c -F "Steps carry their tests and a Tests line" ../tools-public/planning/vibe-coder.md`, `rg -c -F "name the assertion that fails there" ../tools-public/planning/debt-collector.md`, and `rg -c -F "Planned regression tests are checked against the endpoint code" ../tools-public/planning/debt-collector.md`. + - `rg -n "^" ../tools-public/planning/vibe-coder.md` still prints exactly two lines, opening then closing. + - `git -C ../tools-public diff --stat 6a038da` lists only `planning/vibe-coder.md` and `planning/debt-collector.md`, and `git -C ../tools-public diff 6a038da` touches only the lines named below. + - `git -C ../promptforge-design diff --stat a34adaa -- design` lists only the spec file, and `git -C ../promptforge-design status --porcelain` still lists ` M research/2026-10-10-subagent-tool-naming.md` and `?? research/2026-10-10-subagent-tool-catalog.md`, unstaged. + - Exit criteria: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` + - Stage the gateway sidecar: `cargo build --locked -p gateway --no-default-features`, then `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe`. Run `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` and `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, then always run `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc`, even when a command failed. + - `cargo fmt --all --check` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` and `$env:RUSTDOCFLAGS='-D warnings'; cargo doc -p promptforge --no-deps` + - `cargo +nightly-2026-09-05 xtask api --check` + - `cargo test -p build-xtask` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + - At `FULL` scope this list, plus the survey's build command, is the whole gate. The survey's "CI also runs" extras and the Workshop UI npm suites are not part of it. +- Shape: text and a test-file move with no failing-test-first shape. The test counts, the build ceiling, the docs builds, and the text searches above are its proof, so write no new test. +- `crates/promptforge-plugin/src/context.rs` (D1-11): in the `ToolContext` doc, after "...and the call's operations before the chain's next step." (line 19) and before "It is rooted at `/`.", insert: "Calls a model asks for in one round fork their identities from the chain together, so they are concurrent with each other: one call's write to a path conflicts with a sibling's read or write of it, as between fanout arms." Rewrap the `///` lines to the surrounding width. The sentence states the rule and names no tests. No new test is added, because `two_writes_by_two_identities_conflict` (`crates/promptforge-internal/vfs/src/handle/tests/claims.rs` line 62) and `a_sibling_read_of_a_siblings_write_conflicts` (`crates/promptforge-internal/vfs/src/handle/tests/happens_before.rs` line 156) already pin the rule. +- `vibe/2026-10-11-3-parallel-tool-calls.md` line 339 (D1-13): replace "and a batch keeps today's effect ids and issue order." with "and a batch keeps today's issue order. Its effect ids match today's only in a run with one live chain; with several, ids depend on how the chains interleave." Line 340, the falsifier, stays. The closed record is corrected in place, as `8cade0de4` and `0938ee342` did. +- `crates/promptforge-internal/lua/src/protocol.rs` lines 4-7 (D1-14): "the `chat` and `tool_call` rounds the `models.loop` shim yields" becomes "the `chat`, `tool_calls`, and `tool_call` rounds the `models.loop` shim yields". Rewrap the `//!` lines to the surrounding width. +- Parse-test split, so the next test author has room under the 500-line ceiling: + - Move the `tool_call` tests in `crates/promptforge-internal/lua/src/protocol/tests/parse.rs` (lines 93-287 of its 499) into a new `crates/promptforge-internal/lua/src/protocol/tests/parse_tool_call.rs`. Declare `mod parse_tool_call;` in `crates/promptforge-internal/lua/src/protocol/tests.rs` with the other `parse*` modules (lines 105-109), in alphabetical order. + - It is a pure move: the same test names and bodies. The new file opens with a `//!` line naming its one concern and `use super::*;`, as `parse_chat.rs` does, plus only the imports the moved tests need. Drop any import from `parse.rs` that only the moved tests used. +- Spec, `../promptforge-design/design/architect-promptforge-prompting-language.md` (572 lines). User: "make the tasks.fanout spec entry describe today's behavior". + - Rewrite the `tasks.fanout(worker, collection)` entry (lines 414-418) to today's contract: + - `worker` is a section heading string, such as `"### Worker"`, run once per member. + - Each arm sees its member as `item` and its 1-based position as `sys.index`. + - The collection is any Lua table: array members first, then hash pairs sorted by key as `{ key, value }`. A non-table raises an error that points at `prompt.list(heading)`, and an empty collection raises. + - It returns the arms' results in member order as read-only tables with `text`, `ok`, `item`, and `exhausted`, and `tostring` of a result is its `text`. + - An arm that exhausts its tool loop leaves a stub result and the rest continue. Any other arm failure cancels the live arms and re-raises. + - Recursion is capped at 8. + - Remove the entry's `**Target contract:**` paragraph (line 418: the `call_async`, `args` `{ index, value }`, and `await_all` contract), and make its status line `**Implementation status:** Active.`, the form its neighbors use (line 388). + - Rename the "Deferred fanout review" example (lines 476-505) to "Fanout review". Its prose no longer calls the helper a wrapper over `call_async` and `await_all`, and its body reads `item` and each result's `text` in place of `args.value` and joined results. Update any anchor in the spec that links the old title. +- `../tools-public/planning/vibe-coder.md`, inside `` (lines 230-270). This run already read that block, and the edits are additions, so later dispatches still work. + - In the step template (lines 250-260), add the line `- Tests: commands Verify runs as written, or none` directly below the template's `- Component:` line. It defines the Tests line that `` already names (line 397). + - In the paragraph at line 262, directly after "Each step names concrete artifacts - files, modules, functions, structs - without a full implementation.", insert: "The coding, review, and fix sub-agents read only the implementation contract, the project survey, and their step, so copy into each step every test case, error text, and acceptance detail it needs from the product, verification, and decision contracts. Never assign the plan's seed copy, `vibe/ACTIVE`, or the close commit to a step, since the Run Sequence owns them." + - After the last attribution line (line 574), add one blank line, then `*2026-10-11 - Claude Opus 5.5 (Cursor agent). Steps carry their tests and a Tests line; seeding and the close stay out of steps.*` +- `../tools-public/planning/debt-collector.md`, in `## Removal Plan`: + - Directly after "Give each fix of incorrect behavior a regression test that fails before the change; a test that already passes proves the finding false, so keep the test and skip the change." (line 178), insert: "Before writing a regression test into the plan, read the endpoint code it exercises and name the assertion that fails there; an assertion the endpoint already satisfies is not that test." + - After the last attribution line (line 255), add one blank line, then `*2026-10-11 - Claude Opus 5.5 (Cursor agent). Planned regression tests are checked against the endpoint code.*` +- Acceptance: nothing a prompt author, Plugin, or Host can observe changes. No edit reaches `../promptforge-docs`, `ToolCaller`, or `crates/promptforge/public-api.txt`, and every `.rs` file in `crates/` stays at or under 500 lines. +- Commits: + - The coding sub-agent leaves the edits in `../promptforge-design` and `../tools-public` unstaged. The session commits each file alone in its own repository after the coding sub-agent returns and before this step's `promptforge` commit is finalized, and amends that commit after any later round changes the file. Nothing is pushed. + - `../promptforge-design`: `Describe tasks.fanout as it ships`, staging only the spec file. + - `../tools-public`: `Carry each step's tests into the step` with only `planning/vibe-coder.md`, then `Check planned regression tests against the endpoint` with only `planning/debt-collector.md`. + - `promptforge`: `Document sibling calls and split the tool_call parse tests`, with a prose body, a bullet list of changes, `Design:` lines, and the trailer `Plan: vibe/2026-10-11-5-easy-fixes.md`. + + + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..850d2c112 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-10-11-5-easy-fixes.md From 2da0fe839a076f48187517f359e2ed3b097703e2 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 15:32:10 -0700 Subject: [PATCH 27/34] Document sibling calls and split the tool_call parse tests Tool calls a model asks for in one round now carry their concurrency rule in the documentation Plugin authors read: they fork from the calling chain together, so one call's write to a path conflicts with a sibling's read or write of it. The protocol overview now lists the batched round among the rounds the model loop yields. The tool-call request parsing tests move unchanged into a file of their own, which gives the general parse tests room to grow. Nothing a script, Plugin, or Host can observe changes. - `parse_tool_call` is a new test module, declared beside `parse`, `parse_chat`, and `parse_tasks`. It takes the eleven `tool_call` request tests from `parse.rs` with the same names and bodies, and `parse.rs` keeps the `infer`, `call`, `tool_calls`, and `local_tool_done` cases and the shared malformed-yield rejections. - `crates/promptforge-plugin/src/context.rs` states the sibling rule in the `ToolContext` access doc, between the chain ordering and the root path. Only the doc comment changes. - `tool_calls` now appears in the protocol module overview with `chat` and `tool_call` as a round the `models.loop` shim yields on the author's behalf. - `Request::from_yield` and every other non-test item are unchanged; outside the test modules, the Rust edits are doc comments only. Plan: vibe/2026-10-11-5-easy-fixes.md --- .../promptforge-internal/lua/src/protocol.rs | 18 +- .../lua/src/protocol/tests.rs | 8 +- .../lua/src/protocol/tests/parse.rs | 204 +----------------- .../lua/src/protocol/tests/parse_tool_call.rs | 203 +++++++++++++++++ crates/promptforge-plugin/src/context.rs | 11 +- vibe/2026-10-11-3-parallel-tool-calls.md | 2 +- vibe/2026-10-11-5-easy-fixes.md | 2 +- 7 files changed, 230 insertions(+), 218 deletions(-) create mode 100644 crates/promptforge-internal/lua/src/protocol/tests/parse_tool_call.rs diff --git a/crates/promptforge-internal/lua/src/protocol.rs b/crates/promptforge-internal/lua/src/protocol.rs index d68c65f75..f54dba083 100644 --- a/crates/promptforge-internal/lua/src/protocol.rs +++ b/crates/promptforge-internal/lua/src/protocol.rs @@ -1,15 +1,15 @@ //! The coroutine protocol: validated request and answer types for the //! yield/resume boundary between section Lua and the scheduler driver. //! -//! A suspending Engine call (`models.infer(prompt)`, `h:infer(prompt)`, `call`, -//! `tasks.spawn`, `tasks.fanout`, `tools.call`, `store.*`, and -//! the `chat` and `tool_call` rounds the `models.loop` shim yields on the -//! author's behalf) is a Lua-side shim that yields a request table; the driver validates the -//! yield into a [`Request`], dispatches it, and resumes the coroutine with -//! the `(ok, result)` envelope rendered from an [`Answer`]. The two enums -//! are the audit surface: what a script can cause the Engine to do is one -//! short read, and each variant's fields are the compiler-checked -//! per-message contract. +//! A suspending Engine call (`models.infer(prompt)`, `h:infer(prompt)`, +//! `call`, `tasks.spawn`, `tasks.fanout`, `tools.call`, `store.*`, and the +//! `chat`, `tool_calls`, and `tool_call` rounds the `models.loop` shim +//! yields on the author's behalf) is a Lua-side shim that yields a request +//! table; the driver validates the yield into a [`Request`], dispatches +//! it, and resumes the coroutine with the `(ok, result)` envelope rendered +//! from an [`Answer`]. The two enums are the audit surface: what a script +//! can cause the Engine to do is one short read, and each variant's fields +//! are the compiler-checked per-message contract. //! //! The submodules split the protocol along that boundary: `request` the //! request vocabulary (the [`Request`] and [`VfsOp`] enums and the diff --git a/crates/promptforge-internal/lua/src/protocol/tests.rs b/crates/promptforge-internal/lua/src/protocol/tests.rs index e835c3e17..ef8b2907e 100644 --- a/crates/promptforge-internal/lua/src/protocol/tests.rs +++ b/crates/promptforge-internal/lua/src/protocol/tests.rs @@ -2,9 +2,10 @@ //! //! The submodules follow the protocol's own split: `parse` the generic //! yield-to-request validation, `parse_chat` the message-list request, -//! `parse_tasks` the task-operation requests, `answer` the -//! answer-to-envelope round trips, and `answer_chat` the `chat` answer's -//! typed error. The helpers below are shared. +//! `parse_tasks` the task-operation requests, `parse_tool_call` the +//! `tool_call` request, `answer` the answer-to-envelope round trips, and +//! `answer_chat` the `chat` answer's typed error. The helpers below are +//! shared. use std::num::NonZeroU32; @@ -107,3 +108,4 @@ mod answer_chat; mod parse; mod parse_chat; mod parse_tasks; +mod parse_tool_call; diff --git a/crates/promptforge-internal/lua/src/protocol/tests/parse.rs b/crates/promptforge-internal/lua/src/protocol/tests/parse.rs index b36cec86d..b549a483c 100644 --- a/crates/promptforge-internal/lua/src/protocol/tests/parse.rs +++ b/crates/promptforge-internal/lua/src/protocol/tests/parse.rs @@ -1,7 +1,8 @@ //! Yield-to-request parsing for the leaf and structural requests (`infer`, -//! `call`, `tool_call`, `tool_calls`, `local_tool_done`), and the -//! malformed-yield rejections shared by every op. The task-operation -//! requests (`spawn`, `timer`, `concurrency`) are in `parse_tasks`. +//! `call`, `tool_calls`, `local_tool_done`), and the malformed-yield +//! rejections shared by every op. The `tool_call` request is in +//! `parse_tool_call` and the task-operation requests (`spawn`, `timer`, +//! `concurrency`) are in `parse_tasks`. use super::*; @@ -89,203 +90,6 @@ fn a_fanout_op_is_no_longer_a_request() { assert_direct_yield(Request::from_yield(&lua, &Value::Table(table))); } -#[test] -fn tool_call_parses_alias_and_args() { - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - let args = lua.create_table().expect("table creation cannot fail"); - args.raw_set("value", "hi").expect("raw_set"); - table.raw_set("args", args).expect("raw_set"); - let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); - match request { - Request::ToolCall { - alias, - args, - call_id, - turn, - } => { - assert_eq!(alias, "echo"); - assert_eq!(args, json!({ "value": "hi" })); - assert_eq!(call_id, None, "a script call omits the call id"); - assert_eq!(turn, None, "a script call omits the turn"); - } - other => panic!("expected a tool_call request, got {other:?}"), - } -} - -#[test] -fn tool_call_with_a_call_id_parses_it_as_a_model_issued_call() { - // The loop shim sets `call_id` from the model's tool call; the request - // stores it so the driver resumes with content and fires ToolResult - // under that id. - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - table.raw_set("call_id", "call_7").expect("raw_set"); - let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); - match request { - Request::ToolCall { alias, call_id, .. } => { - assert_eq!(alias, "echo"); - assert_eq!(call_id.as_deref(), Some("call_7")); - } - other => panic!("expected a tool_call request, got {other:?}"), - } -} - -#[test] -fn a_non_string_call_id_is_a_malformed_yield() { - // `call_id` is shim-produced, never author-supplied: a wrong shape is - // a corrupted yield, not a catchable call error. - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - table.raw_set("call_id", 7).expect("raw_set"); - assert_direct_yield(Request::from_yield(&lua, &Value::Table(table))); -} - -#[test] -fn a_tool_call_without_a_turn_parses_it_as_none() { - // A script call and the test-only `tools.call_as_model` hook leave - // `turn` out; the scheduler falls back to the live counter. - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - table.raw_set("call_id", "call_7").expect("raw_set"); - let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); - match request { - Request::ToolCall { turn, .. } => assert_eq!(turn, None), - other => panic!("expected a tool_call request, got {other:?}"), - } -} - -#[test] -fn a_tool_call_with_a_turn_parses_the_requesting_round() { - // The loop shim passes back the turn its chat round reported the - // batch under. - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - table.raw_set("call_id", "call_7").expect("raw_set"); - table.raw_set("turn", 3).expect("raw_set"); - let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); - match request { - Request::ToolCall { call_id, turn, .. } => { - assert_eq!(call_id.as_deref(), Some("call_7")); - assert_eq!(turn, Some(3)); - } - other => panic!("expected a tool_call request, got {other:?}"), - } -} - -#[test] -fn a_turn_outside_u32_or_not_an_integer_is_a_malformed_yield() { - // `turn` is shim-produced, never author-supplied: a wrong shape is a - // corrupted yield, not a catchable call error. - let lua = Lua::new(); - let bad_turns = [ - Value::Integer(-1), - Value::Integer(i64::from(u32::MAX) + 1), - Value::Number(2.0), - Value::Number(2.5), - Value::String(lua.create_string("2").expect("string")), - ]; - for bad in bad_turns { - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - table.raw_set("call_id", "call_7").expect("raw_set"); - table.raw_set("turn", bad).expect("raw_set"); - assert_direct_yield(Request::from_yield(&lua, &Value::Table(table))); - } -} - -#[test] -fn tool_call_without_args_parses_the_empty_object() { - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); - match request { - Request::ToolCall { args, .. } => assert_eq!(args, json!({})), - other => panic!("expected a tool_call request, got {other:?}"), - } -} - -#[test] -fn a_tool_call_with_a_tool_object_decodes_to_its_id() { - // The id-or-tool-object polymorphism at the protocol boundary: a tool - // object stands for its canonical id, never its wire name. - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - let descriptor = promptforge_types::tools::ToolDescriptor::new( - promptforge_types::tools::ToolId::parse("tools/echo").expect("valid id"), - "echo tool", - json!({ "type": "object" }), - ); - let handle = crate::LuaToolHandle::from_binding(&crate::ToolBinding::from_descriptor( - "tools_echo", - &descriptor, - )); - let userdata = lua.create_userdata(handle).expect("userdata"); - table.raw_set("alias", userdata).expect("raw_set"); - let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); - match request { - Request::ToolCall { alias, .. } => assert_eq!(alias, "tools/echo"), - other => panic!("expected a tool_call request, got {other:?}"), - } -} - -#[test] -fn a_tool_call_with_a_non_alias_alias_is_the_calls_error() { - // The author-facing argument error returns as the call's answer, - // framed byte-identically with the other author-argument failures. - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", 42).expect("raw_set"); - match Request::from_yield(&lua, &Value::Table(table)) { - YieldParse::Call(Answer::ToolCallResult(Err(Error::Lua(message)))) => { - assert_eq!( - message, - "tools.call takes a tool id, a local alias, or a tool object, got integer" - ); - } - other => panic!("expected the alias call error, got {other:?}"), - } -} - -#[test] -fn a_tool_call_with_a_non_table_args_is_the_calls_error() { - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - table.raw_set("args", 42).expect("raw_set"); - match Request::from_yield(&lua, &Value::Table(table)) { - YieldParse::Call(Answer::ToolCallResult(Err(Error::Lua(message)))) => { - assert_eq!(message, "args must be a table, got integer"); - } - other => panic!("expected the args call error, got {other:?}"), - } -} - -#[test] -fn a_tool_call_with_an_unrepresentable_args_table_is_the_calls_error() { - let lua = Lua::new(); - let table = request_table(&lua, "tool_call"); - table.raw_set("alias", "echo").expect("raw_set"); - let args = lua.create_table().expect("table creation cannot fail"); - let member = lua - .create_function(|_, ()| Ok(())) - .expect("function creation cannot fail"); - args.raw_set("f", member).expect("raw_set"); - table.raw_set("args", args).expect("raw_set"); - match Request::from_yield(&lua, &Value::Table(table)) { - YieldParse::Call(Answer::ToolCallResult(Err(Error::Lua(message)))) => { - assert_eq!(message, "args must be a JSON-representable table"); - } - other => panic!("expected the args call error, got {other:?}"), - } -} - #[test] fn a_tool_calls_yield_missing_its_turn_a_table_per_call_or_a_call_id_is_a_malformed_yield() { // Only the loop's step yields `tool_calls`, so a missing or wrong turn, diff --git a/crates/promptforge-internal/lua/src/protocol/tests/parse_tool_call.rs b/crates/promptforge-internal/lua/src/protocol/tests/parse_tool_call.rs new file mode 100644 index 000000000..e5e83c0f5 --- /dev/null +++ b/crates/promptforge-internal/lua/src/protocol/tests/parse_tool_call.rs @@ -0,0 +1,203 @@ +//! Yield parsing for the `tool_call` request: the alias or tool object, +//! the args table, and the shim-produced call id and turn, with every +//! author-argument failure as the call's own answer and every wrong shim +//! field as a malformed yield. + +use super::*; + +#[test] +fn tool_call_parses_alias_and_args() { + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + let args = lua.create_table().expect("table creation cannot fail"); + args.raw_set("value", "hi").expect("raw_set"); + table.raw_set("args", args).expect("raw_set"); + let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); + match request { + Request::ToolCall { + alias, + args, + call_id, + turn, + } => { + assert_eq!(alias, "echo"); + assert_eq!(args, json!({ "value": "hi" })); + assert_eq!(call_id, None, "a script call omits the call id"); + assert_eq!(turn, None, "a script call omits the turn"); + } + other => panic!("expected a tool_call request, got {other:?}"), + } +} + +#[test] +fn tool_call_with_a_call_id_parses_it_as_a_model_issued_call() { + // The loop shim sets `call_id` from the model's tool call; the request + // stores it so the driver resumes with content and fires ToolResult + // under that id. + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + table.raw_set("call_id", "call_7").expect("raw_set"); + let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); + match request { + Request::ToolCall { alias, call_id, .. } => { + assert_eq!(alias, "echo"); + assert_eq!(call_id.as_deref(), Some("call_7")); + } + other => panic!("expected a tool_call request, got {other:?}"), + } +} + +#[test] +fn a_non_string_call_id_is_a_malformed_yield() { + // `call_id` is shim-produced, never author-supplied: a wrong shape is + // a corrupted yield, not a catchable call error. + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + table.raw_set("call_id", 7).expect("raw_set"); + assert_direct_yield(Request::from_yield(&lua, &Value::Table(table))); +} + +#[test] +fn a_tool_call_without_a_turn_parses_it_as_none() { + // A script call and the test-only `tools.call_as_model` hook leave + // `turn` out; the scheduler falls back to the live counter. + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + table.raw_set("call_id", "call_7").expect("raw_set"); + let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); + match request { + Request::ToolCall { turn, .. } => assert_eq!(turn, None), + other => panic!("expected a tool_call request, got {other:?}"), + } +} + +#[test] +fn a_tool_call_with_a_turn_parses_the_requesting_round() { + // The loop shim passes back the turn its chat round reported the + // batch under. + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + table.raw_set("call_id", "call_7").expect("raw_set"); + table.raw_set("turn", 3).expect("raw_set"); + let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); + match request { + Request::ToolCall { call_id, turn, .. } => { + assert_eq!(call_id.as_deref(), Some("call_7")); + assert_eq!(turn, Some(3)); + } + other => panic!("expected a tool_call request, got {other:?}"), + } +} + +#[test] +fn a_turn_outside_u32_or_not_an_integer_is_a_malformed_yield() { + // `turn` is shim-produced, never author-supplied: a wrong shape is a + // corrupted yield, not a catchable call error. + let lua = Lua::new(); + let bad_turns = [ + Value::Integer(-1), + Value::Integer(i64::from(u32::MAX) + 1), + Value::Number(2.0), + Value::Number(2.5), + Value::String(lua.create_string("2").expect("string")), + ]; + for bad in bad_turns { + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + table.raw_set("call_id", "call_7").expect("raw_set"); + table.raw_set("turn", bad).expect("raw_set"); + assert_direct_yield(Request::from_yield(&lua, &Value::Table(table))); + } +} + +#[test] +fn tool_call_without_args_parses_the_empty_object() { + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); + match request { + Request::ToolCall { args, .. } => assert_eq!(args, json!({})), + other => panic!("expected a tool_call request, got {other:?}"), + } +} + +#[test] +fn a_tool_call_with_a_tool_object_decodes_to_its_id() { + // The id-or-tool-object polymorphism at the protocol boundary: a tool + // object stands for its canonical id, never its wire name. + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + let descriptor = promptforge_types::tools::ToolDescriptor::new( + promptforge_types::tools::ToolId::parse("tools/echo").expect("valid id"), + "echo tool", + json!({ "type": "object" }), + ); + let handle = crate::LuaToolHandle::from_binding(&crate::ToolBinding::from_descriptor( + "tools_echo", + &descriptor, + )); + let userdata = lua.create_userdata(handle).expect("userdata"); + table.raw_set("alias", userdata).expect("raw_set"); + let request = expect_request(Request::from_yield(&lua, &Value::Table(table))); + match request { + Request::ToolCall { alias, .. } => assert_eq!(alias, "tools/echo"), + other => panic!("expected a tool_call request, got {other:?}"), + } +} + +#[test] +fn a_tool_call_with_a_non_alias_alias_is_the_calls_error() { + // The author-facing argument error returns as the call's answer, + // framed byte-identically with the other author-argument failures. + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", 42).expect("raw_set"); + match Request::from_yield(&lua, &Value::Table(table)) { + YieldParse::Call(Answer::ToolCallResult(Err(Error::Lua(message)))) => { + assert_eq!( + message, + "tools.call takes a tool id, a local alias, or a tool object, got integer" + ); + } + other => panic!("expected the alias call error, got {other:?}"), + } +} + +#[test] +fn a_tool_call_with_a_non_table_args_is_the_calls_error() { + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + table.raw_set("args", 42).expect("raw_set"); + match Request::from_yield(&lua, &Value::Table(table)) { + YieldParse::Call(Answer::ToolCallResult(Err(Error::Lua(message)))) => { + assert_eq!(message, "args must be a table, got integer"); + } + other => panic!("expected the args call error, got {other:?}"), + } +} + +#[test] +fn a_tool_call_with_an_unrepresentable_args_table_is_the_calls_error() { + let lua = Lua::new(); + let table = request_table(&lua, "tool_call"); + table.raw_set("alias", "echo").expect("raw_set"); + let args = lua.create_table().expect("table creation cannot fail"); + let member = lua + .create_function(|_, ()| Ok(())) + .expect("function creation cannot fail"); + args.raw_set("f", member).expect("raw_set"); + table.raw_set("args", args).expect("raw_set"); + match Request::from_yield(&lua, &Value::Table(table)) { + YieldParse::Call(Answer::ToolCallResult(Err(Error::Lua(message)))) => { + assert_eq!(message, "args must be a JSON-representable table"); + } + other => panic!("expected the args call error, got {other:?}"), + } +} diff --git a/crates/promptforge-plugin/src/context.rs b/crates/promptforge-plugin/src/context.rs index 15d739954..08c141316 100644 --- a/crates/promptforge-plugin/src/context.rs +++ b/crates/promptforge-plugin/src/context.rs @@ -16,10 +16,13 @@ mod tests; /// /// The access is the call's own identity, forked from the calling chain's, /// so the chain's earlier work happens before the call's operations and -/// the call's operations before the chain's next step. It is rooted at -/// `/`. The context borrows everything for the call alone, so a Plugin -/// cannot store the access, move it into a spawned task or thread, or keep -/// it past the call. +/// the call's operations before the chain's next step. Calls a model asks +/// for in one round fork their identities from the chain together, so they +/// are concurrent with each other: one call's write to a path conflicts +/// with a sibling's read or write of it, as between fanout arms. It is +/// rooted at `/`. The context borrows everything for the call alone, so a +/// Plugin cannot store the access, move it into a spawned task or thread, +/// or keep it past the call. /// /// The services are the run's, such as its input broker. The Host-wide /// services never reach a call; a Plugin keeps any it needs from its diff --git a/vibe/2026-10-11-3-parallel-tool-calls.md b/vibe/2026-10-11-3-parallel-tool-calls.md index 74910c7a3..faf237fd4 100644 --- a/vibe/2026-10-11-3-parallel-tool-calls.md +++ b/vibe/2026-10-11-3-parallel-tool-calls.md @@ -336,7 +336,7 @@ This plan lands the parallel-tool-call and `tasks.fanout` parts of the subagents - The recursion-cap message names `tasks.fanout`; the collection key and member messages keep "fanout" as the operation's noun. - No replay flag gates the batch path. Raised during decomposition and settled by the session from the evidence below. - `crates/promptforge-internal/types/src/replay.rs` (lines 1-66) says an Engine change that alters a recorded run's behavior can stay replayable by running the new behavior live and setting a `Flags` bit the replay honors. - - Rationale: at the baseline `Flags` defines nothing beyond `Flags::EMPTY`, no code reads a bit for behavior, and no run-log replayer feeds recorded answers back (`crates/harness-internal/runner/src/recorder.rs`, `effect_loop.rs`). Live resume matches answers by `EffectId` (`crates/promptforge-internal/engine/src/execute/run/effect.rs`), and a batch keeps today's effect ids and issue order. A flag nothing reads would be dead code. + - Rationale: at the baseline `Flags` defines nothing beyond `Flags::EMPTY`, no code reads a bit for behavior, and no run-log replayer feeds recorded answers back (`crates/harness-internal/runner/src/recorder.rs`, `effect_loop.rs`). Live resume matches answers by `EffectId` (`crates/promptforge-internal/engine/src/execute/run/effect.rs`), and a batch keeps today's issue order. Its effect ids match today's only in a run with one live chain; with several, ids depend on how the chains interleave. A flag nothing reads would be dead code. - Falsifier: a replayer that matches answers by step rather than by effect id or issue order. Adding one later would need its own flag for runs recorded from this change on. - Rejected alternatives: - Splitting rounds in the loop machine, with a local-tool predicate and the four names moved into the Lua crate. Reason: two homes for the routing rules, and six more files touched. diff --git a/vibe/2026-10-11-5-easy-fixes.md b/vibe/2026-10-11-5-easy-fixes.md index d65c32eff..6676d925c 100644 --- a/vibe/2026-10-11-5-easy-fixes.md +++ b/vibe/2026-10-11-5-easy-fixes.md @@ -359,7 +359,7 @@ Two steps, each one `promptforge` commit, run in order. They are independent; St -### Step 2: Document sibling calls, split the tool_call parse tests, and fix the spec and tools +### Step 2: Document sibling calls, split the tool_call parse tests, and fix the spec and tools [completed] - Component: `none` - Tests: Verify runs every command below as written, in this order, at every scope. Run them from the `promptforge` repository root. A check that "prints nothing" passes when the command prints no output, even though `rg` then exits 1. From bc49d9670c662a6504ebcd83a7470bd0e7f96631 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 15:46:07 -0700 Subject: [PATCH 28/34] Close plan: easy-fixes Plan: vibe/2026-10-11-5-easy-fixes.md --- vibe/2026-10-11-5-easy-fixes.md | 12 ++++++------ vibe/ACTIVE | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/2026-10-11-5-easy-fixes.md b/vibe/2026-10-11-5-easy-fixes.md index 6676d925c..fb6fede88 100644 --- a/vibe/2026-10-11-5-easy-fixes.md +++ b/vibe/2026-10-11-5-easy-fixes.md @@ -4,22 +4,22 @@ overview: "Land the small fixes found after the parallel-tool-calls run: two pri todos: - id: route content: "Step 1: ToolRoute and tool_route shared by prepare_tool_call and the batch; prepare_catalog_call; delete runs_alone and the batch fallback arm" - status: pending + status: completed - id: callsite content: "Step 1: private CallSite enum through prepare_tool_call, ToolCallContinuation, and apply.rs; ToolCaller unchanged" - status: pending + status: completed - id: text content: "Step 2: ToolContext sibling sentence, vibe record effect-id correction, protocol.rs line, parse_tool_call.rs split" - status: pending + status: completed - id: spec content: "Step 2: tasks.fanout spec entry and review example describe today's behavior, committed alone in promptforge-design" - status: pending + status: completed - id: tools content: "Step 2: vibe-coder.md decomposition fixes and debt-collector.md test check, one tools-public commit per file" - status: pending + status: completed - id: verify content: "Each step: Testing Plan checks plus both nextest runs, both clippy runs, fmt, docs, xtask api --check, build-xtask, docs-claims" - status: pending + status: completed isProject: false --- diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 850d2c112..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-10-11-5-easy-fixes.md From 1f7f88dd73b781d2f7e9d8f6d49469ae616757e4 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 19:01:26 -0700 Subject: [PATCH 29/34] Report the tool calls each step withdraws Each step of a run now lists the tool calls it orphaned while the run goes on, so the caller can drop work whose answer will be discarded. The list is sorted by effect id, and effect ids now carry a public order that follows issue order within a run. A decided run lists nothing, because its teardown orphans the remaining effects without listing them and the caller drops a decided run's work whole. The caller still owes one answer for every listed call. The harness driver and the test drivers ignore the list, so run behavior does not change. - `Step::Pending` gains a public `withdrawn` field rather than a query on the run. The list belongs to the step that produced it, so a caller cannot read it at the wrong time. - `EffectId` derives `PartialOrd` and `Ord`, and its doc promises that an effect issued later in a run has a greater id. The order is public API, and the chain abort now sorts ids by it directly. - `Scheduler` collects orphaned tool-call ids in its own `withdrawn` buffer, which each step drains and sorts into `Step::Pending`. - `parallel_tool_calls-withdrawn.rs` holds four new tests in a child module loaded through `#[path]`, reaching the parent's private helpers through `super::*`. - `abort_effect` lists an id only when it removes a pending tool-call entry. Each id appears at most once, and no other kind of effect is listed. - `withdrawn` stays in effect-id order when one step cancels tasks in reverse spawn order, and it may name a call that the same step's `effects` also lists, because one step can issue a call and then orphan it. Tests pin both cases. - `self.withdrawn.clear()` runs at the run's end after teardown. Settling the abandoned tasks fills the buffer but teardown orphans the rest unlisted, so a decided run lists none rather than a partial set. - `crates/harness-internal/runner/src/effect_loop.rs` matches `Step::Pending` with `..` and ignores the list, as do the test drivers and the example, so no tool call is dropped early. Design: new surface-growth @ crates/promptforge-internal/engine/src/execute/run.rs::Step::Pending boundary: pub instead-of: temporal-coupling: a Run query valid only until the next step Design: new surface-growth @ crates/promptforge-internal/engine/src/execute/run/effect.rs::EffectId boundary: pub Plan: vibe/2026-10-11-6-tool-call-cap.md --- .../runner/src/effect_loop.rs | 4 +- .../engine/src/execute/run.rs | 9 +- .../engine/src/execute/run/effect.rs | 5 +- .../engine/src/execute/run/tests.rs | 5 +- .../engine/src/execute/scheduler.rs | 4 + .../engine/src/execute/scheduler/chain.rs | 6 +- .../engine/src/execute/scheduler/drive.rs | 11 +- .../tests/parallel_tool_calls-withdrawn.rs | 241 +++++++ .../src/execute/tests/parallel_tool_calls.rs | 14 +- .../src/execute/tests/run_termination.rs | 4 +- .../engine/src/execute/tests/serial_driver.rs | 1 + .../engine/src/test_support.rs | 4 +- .../engine/src/test_support/tokio_driver.rs | 4 +- crates/promptforge/examples/greeter.rs | 4 +- crates/promptforge/public-api.txt | 3 +- crates/promptforge/tests/suite/event.rs | 4 +- crates/promptforge/tests/suite/ids.rs | 1 + vibe/2026-10-11-6-tool-call-cap.md | 660 ++++++++++++++++++ vibe/ACTIVE | 1 + 19 files changed, 968 insertions(+), 17 deletions(-) create mode 100644 crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls-withdrawn.rs create mode 100644 vibe/2026-10-11-6-tool-call-cap.md create mode 100644 vibe/ACTIVE diff --git a/crates/harness-internal/runner/src/effect_loop.rs b/crates/harness-internal/runner/src/effect_loop.rs index ac235c366..d3a8b937b 100644 --- a/crates/harness-internal/runner/src/effect_loop.rs +++ b/crates/harness-internal/runner/src/effect_loop.rs @@ -161,7 +161,9 @@ impl Driver { self.recorder.end_run(self.run_id, outcome.clone()).await?; return Ok(outcome); } - Step::Pending { effects, events } => { + Step::Pending { + effects, events, .. + } => { // The run's own word, not a scan of its events: the // events are a report, not a control channel. A cancel // that fired before this step is handed to the run here diff --git a/crates/promptforge-internal/engine/src/execute/run.rs b/crates/promptforge-internal/engine/src/execute/run.rs index bcd82a8a5..7f22dddbe 100644 --- a/crates/promptforge-internal/engine/src/execute/run.rs +++ b/crates/promptforge-internal/engine/src/execute/run.rs @@ -38,13 +38,20 @@ pub enum Step { /// perform, in issue order. Each comes with the provenance of the task /// that built it. An empty list means every chain is waiting on an /// effect that an earlier step issued. `events` lists the events the - /// step reported, in order. + /// step reported, in order. `withdrawn` lists the tool calls the step + /// orphaned while the run goes on. It may name a call that `effects` + /// lists, because one step can issue a call and then orphan it. Pending { /// The effects the caller must perform and answer through /// [`Run::resume`]. effects: Vec<(EffectId, Provenance, Effect)>, /// The events the step reported. events: Vec, + /// The tool-call effects this step orphaned, in effect-id order. + /// Their answers will be discarded, so the caller may drop each one + /// it still holds; it still owes each one answer. Empty once the + /// run is decided, when the caller may drop everything it holds. + withdrawn: Vec, }, /// The run is over. /// diff --git a/crates/promptforge-internal/engine/src/execute/run/effect.rs b/crates/promptforge-internal/engine/src/execute/run/effect.rs index a763e2a4d..e5423c2b1 100644 --- a/crates/promptforge-internal/engine/src/execute/run/effect.rs +++ b/crates/promptforge-internal/engine/src/execute/run/effect.rs @@ -39,8 +39,9 @@ use crate::execute::protocol::{VfsOp, VfsOutcome}; /// [`Effect`] with its [`EffectAnswer`]. /// /// The id is opaque. The run allocates it from a run-wide counter, so the -/// same effect may get a different id in another run. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +/// same effect may get a different id in another run. Ids order by issue +/// within a run: an effect issued later has a greater id. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct EffectId(pub(crate) u64); impl EffectId { diff --git a/crates/promptforge-internal/engine/src/execute/run/tests.rs b/crates/promptforge-internal/engine/src/execute/run/tests.rs index dabc6bafc..1cab1ac94 100644 --- a/crates/promptforge-internal/engine/src/execute/run/tests.rs +++ b/crates/promptforge-internal/engine/src/execute/run/tests.rs @@ -283,7 +283,10 @@ fn done_is_withheld_while_a_store_effect_is_outstanding_and_delivered_after_drop // its answer, so `Done` waits. run.cancel(); let step = run.step(); - let Step::Pending { effects, events } = step else { + let Step::Pending { + effects, events, .. + } = step + else { panic!("Done is withheld while the store effect is unanswered, got {step:?}"); }; assert!(effects.is_empty(), "a torn-down run issues nothing"); diff --git a/crates/promptforge-internal/engine/src/execute/scheduler.rs b/crates/promptforge-internal/engine/src/execute/scheduler.rs index 27515cfa9..8e8dfe052 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler.rs @@ -218,6 +218,9 @@ pub(crate) struct Scheduler { /// set when its answer arrives, so the set stays bounded by the /// orphans whose answers have not landed. orphaned: HashSet, + /// Tool-call effects orphaned since the last step, drained into + /// `Step::Pending`. + withdrawn: Vec, /// The run's phase. phase: Phase, /// The most chains one run may start: the arena indexes chains by @@ -262,6 +265,7 @@ impl Scheduler { tasks: HashMap::new(), issued: Vec::new(), orphaned: HashSet::new(), + withdrawn: Vec::new(), phase: Phase::Fresh, max_chains: u32::MAX as usize, next_effect: 0, diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/chain.rs b/crates/promptforge-internal/engine/src/execute/scheduler/chain.rs index 89234a550..b084c6c8f 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/chain.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/chain.rs @@ -253,7 +253,7 @@ impl Scheduler { .filter(|(_, pending)| pending.chain == id) .map(|(effect, _)| *effect) .collect(); - effects.sort_unstable_by_key(|effect| effect.get()); + effects.sort_unstable(); for effect in effects { self.abort_effect(effect); } @@ -290,7 +290,8 @@ impl Scheduler { /// when it arrives, is discarded rather than failing the run. A tool /// call's identity ends here as its answer would end it, so what the /// tool already did is joined into the parked chain and anything it - /// tries later is refused. The caller still owes the answer: `Done` + /// tries later is refused. A tool call is also listed as withdrawn, + /// for the step to report. The caller still owes the answer: `Done` /// waits for every issued effect, so the run ends only once the /// caller has answered all of them. pub(super) fn abort_effect(&mut self, effect: EffectId) { @@ -300,6 +301,7 @@ impl Scheduler { }) = self.pending.remove(&effect) { self.end_tool_call(chain, call.exec); + self.withdrawn.push(effect); } self.orphaned.insert(effect); } diff --git a/crates/promptforge-internal/engine/src/execute/scheduler/drive.rs b/crates/promptforge-internal/engine/src/execute/scheduler/drive.rs index 93b18e783..f9ab86b52 100644 --- a/crates/promptforge-internal/engine/src/execute/scheduler/drive.rs +++ b/crates/promptforge-internal/engine/src/execute/scheduler/drive.rs @@ -54,6 +54,8 @@ impl Scheduler { ))); } let effects = std::mem::take(&mut self.issued); + let mut withdrawn = std::mem::take(&mut self.withdrawn); + withdrawn.sort_unstable(); let events = self.ctx.take_events(); match &self.phase { Phase::Ending(_) if self.pending.is_empty() && self.orphaned.is_empty() => { @@ -79,7 +81,11 @@ impl Scheduler { ))), events, }, - Phase::Fresh | Phase::Running | Phase::Ending(_) => Step::Pending { effects, events }, + Phase::Fresh | Phase::Running | Phase::Ending(_) => Step::Pending { + effects, + events, + withdrawn, + }, } } @@ -175,6 +181,9 @@ impl Scheduler { } self.settle_all_tasks(AbandonReason::RunTerminated); self.teardown(); + // Settling withdrew the abandoned tasks' calls, but teardown orphans + // the rest unlisted, so a decided run lists none rather than some. + self.withdrawn.clear(); self.ctx.emitter().report( self.ctx.prompt().title(), if result.is_ok() { diff --git a/crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls-withdrawn.rs b/crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls-withdrawn.rs new file mode 100644 index 000000000..30e997ee2 --- /dev/null +++ b/crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls-withdrawn.rs @@ -0,0 +1,241 @@ +//! The step's `withdrawn` list: a step that cancels a task parked on a +//! batch lists the calls it orphaned, the calls the same step issued +//! included, in effect-id order however many tasks it cancels, and a +//! decided run's step lists none. + +use super::*; + +/// The effects one step issued, in issue order, and the tool calls it +/// withdrew. Panics when the run is over. +fn withdrawals(step: Step) -> (Vec<(EffectId, Effect)>, Vec) { + let Step::Pending { + effects, withdrawn, .. + } = step + else { + panic!("the run is still going: {step:?}"); + }; + let effects = effects + .into_iter() + .map(|(id, _, effect)| (id, effect)) + .collect(); + (effects, withdrawn) +} + +/// Main spawns Child and waits on one model round while Child parks on a +/// tool-call batch, then cancels Child and lingers on a second round. +const CANCELLED_CHILD: &str = "---\nname: loop\ndescription: d\npromptforge: 0\n---\n\n# Loop\n\n\ + ## Main\n\n```lua\n\ + local t = tasks.spawn('## Child')\n\ + models.infer('wait')\n\ + tasks.cancel(t)\n\ + models.infer('linger')\n\ + return 'cancelled'\n```\n\n\ + ## Child\n\n```lua\n\ + local msgs = messages.new()\n\ + msgs:user('go')\n\ + models.loop(msgs)\n\ + return 'child done'\n```\n"; + +/// A run of [`CANCELLED_CHILD`] past its first step, with the ids of +/// Main's `wait` round and Child's loop round. The fixture must outlive +/// the run. +fn cancelled_child() -> (Run, RunFixture, EffectId, EffectId) { + let prompt = parse(CANCELLED_CHILD); + let (state, fixture) = loop_context(&prompt, echo_tools()); + let mut run = Run::from_state(state); + let (rounds, withdrawn) = withdrawals(run.step()); + assert!( + withdrawn.is_empty(), + "nothing is withdrawn yet: {withdrawn:?}" + ); + let origin = |wanted: ReplyOrigin| { + rounds + .iter() + .find_map(|(id, effect)| match effect { + Effect::Chat { round, .. } if round.origin == wanted => Some(*id), + _ => None, + }) + .unwrap_or_else(|| panic!("a {wanted:?} round: {rounds:?}")) + }; + let (infer, round) = (origin(ReplyOrigin::Infer), origin(ReplyOrigin::Chat)); + (run, fixture, infer, round) +} + +/// Child's round answer: two echo calls, `c1` then `c2`. +fn two_echoes() -> EffectAnswer { + calls_reply(&[ + ("c1", "echo", json!({ "value": "one" })), + ("c2", "echo", json!({ "value": "two" })), + ]) +} + +/// The ids of the tool calls among `effects`, in issue order. +fn tool_call_ids(effects: &[(EffectId, Effect)]) -> Vec { + effects + .iter() + .filter(|(_, effect)| matches!(effect, Effect::ToolCall { .. })) + .map(|(id, _)| *id) + .collect() +} + +#[test] +fn cancelling_a_task_withdraws_its_unanswered_tool_calls() { + let (mut run, _fixture, infer, round) = cancelled_child(); + run.resume(round, two_echoes()); + let (issued, withdrawn) = withdrawals(run.step()); + let calls = tool_call_ids(&issued); + assert_eq!(calls.len(), 2, "both calls are in flight: {issued:?}"); + assert!( + withdrawn.is_empty(), + "issuing withdraws nothing: {withdrawn:?}" + ); + run.resume(infer, text_reply("go")); + let (linger, effect) = { + let (mut issued, withdrawn) = withdrawals(run.step()); + assert_eq!( + withdrawn, calls, + "the cancel withdraws both calls, c1 first" + ); + assert_eq!(issued.len(), 1, "only Main's next round: {issued:?}"); + issued.remove(0) + }; + assert!( + matches!(&effect, Effect::Chat { round, .. } if round.origin == ReplyOrigin::Infer), + "the cancel issues only Main's next round: {effect:?}" + ); + for id in calls { + run.resume(id, EffectAnswer::Dropped); + } + run.resume(linger, text_reply("done")); + let Step::Done { result, .. } = run.step() else { + panic!("Main's second round ends the run"); + }; + assert_eq!(text_of(result), "cancelled"); +} + +#[test] +fn a_call_issued_and_withdrawn_in_one_step_is_listed_twice() { + let (mut run, _fixture, infer, round) = cancelled_child(); + run.resume(round, two_echoes()); + run.resume(infer, text_reply("go")); + let (issued, withdrawn) = withdrawals(run.step()); + let calls = tool_call_ids(&issued); + assert_eq!( + calls.len(), + 2, + "Child issues both calls before Main cancels it: {issued:?}" + ); + assert_eq!(withdrawn, calls, "the same step withdraws both calls"); + for (id, effect) in &issued { + let answer = match effect { + Effect::ToolCall { .. } => EffectAnswer::Dropped, + _ => text_reply("done"), + }; + run.resume(*id, answer); + } + let Step::Done { result, .. } = run.step() else { + panic!("Main's second round ends the run"); + }; + assert_eq!(text_of(result), "cancelled"); +} + +#[test] +fn a_decided_run_lists_no_withdrawn_calls() { + let (mut run, _fixture, infer, round) = cancelled_child(); + run.resume(round, two_echoes()); + let (issued, _) = withdrawals(run.step()); + let calls = tool_call_ids(&issued); + assert_eq!(calls.len(), 2, "both calls are in flight: {issued:?}"); + run.cancel(); + let (issued, withdrawn) = withdrawals(run.step()); + assert!(run.decided(), "the cancel decides the run"); + assert!( + issued.is_empty(), + "a torn-down run issues nothing: {issued:?}" + ); + assert!( + withdrawn.is_empty(), + "the run's end abandoned Child, yet a decided run lists none: {withdrawn:?}" + ); + for id in calls.into_iter().chain([infer]) { + run.resume(id, EffectAnswer::Dropped); + } + let Step::Done { result, .. } = run.step() else { + panic!("every orphan is answered, so the run is done"); + }; + assert!( + matches!(result, RunResult::Cancelled), + "the run ends cancelled: {result:?}" + ); +} + +/// Main spawns A and then B, each parked on a one-call batch, waits on a +/// model round, then cancels B before A and lingers on a second round. +const CANCELLED_PAIR: &str = "---\nname: loop\ndescription: d\npromptforge: 0\n---\n\n# Loop\n\n\ + ## Main\n\n```lua\n\ + local a = tasks.spawn('## A')\n\ + local b = tasks.spawn('## B')\n\ + models.infer('wait')\n\ + tasks.cancel(b)\n\ + tasks.cancel(a)\n\ + models.infer('linger')\n\ + return 'cancelled'\n```\n\n\ + ## A\n\n```lua\n\ + local msgs = messages.new()\n\ + msgs:user('a')\n\ + models.loop(msgs)\n\ + return 'a done'\n```\n\n\ + ## B\n\n```lua\n\ + local msgs = messages.new()\n\ + msgs:user('b')\n\ + models.loop(msgs)\n\ + return 'b done'\n```\n"; + +#[test] +fn one_step_lists_every_tasks_withdrawn_calls_in_effect_id_order() { + let prompt = parse(CANCELLED_PAIR); + let (state, _fixture) = loop_context(&prompt, echo_tools()); + let mut run = Run::from_state(state); + let (rounds, _) = withdrawals(run.step()); + let asking = |user: &str| { + rounds + .iter() + .find_map(|(id, effect)| match effect { + Effect::Chat { messages, .. } + if messages.iter().any(|message| message.content() == user) => + { + Some(*id) + } + _ => None, + }) + .unwrap_or_else(|| panic!("a round asking {user:?}: {rounds:?}")) + }; + for task in ["a", "b"] { + let call = (task, "echo", json!({ "value": task })); + run.resume(asking(task), calls_reply(&[call])); + } + let (issued, _) = withdrawals(run.step()); + let values: Vec<&str> = issued + .iter() + .map(|(_, effect)| call_value(effect)) + .collect(); + assert_eq!(values, ["a", "b"], "A's call issues before B's"); + run.resume(asking("wait"), text_reply("go")); + let (linger, withdrawn) = withdrawals(run.step()); + assert_eq!( + withdrawn, + tool_call_ids(&issued), + "Main cancels B first, yet A's call is listed first" + ); + for (id, _) in &issued { + run.resume(*id, EffectAnswer::Dropped); + } + let [(linger, _)] = linger.as_slice() else { + panic!("the cancels issue only Main's next round: {linger:?}"); + }; + run.resume(*linger, text_reply("done")); + let Step::Done { result, .. } = run.step() else { + panic!("Main's second round ends the run"); + }; + assert_eq!(text_of(result), "cancelled"); +} diff --git a/crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls.rs b/crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls.rs index 576cf115c..654a62cdd 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls.rs @@ -4,8 +4,10 @@ //! records append in call order. A local tool or a task built-in stops the //! batch and runs alone between its neighbors, a call that cannot be //! bound stops it and raises once the issued calls land, and a task -//! cancelled while parked on a batch orphans every call in it. Each test -//! steps a `Run` by hand, so it sees exactly what each step issued. +//! cancelled while parked on a batch orphans every call in it and lists +//! each in the step's `withdrawn`, which a decided run leaves empty. Each +//! test steps a `Run` by hand, so it sees exactly what each step issued +//! and withdrew. use promptforge_types::event::{Event, ReplyOrigin}; @@ -18,7 +20,10 @@ use crate::model::{Completion, CompletionResult, ToolCall}; /// The effects one step issued, in issue order, and the events it /// reported. Panics when the run is over. fn pending(step: Step) -> (Vec<(EffectId, Effect)>, Vec) { - let Step::Pending { effects, events } = step else { + let Step::Pending { + effects, events, .. + } = step + else { panic!("the run is still going: {step:?}"); }; let effects = effects @@ -294,3 +299,6 @@ fn cancelling_a_task_parked_on_a_batch_orphans_every_call() { }; assert_eq!(text_of(result), "cancelled"); } + +#[path = "parallel_tool_calls-withdrawn.rs"] +mod withdrawn; diff --git a/crates/promptforge-internal/engine/src/execute/tests/run_termination.rs b/crates/promptforge-internal/engine/src/execute/tests/run_termination.rs index 71ad2c2b2..e022cd81c 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/run_termination.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/run_termination.rs @@ -46,7 +46,9 @@ fn cancel_at_first_chat_round(mut run: Run, recorder: &TaskRecorder) -> RunResul forward(events, recorder); return result; } - Step::Pending { effects, events } => { + Step::Pending { + effects, events, .. + } => { forward(events, recorder); assert!( !effects.is_empty() || cancelled, diff --git a/crates/promptforge-internal/engine/src/execute/tests/serial_driver.rs b/crates/promptforge-internal/engine/src/execute/tests/serial_driver.rs index 8cfae15f9..0407d9d9c 100644 --- a/crates/promptforge-internal/engine/src/execute/tests/serial_driver.rs +++ b/crates/promptforge-internal/engine/src/execute/tests/serial_driver.rs @@ -199,6 +199,7 @@ pub(super) fn drive_batched(mut run: Run, batching: Batching) -> Outcome { Step::Pending { effects: issued, events: more, + .. } => { events.extend(more); for (id, provenance, effect) in issued { diff --git a/crates/promptforge-internal/engine/src/test_support.rs b/crates/promptforge-internal/engine/src/test_support.rs index 99004ee88..ea940dbc2 100644 --- a/crates/promptforge-internal/engine/src/test_support.rs +++ b/crates/promptforge-internal/engine/src/test_support.rs @@ -101,7 +101,9 @@ pub fn drive( reported.extend(events); return (result, reported); } - Step::Pending { effects, events } => { + Step::Pending { + effects, events, .. + } => { reported.extend(events); if effects.is_empty() { // Every effect is answered the step it is issued, so a diff --git a/crates/promptforge-internal/engine/src/test_support/tokio_driver.rs b/crates/promptforge-internal/engine/src/test_support/tokio_driver.rs index c461babf5..85985f8c6 100644 --- a/crates/promptforge-internal/engine/src/test_support/tokio_driver.rs +++ b/crates/promptforge-internal/engine/src/test_support/tokio_driver.rs @@ -197,7 +197,9 @@ impl<'a> TokioDriver<'a> { RunResult::Failure(error) => Err(error.into_inner()), }; } - Step::Pending { effects, events } => { + Step::Pending { + effects, events, .. + } => { // The run's own word, not a scan of its events: the // events are a report only. let decided = self.run.decided(); diff --git a/crates/promptforge/examples/greeter.rs b/crates/promptforge/examples/greeter.rs index 37098ab05..34eafc6cb 100644 --- a/crates/promptforge/examples/greeter.rs +++ b/crates/promptforge/examples/greeter.rs @@ -104,7 +104,9 @@ pub fn greet() -> Result> { // effect exactly once. let result = loop { match run.step() { - Step::Pending { effects, events } => { + Step::Pending { + effects, events, .. + } => { log.extend(events); for (id, _provenance, effect) in effects { if run.decided() { diff --git a/crates/promptforge/public-api.txt b/crates/promptforge/public-api.txt index 83ef21fea..3e7cc6e30 100644 --- a/crates/promptforge/public-api.txt +++ b/crates/promptforge/public-api.txt @@ -211,7 +211,7 @@ derives promptforge::effect::AnswerRecord: Clone, Debug, Deserialize, PartialEq, derives promptforge::effect::ChatAnswerRecord: Clone, Debug, Deserialize, Eq, PartialEq, Serialize derives promptforge::effect::Effect: Debug derives promptforge::effect::EffectAnswer: Debug -derives promptforge::effect::EffectId: Clone, Copy, Debug, Eq, Hash, PartialEq +derives promptforge::effect::EffectId: Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd derives promptforge::effect::EffectRecord: Clone, Debug, Deserialize, PartialEq, Serialize derives promptforge::effect::Round: Clone, Copy, Debug, Eq, PartialEq derives promptforge::effect::ToolAnswerRecord: Clone, Debug, Deserialize, Eq, PartialEq, Serialize @@ -735,6 +735,7 @@ pub promptforge::Step::Done::result: promptforge::RunResult pub promptforge::Step::Pending { .. } pub promptforge::Step::Pending::effects: alloc::vec::Vec<(promptforge::effect::EffectId, promptforge::ids::Provenance, promptforge::effect::Effect)> pub promptforge::Step::Pending::events: alloc::vec::Vec +pub promptforge::Step::Pending::withdrawn: alloc::vec::Vec pub promptforge::UnavailablePlugin::plugin: promptforge::plugins::PluginId pub promptforge::UnavailablePlugin::reason: alloc::string::String pub promptforge::UnmetRequirement::actual: alloc::string::String diff --git a/crates/promptforge/tests/suite/event.rs b/crates/promptforge/tests/suite/event.rs index 576349152..6ec939950 100644 --- a/crates/promptforge/tests/suite/event.rs +++ b/crates/promptforge/tests/suite/event.rs @@ -55,7 +55,9 @@ fn answer(effect: Effect) -> EffectAnswer { fn drive(mut run: Run, mut record: impl FnMut(Vec)) -> RunResult { loop { match run.step() { - Step::Pending { effects, events } => { + Step::Pending { + effects, events, .. + } => { record(events); for (id, _provenance, effect) in effects { run.resume(id, answer(effect)); diff --git a/crates/promptforge/tests/suite/ids.rs b/crates/promptforge/tests/suite/ids.rs index 1c571174f..f098773b3 100644 --- a/crates/promptforge/tests/suite/ids.rs +++ b/crates/promptforge/tests/suite/ids.rs @@ -101,6 +101,7 @@ fn drive( Step::Pending { effects: batch, events: reported, + .. } => { assert!( !batch.is_empty() || run.decided(), diff --git a/vibe/2026-10-11-6-tool-call-cap.md b/vibe/2026-10-11-6-tool-call-cap.md new file mode 100644 index 000000000..e35ba84ac --- /dev/null +++ b/vibe/2026-10-11-6-tool-call-cap.md @@ -0,0 +1,660 @@ +--- +name: Tool call cap, oldest first +overview: Bound the Plugin tool calls running at once across a run, with the cap living only in the Harness. The Engine lists the tool calls it withdraws when a task ends early. The Harness runs at most the cap and, whenever a slot frees, starts the waiting call with the lowest effect id, so the oldest batch finishes first. There is no queue in the Engine. +todos: + - id: engine-withdrawn + content: "Step 1: EffectId derives PartialOrd and Ord; Step::Pending gains withdrawn, filled by abort_effect for tool-call continuations and cleared when the run is decided; update every exhaustive Step::Pending pattern; bless and review public-api.txt; three Engine tests in parallel_tool_calls.rs" + status: pending + - id: harness-cap + content: "Step 2: ToolPerformer::max_in_flight, a default method returning 10; Flights keeps waiting futures in a BTreeMap by EffectId, caps tool calls in flight, starts the lowest id when a slot frees, and drains waiting calls in abort(reach), which gains Reach::Withdrawn; start drops its survives_stop parameter" + status: pending + - id: harness-driver + content: "Step 2: Driver builds Flights with the performer's cap, sends tool calls through Flights::queue, and drops the step's withdrawn ids through drop_in_flight after starting its new effects; RunControl and Harness::new docs name the cap and waiting calls" + status: pending + - id: harness-tests + content: "Step 2: end-to-end tests: the cap holds extra calls, waiting calls start oldest first, a call withdrawn in the step that issued it never starts, cancelling a task drops its waiting and running calls, a cancel and a stop drop waiting calls unstarted, a stop starts a waiting call that survives stops" + status: pending + - id: verify + content: "Each step: both nextest runs, both clippy runs, fmt, both docs runs, xtask api --check, build-xtask, docs-claims" + status: pending +isProject: false +--- + +# Cap tool calls in flight, oldest first + +### Context for a fresh reader + +- Repository: `c:\Users\Vinnie\cursor\promptforge` (git, branch `master`). Paths are relative to it. +- Baseline: commit `bc49d9670` (Close plan: easy-fixes). Every line number and count here was verified there; search by name if they have moved. +- Shell: PowerShell on Windows. `$env:NAME='value'; ` sets a variable for the rest of the session, which is harmless for the gates below. +- Terms: + - *Engine*, *Harness*, and *Host*: the Engine runs a prompt and performs no I/O. The Harness performs its effects for a Host, the application running the prompt, such as the Workshop desktop app. The Engine and the Harness talk only through `run.step()`, which returns `Step::Pending { effects, events }` or `Step::Done`, and `run.resume(id, answer)`. + - *Plugin*: a package of tools installed under a local name. A call to one of its tools is a *catalog call*, which the Engine issues as an `Effect::ToolCall`. + - *chain*, *task*, and *fanout arm*: a chain is one line of execution in a run, a section's Lua running as a coroutine. A task is a chain that runs in the background, started with `tasks.spawn`. `tasks.fanout` starts one task, an arm, per item, and the arms run at the same time. + - *effect*: one thing the Engine needs done, such as a model round or a tool call. Each has an `EffectId` from one run-wide counter, so ids follow issue order. + - *round* and *batch*: one model reply inside `models.loop`, and the tool calls it asks for. Since `1ce5831d0`, the Engine issues a round's catalog calls in one step, one after another in call order, so a batch's calls get consecutive effect ids. It resumes the chain only when every call is answered. When the Engine stops short at a local tool or a task built-in, the calls after it go out later as a new batch. A script `tools.call` is a batch of one. + - *orphan*: an issued effect whose chain the Engine has ended, for example by `tasks.cancel`. The Engine discards its answer but keeps the run open until the answer arrives. + - *withdrawn*: a tool-call orphan the Engine lists in the step that orphaned it, so the caller can drop it. + - *Stop* and *Cancel*: + - Stop is the Workshop button. The Harness's `stop_round` drops every in-flight effect except tool calls whose descriptor sets `survives_stop` (the operator question, `user-input/ask`), and the run goes on. + - Cancel ends the whole run. The run becomes *decided*, and the Harness drops everything. +- Plan record and commits: + - When execution starts, copy this plan to `vibe/2026-10-11-6-tool-call-cap.md`, the next free number for that day. Mark each step `[completed]` in that copy as it lands. + - Each commit message has a one-line summary, a body, `Design:` lines, and a `Plan: vibe/2026-10-11-6-tool-call-cap.md` trailer. Close with `Close plan: tool-call-cap`. Nothing is pushed. + + + +## Product Requirements + +This plan bounds how many Plugin tool calls run at once across a whole run. The cap lives only in the Harness, which enforces it. The Engine lists the tool calls it withdraws when a task ends early. The Harness starts the oldest waiting call whenever a slot frees, so the oldest batch finishes first, and drops waiting calls on Stop, Cancel, or withdrawal. + +- Problem and users: + - Nothing limits tool calls in flight today. One round can ask for 50 fetches, or 8 fanout arms can each ask for 10, which is 80 at once. That can overload sites, hit provider rate limits, or swamp the Host. + - This affects prompt authors, whose runs fan out, and Hosts such as the Workshop, which pay for the load. +- Goals: + - At most `ToolPerformer::max_in_flight()` tool calls run at once in a run. The default is 10. + - Oldest first: when a slot frees, the waiting call with the lowest effect id starts. A batch's calls have consecutive ids, so the oldest batch finishes before a newer one starts, and its chain gets its result as early as possible. + - Nothing starves: every new call goes behind every call already waiting. A chain that runs forever has at most one batch out at a time, and each new batch goes to the back. + - Stop and Cancel reach waiting calls as well as running ones. + - A withdrawn call never starts, and one already running is dropped. +- Non-goals: + - Changing what a stopped round returns. `models.loop` still discards the round. + - Tool effect labels or an Ask mode. + - Capping model rounds, timers, or store effects. + - Withdrawing orphaned model rounds. + - A queue in the Engine, or sharing `max_concurrency`. + - A fill order that lets a later batch go ahead of a big batch's leftovers. Waiting needs more than the cap of calls at once, which mostly takes a fanout, and oldest-first cannot starve. + - A Host-facing setting for the cap, or changing it mid-run. The real tool performer keeps the default of 10. +- Success criteria: + - No run ever has more than the cap of tool calls running. + - When a slot frees, the waiting call with the lowest effect id starts. + - A Stop answers every waiting call that does not survive stops with `Dropped`, without starting it. A waiting call that survives stops starts once a slot frees. + - A Cancel answers every waiting call `Dropped` without starting it. + - Cancelling a task drops its waiting and running calls, and the run finishes without them. + - Every existing test passes. Existing tests change only by the mechanical updates to exhaustive `Step::Pending` patterns; new tests and helpers may join existing files. + - Every exit gate passes. +- Constraints: + - The Engine stays sans-IO and deterministic. The order in which answers land stays an input the Engine records, so the Harness's start order changes no replay. + - Every `.rs` file stays at or under 500 lines; the ceiling fails a file over 500. `harness-internal/runner/src/effect_loop.rs` is 487, so it may grow by at most 13 lines. The Harness's new logic and its prose go in `effect_loop-flight.rs` (181 lines). + - Clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, and dead code is fatal under the gate. + - Engine docs and comments call the Harness "the caller" and name the Harness only to say a responsibility belongs to it, never how the Harness performs it (root `AGENTS.md`). `docs-claims.mjs` catches only lowercase and compound forms, so this is checked by review. +- Open questions: none. + +## Functional Specification + +Prompts behave as before, except that tool calls beyond the cap wait in the Harness and start oldest first. Stop, Cancel, and withdrawal now also reach calls that are waiting. + +- Actors and workflows: + - The Engine issues every tool call as today. Each step lists the tool calls it withdrew. + - The Harness starts tool calls while fewer than the cap are running and holds the rest. Each time a running tool call lands or is dropped, it starts the waiting call with the lowest effect id. +- Inputs and outputs: + - `EffectId` derives `PartialOrd` and `Ord`, so ids order by issue within a run. + - `ToolPerformer::max_in_flight()` is a new trait method with a default of 10. A performer overrides it to set a different cap. + - `Step::Pending` gains `withdrawn: Vec`: the tool-call effects this step orphaned while the run goes on, in effect-id order. A decided run's steps list none. +- States and validation: + - **Order.** The lowest effect id starts first. Ids come from one run-wide counter in issue order (`Scheduler::issue`, `scheduler.rs` lines 305-307), and the Engine issues a batch's calls one after another, so a batch's calls are consecutive and the oldest batch goes first. + - **Example.** With a cap of 10, batch A asks for 12 calls first, and batch B asks for 4 in a later step while A1 to A10 run: + - A1 to A10, effect ids 1 to 10, start at once. + - A11 and A12, ids 11 and 12, start next, as slots free. + - B1 to B4, ids 13 to 16, start after A's. +- Errors and recovery: + - Stop drops running and waiting tool calls that do not survive stops, and answers each `Dropped`. A call that survives stops keeps running, or keeps waiting and starts once a slot frees. + - A decided run, or a Cancel, drops every waiting call along with everything in flight. + - Withdrawn calls: for each id, a waiting call is dropped unstarted and answered `Dropped`, and a running call is dropped as Stop drops one and answered `Dropped`. Each effect gets exactly one answer, and an id the Harness no longer holds is ignored. The Engine discards answers for orphans, as today. + - Store effects and model rounds are never withdrawn. +- Security and privacy behavior: unchanged. Each call still forks its own access when it is issued, so a waiting call already holds its identity. A batch's calls are concurrent siblings either way, so store claims behave as today. +- Acceptance criteria: the success criteria, each pinned by a test in the Testing Plan. + + + + +## Technical Design + +The Harness owns the cap and the order. The Engine adds one list per step and one derive, and changes nothing in its batch code or in `Effect::ToolCall`. The Harness keeps waiting calls inside `Flights`, which already owns every in-flight effect and its Stop and Cancel paths. A waiting call is just a performer future that hasn't been polled yet, because the loop builds each tool call's future as an `async move` block that calls the performer only when polled (`effect_loop.rs` lines 340-353). + +```mermaid +flowchart LR + engineIssue["Engine issues"] -->|"tool call"| flights["Flights line"] + flights -->|"oldest, under cap"| running["running calls"] + running --> answer["answer lands"] + answer -->|"run.resume"| engineIssue + answer -->|"frees a slot"| flights +``` + +The declarations, with `// ...` standing for existing members that don't change. The `///` lines are the code's own docs; Engine docs follow the wording rule in Constraints. + +```rust +// crates/promptforge-internal/engine/src/execute/run/effect.rs +/// ... the existing doc, plus: Ids order by issue within a run: an effect +/// issued later has a greater id. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct EffectId(pub(crate) u64); + +// crates/promptforge-internal/engine/src/execute/run.rs +pub enum Step { + Pending { + effects: Vec<(EffectId, Provenance, Effect)>, + events: Vec, + /// The tool-call effects this step orphaned, in effect-id order. + /// Their answers will be discarded, so the caller may drop each one + /// it still holds; it still owes each one answer. Empty once the + /// run is decided, when the caller may drop everything it holds. + withdrawn: Vec, + }, + // ... +} + +// crates/promptforge-internal/engine/src/execute/scheduler.rs, new `Scheduler` field +/// Tool-call effects orphaned since the last step, drained into `Step::Pending`. +withdrawn: Vec, + +// crates/harness-internal/runner/src/performers.rs +/// The tool calls a run has in flight at once unless its performer says +/// otherwise. +const DEFAULT_MAX_IN_FLIGHT: NonZeroUsize = NonZeroUsize::MIN.saturating_add(9); + +pub trait ToolPerformer: Send + Sync { + // ... call, survives_stop + /// The most tool calls the Harness runs at once across the run; calls + /// beyond it wait and start oldest first. The default is 10. + fn max_in_flight(&self) -> NonZeroUsize { + DEFAULT_MAX_IN_FLIGHT + } +} + +// crates/harness-internal/runner/src/effect_loop-flight.rs +/// Which held effects an abort reaches. +#[derive(Clone, Copy, Debug)] +pub(super) enum Reach<'a> { + // ... All, AllButSurvivors + /// The listed effects, waiting or in flight: the tool calls a step + /// withdrew, in effect order. + Withdrawn(&'a [EffectId]), +} + +struct InFlight { + // ... provenance, abort, survives_stop + /// Whether the effect is a tool call, which counts toward the cap. + tool_call: bool, +} + +/// A tool call waiting for a slot: its performer future, not yet polled. +struct Waiting { + provenance: Provenance, + survives_stop: bool, + answer: BoxFuture, +} + +pub(super) struct Flights { + // ... futures, effects + /// Tool calls waiting for a slot; the lowest id starts first. + waiting: BTreeMap, + /// The most tool calls in flight at once. + cap: NonZeroUsize, +} + +impl Flights { + pub(super) fn new(cap: NonZeroUsize) -> Self; + // `start` loses its `survives_stop` parameter, which only tool calls set. + /// Puts a chat or timer effect's future in flight. It never waits and + /// never counts toward the cap. + pub(super) fn start( + &mut self, + id: EffectId, + provenance: Provenance, + answer: BoxFuture, + ); + /// Queues a tool call's future, then starts waiting calls, lowest id + /// first, while fewer than `cap` tool calls are in flight. + pub(super) fn queue( + &mut self, + id: EffectId, + provenance: Provenance, + survives_stop: bool, + answer: BoxFuture, + ); + // `abort(reach)` keeps its signature and also takes waiting calls. +} +``` + +### Step 1: the Engine reports withdrawn calls + +- In [crates/promptforge-internal/engine/src/execute/run/effect.rs](crates/promptforge-internal/engine/src/execute/run/effect.rs): `EffectId` (line 43) adds `PartialOrd, Ord` to its derives, and its doc (lines 38-42) gains the sentence that ids order by issue within a run. `abort_subtree`'s `sort_unstable_by_key(|effect| effect.get())` (`chain.rs` line 256) can become `sort_unstable()`. +- In [crates/promptforge-internal/engine/src/execute/scheduler.rs](crates/promptforge-internal/engine/src/execute/scheduler.rs): `Scheduler` gains `withdrawn` beside `orphaned` (line 220), set empty in `new` (lines 255-270). +- In [crates/promptforge-internal/engine/src/execute/scheduler/chain.rs](crates/promptforge-internal/engine/src/execute/scheduler/chain.rs), `abort_effect` (lines 296-305): inside its `Continuation::ToolCall` arm, push the id onto `self.withdrawn`, and add a sentence to its doc. Every path that ends a chain with tool calls out reaches this arm: `tasks.cancel` (`cancel_task`, `waits.rs` line 311), an owner ending with live tasks (`abandon_owned_tasks`, `task_end.rs` line 153), and an owner aborted (`abort_subtree`). +- In [crates/promptforge-internal/engine/src/execute/scheduler/drive.rs](crates/promptforge-internal/engine/src/execute/scheduler/drive.rs): + - `step` takes the buffer beside `effects` (line 56), sorts it, and returns it as `Step::Pending.withdrawn` (line 82). An id is pushed at most once, because `abort_effect` pushes only when it removes a pending entry. + - `end` (lines 172-186) clears the buffer after `teardown`. A run's end abandons its live tasks through `settle_all_tasks`, which reaches `abort_effect` and fills the buffer, while `teardown` orphans the remaining pending effects straight from the table (lines 218-219), which doesn't. Clearing gives the field's simple promise: a decided run's steps list none, because the caller drops a decided run's effects whole. +- In [crates/promptforge-internal/engine/src/execute/run.rs](crates/promptforge-internal/engine/src/execute/run.rs): `Step::Pending` (lines 34-48) gains `withdrawn`, and the variant's doc gains a sentence on it. +- Every exhaustive `Step::Pending` pattern gains `withdrawn` or `..`; the compiler finds them. At baseline they are: + - `crates/harness-internal/runner/src/effect_loop.rs` line 164; + - `engine/src/test_support.rs` line 104 and `engine/src/test_support/tokio_driver.rs` line 200; + - `engine/src/execute/tests/serial_driver.rs` line 199, `execute/tests/parallel_tool_calls.rs` line 21, `execute/tests/run_termination.rs` line 49, and `execute/run/tests.rs` line 286 (487 lines; edit in place); + - `crates/promptforge/tests/suite/event.rs` line 58 and `tests/suite/ids.rs` line 101; + - `crates/promptforge/examples/greeter.rs` line 107, which `tests/suite/greeter.rs` compiles through `#[path]`. + + Every other match already uses `..`. In this step the Harness ignores `withdrawn`, so behavior is unchanged. +- `crates/promptforge/public-api.txt` is regenerated with `cargo +nightly-2026-09-05 xtask api --bless`, and the diff is reviewed. It should show only `EffectId`'s derives line (line 214) gaining `Ord` and `PartialOrd`, and a new `Step::Pending::withdrawn` line beside lines 736-737. + +### Step 2: the Harness caps, orders, and drops + +- In [crates/harness-internal/runner/src/performers.rs](crates/harness-internal/runner/src/performers.rs): `ToolPerformer` (lines 76-98) gains `max_in_flight()`, whose default body returns the private constant `DEFAULT_MAX_IN_FLIGHT`, `NonZeroUsize::MIN.saturating_add(9)`, so no `unwrap` is needed. No implementor changes. The real performer, `HostRunContext` (`host-run.rs` line 169), keeps the default, and test tools override it to set a small cap. The `harness` facade doesn't re-export `ToolPerformer`, so no public listing changes. +- In [crates/harness-internal/runner/src/effect_loop-flight.rs](crates/harness-internal/runner/src/effect_loop-flight.rs): + - `Reach` (lines 30-37) gains `Withdrawn(&'a [EffectId])`, with a lifetime so it stays `Copy`. A withdrawal is a third reach beside a cancel and a stop, so it reuses `abort` and `drop_in_flight` instead of adding a method. + - `InFlight` gains `tool_call`. The body of `start` (the span, `catch_unwind`, and `Abortable` wrapping) moves into a private helper with the `tool_call` flag. `start` calls it for chat and timer effects, and the fill helper calls it for tool calls. `start` drops its `survives_stop` parameter, which is always `false` for chat and timer effects. + - `Flights` gains `waiting` and `cap`, and `new` takes the cap. + - **`queue`** inserts the call into `waiting`, then fills. + - **Fill**, a private helper: while fewer than `cap` tool calls are in flight and a call waits, it starts the waiting call with the lowest id (`BTreeMap::pop_first`). Counting tool calls by scanning `effects` is fine, since the map holds at most the cap plus the run's chat and timer effects. + - **`poll_landed`** fills after removing a landed tool call, the panic path included, so a freed slot fills at once. The late end of an aborted future frees nothing, because its slot was filled when it was aborted. + - **`abort(reach)`** takes waiting calls too, by the same reach: all of them for `Reach::All`, those that do not survive stops for `Reach::AllButSurvivors`, and the listed ones for `Reach::Withdrawn`. Inside the existing filter closure, a `Withdrawn` list is searched with `binary_search`, since the Engine sorts it. + - It drops the waiting calls' unpolled futures, aborts the reached calls in flight, and returns both merged in effect order. + - It fills once, after taking everything, so it never starts a call the same reach is about to take. That fill starts a waiting survivor once a stop has freed a slot, or the next call after a withdrawal. + - Its `chosen.sort_by_key(|id| id.get())` (line 162) can become `sort_unstable()`. + - **`len` and `is_empty`** count waiting calls too, since each still owes an answer. `act_on_stop` compares `len` before and after a stop, so its check stays correct with no change. `DriveError::Stalled` can't fire while calls only wait, because fill leaves a call waiting only while `cap` tool calls are in flight. + - The module doc gains the cap, the waiting line, and the three reaches. Most of the prose goes here. +- In [crates/harness-internal/runner/src/effect_loop.rs](crates/harness-internal/runner/src/effect_loop.rs), about 11 lines net against a budget of 13: + - `drive` reads the cap into a local, `let cap = performers.tool.max_in_flight();`, before building the `Driver`, because the struct literal moves `performers` before it builds `flights`. Then `flights: Flights::new(cap)`. + - `perform`'s `ToolCall` arm sends its future through `self.flights.queue(...)` and returns `None`. The match's tuple becomes just the future, and chat and timer effects call `start` without the flag. Its doc says a tool call may wait. + - The `Step::Pending` arm binds `withdrawn`. After the loop over the step's effects, when `withdrawn` is not empty, it calls `self.drop_in_flight(Reach::Withdrawn(&withdrawn))` and sets `answered`, so the loop steps again instead of awaiting. A one-line comment states the ordering constraint. + - The order matters because an effect can be issued and withdrawn in the same step. + - `drop_in_flight` answers each `Dropped`, then tears down the aborted futures through `answer_landed`. That gives a withdrawal the guarantee it documents for a cancel and a stop, so whatever a dropped performer held is gone before the run steps on. + - A call that the stop check before the effects loop already dropped is no longer held, so the reach skips it. + - The decided branch needs no change, because `abort(Reach::All)` now drains waiting calls too. A step the Engine decided lists no withdrawn ids. A step the branch takes only because the cancel flag fired may list some, and `Reach::All` reaches them with every other held call. + - The module doc (lines 20-35) gains one sentence: tool calls beyond the performer's cap wait in the flights, oldest first, and a cancel, a stop, or a withdrawal reaches them as well. The `drop_in_flight` doc and the `flights` field doc on `Driver` (lines 134-137) mention waiting calls. +- In [crates/harness-internal/runner/src/harness-control.rs](crates/harness-internal/runner/src/harness-control.rs), docs only. The `harness` facade re-exports `RunControl`, so Hosts read these: + - `stop_round` (lines 38-51): a stop also drops the tool calls waiting for a slot, unstarted, except those that survive stops, which start once a slot frees. + - `cancel` (lines 56-61): every waiting call is answered `Dropped` too, unstarted. +- In [crates/harness-internal/runner/src/harness.rs](crates/harness-internal/runner/src/harness.rs), docs only: `Harness::new` (lines 126-132) says a run has at most 10 tool calls in flight, and the rest wait and start oldest first. Hosts see no other trace of the cap. +- Unchanged: the `Harness`, `RunRequest`, and `RunControl` API, `Performers`, `drive_run`, the recorder, the Workshop, `Effect::ToolCall` and its record, the Engine's batch code and `models.loop`, and the `AssistantToolCalls` and `ToolResult` events. + + + + +## Testing Plan + +The Engine tests step a run by hand and read each step's `withdrawn` directly. The Harness tests drive real runs with gated or held tools and observe what started. Through `drive_run`, the test tool overrides `max_in_flight` to set a small cap. Through `Harness::run`, the tests get the real performer's default of 10. `EffectId` has no public constructor, so `Flights` is tested through those runs, not in isolation. + +Every new behavior has a test that fails without it. A missing drop usually shows up as a hang, so every Harness test runs under a timeout: `run_beside`'s 10-second `PATIENCE` (`tests/it/harness.rs` lines 43 and 91-103), or `tokio::time::timeout` around `drive_run` as `tests/it/effect_loop.rs` line 271 does. + +- Integration, Step 1, added to `crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls.rs` (296 lines) beside the cancel test they mirror, `cancelling_a_task_parked_on_a_batch_orphans_every_call` (line 242). Its helpers `calls_reply`, `tool_text`, and `only_effect` are private to that file, which is why the tests go there and not in a new module. + - The three tests share a new `const` holding that test's Main and Child prompt; the existing test keeps its inline copy. + - A new helper returns a step's `withdrawn` beside its effects, since `pending` drops it. + - The module doc gains a clause on the withdrawn list. + - Each test steps the run with `run.step()`. + - `cancelling_a_task_withdraws_its_unanswered_tool_calls`: + - The step that cancels lists both tool-call ids in `withdrawn`, in effect-id order. Every earlier step lists none. + - Answering both `Dropped`, then Main's `linger` round, ends the run `RunResult::Ok("cancelled")`. + - `a_call_issued_and_withdrawn_in_one_step_is_listed_twice`: + - The test resumes the child's round with two calls and then Main's `wait` round before stepping, so the next step runs the child first and Main second. + - That one step lists both calls in `effects` and again in `withdrawn`. This is the contract the Driver's issue-then-withdraw order relies on. + - Answering everything outstanding ends the run `RunResult::Ok("cancelled")`. + - `a_decided_run_lists_no_withdrawn_calls`: + - The test calls `run.cancel()` while the child's two calls are out. + - The next step reports `run.decided()` and lists nothing in `withdrawn`, though the run's end abandoned the child through `abort_effect`. Without the clear in `end`, it would list both. + - Answering everything outstanding `Dropped` reaches `Done` with `RunResult::Cancelled`. +- End-to-end, Step 2, through `drive_run`, in a new `crates/harness-internal/runner/tests/it/effect_loop-cap.rs`. Declare it in `tests/it/effect_loop.rs` beside its siblings as `#[path = "effect_loop-cap.rs"] mod cap;` (lines 28-33), not in `main.rs`. A run with a model round builds its own prompt, model, and tool catalog, as `effect_loop-broker.rs` does (lines 40-58), with the tool offered to the model. Shared fixtures come from `support.rs` and the parent module (`begun_log`, `answers`). + - New helpers go in this file, because `support.rs` is at 486 lines: + - One test tool, built on the pattern of `GatedTool` (`support.rs` lines 372-400), with the cap override, a gate, a log of the calls it started in start order, and a teardown counter. The `Held` fixture can't serve, because its `hold` is private to `scripted.rs`. + - A scripted broker that answers each round by its last message, and can hold a round until a condition is met. + - Before asserting that a call has not started, yield a few times, as `harness-stop.rs` does with 16 `yield_now` calls, so the check isn't vacuous. + - `the_cap_holds_extra_calls_until_one_lands`: + - A cap of 2, and one round of three gated calls. + - Wait until 2 have started, and check that the third has not. + - Add one permit, then check that the third starts. Add the rest, and the run completes. + - `waiting_calls_start_oldest_first`: + - A cap of 1. Chain A's round asks for 3 calls. Then chain B's round, which the broker answers only after a1 has started, asks for 1. + - Wait until B's call is recorded as an effect before opening the gate. The loop queues each effect right after recording it, with no await in between, so on `#[tokio::test]`'s current-thread runtime a recorded call is already queued. + - Open the gate for all four. The tool records start order a1, a2, a3, b1. A wave order or a newest-first order would start b1 before a2. + - `a_call_withdrawn_in_the_step_that_issued_it_never_starts`: + - A cap of 2, and the Engine tests' Main and Child prompt. + - The broker answers the child's round with two gated calls at once. It holds Main's `wait` round until the child's round has completed, then wakes it. Both answers then land in the same `answer_landed` drain, the child's first, so one step issues the two calls and withdraws them. + - Expect no call started, two `Dropped` answers, and the run to complete. If the Driver withdrew before queuing, the reach would find nothing, both calls would start and hold forever, and the timeout would fail the test. + - `cancelling_a_task_drops_its_waiting_and_running_calls`: + - A cap of 1, and a child task whose round asks for two held calls. + - The main chain cancels the child once one of them has started. The trigger is a main model round that the test's broker answers only after that call has started. + - Expect exactly one start, one teardown, both calls answered `Dropped`, and the run to complete. + - `a_cancel_drops_waiting_calls_without_starting_them`: + - A cap of 1, and one round of two held calls. + - Once one has started, fire the run's `CancelHandle`. + - Expect `RunOutcome::Cancelled`, exactly one start, and two `Dropped` answers. A `Reach::All` that missed the waiting call would let fill start it after the cancel, for two starts. +- End-to-end, Step 2, through `Harness::run`, in a new `crates/harness-internal/runner/tests/it/harness-stop-cap.rs`. Declare it in `tests/it/harness.rs` beside `stop` and `stop_timing` (lines 31-36), because `harness-stop.rs` is at 415 lines and two tests with their helpers would pass 500. The file has its own prompt and broker, shaped like `LOOPS_ONCE` and `calls_hold_twice` in `harness-stop.rs`, which are private there. + - `a_stop_drops_waiting_calls_without_starting_them`: + - The default cap of 10, and one round of 11 `harness/hold` calls. + - Wait until 10 have started, then call `stop_round()`. + - Expect `final_text` `"false:cancelled"`, `held.started() == 10`, `held.dropped() == 10`, and 11 `Dropped` answers in `answers_to(&records, "ToolCall")`. + - `a_stop_starts_a_waiting_call_that_survives_stops`: + - The default cap of 10, and one round of 10 `harness/hold` calls followed by one call to the fixture ask tool. The prompt offers both with `tools.always_offer`, and the ask tool's wire name is expected to be `user-input_ask`. The Host is `with_asker(hold_host(..))` with an `Operator`. + - Wait until 10 holds have started, and check that no question has been asked. + - Call `stop_round()`. The question is asked once the stop frees the slots. Answer it. + - Expect the 10 holds dropped, the question's answer recorded as a non-`Dropped` `ToolCall` answer, and `final_text` `"false:cancelled"`, since the batch answers with its first failure in call order. +- Regression: + - The existing tests pass unchanged, apart from the mechanical `Step::Pending` updates. That includes `parallel_tool_calls.rs`, whose existing tests stay as they are beside the new ones, `batch_turn.rs`, `tool_call_access.rs`, and `models_loop.rs`. + - `harness-stop.rs` passes, including the two-calls-in-flight test, under the default cap of 10. The default applies to every test performer too. At baseline no Harness, Workshop, or Plugin test uses `tasks.fanout` or builds a call list in a loop, and the largest round in `harness-stop.rs` asks for two calls. + - The serial and tokio test drivers in `engine/src/test_support*` keep performing every effect, ignore `withdrawn`, and have no cap. +- Exit criteria, run at the end of each step: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` + - The Workshop crates, with the gateway sidecar staged: + - Stage it: `cargo build --locked -p gateway --no-default-features`, then `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe`. + - Run `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` and `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`. + - Always run `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc` afterward, even when a command failed. + - `cargo fmt --all --check` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api`, then `cargo doc -p promptforge --no-deps` with the same flags + - `cargo +nightly-2026-09-05 xtask api --check`, after Step 1's reviewed `--bless` + - `cargo test -p build-xtask`, which includes the 500-line ceiling and `no_doctests` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + + + + +## Decision Record + +The design moved from an Engine queue, to an Engine-ordered line with the cap in both halves, to a cap that lives only in the Harness with waves computed from an Engine index, and finally to oldest-call-first with no index. The user set each turn of that shape. A review against the code showed that the wave rule could starve an earlier batch's leftovers. The user then chose the simplest order that cannot starve. + +- Decisions: + - The Engine hands every call to the Harness, and the Harness enforces the cap. User: "maybe the Engine should not maintain its own queue and it should instead just give the entire batch to the harness, and the harness is told the max tool limit". + - The cap lives only in the Harness. User: "I wish we didn't need the cap in both the engine and the harness". + - Waiting calls start oldest first, by effect id. User: "What if we just complete the whole batch first? Like always prioritize completing a batch because that lets us give, get a result" and "If we get a batch, then we want to finish that batch." + - A batch's chain resumes only when every call in it has an answer, because the model's next request must answer every call it asked for. So finishing the oldest batch first gets a chain its result soonest. + - It cannot starve: every new call goes behind every call already waiting. + - It orders by when a batch was issued, not by how old its chain is. The user worried about "a chain that just runs forever". Such a chain has at most one batch out at a time, and each new batch goes to the back, so it cannot hog the slots. + - Waiting is rare. User: "how often are we going to get 10 tool calls at once? ... 10 concurrent tool calls is quite a lot." Waiting mostly takes a fanout, so the simplest order that cannot starve wins over a cleverer one. + - It keeps an earlier wish: "the harness is dumb it just forms Vec into a heap and picks the lowest number". The effect id is that number, so no new field is needed. + - A withdrawn list reports orphaned tool calls. User: "this seems alright I guess." + - Public format changes are acceptable. User: "public format changing is not a problem at all. no one depends on this yet." + - Defaults chosen in planning, open to override: + - The cap comes from `ToolPerformer::max_in_flight()`, a default method returning 10: + - The tool performer is what runs the calls, so it knows its capacity. + - A default method changes no implementor, no `Performers` literal, and no `drive_run` call. + - Tests override it to set a small cap. + - The default is 10, separate from `max_concurrency`. Harnesses cluster at 8 to 10 (Claude Code 10, Mastra 10, CrewAI 8), and every harness surveyed keeps tool-call and subagent limits separate. + - `EffectId` derives `PartialOrd` and `Ord`, which serve the waiting line's key, the withdrawn sort, and the existing sorts in `abort_subtree` and `Flights::abort`. Its doc states the order as a guarantee: ids order by issue within a run. + - `withdrawn` is a field on `Step::Pending`, not a `Run` query. It's per-step data, so it belongs in the per-step value. A query valid only until the next step would tie callers to calling it in the right order, which is temporal coupling. + - A decided run's steps list no withdrawn calls: `end` clears the buffer. The run's end reaches `abort_effect` for its live tasks but orphans the rest through `teardown`, so a list there would be partial, and the caller drops a decided run's effects whole anyway. + - Waiting calls live inside the Harness's existing `Flights` as unpolled performer futures. Stop, Cancel, and landings already pass through `Flights`, so no new file or `Driver` state is needed. + - A withdrawal is a third `Reach`, so `abort` and `drop_in_flight` serve it. It gets the same merge, sort, fill, and teardown as a cancel or a stop, and `Flights` gains one method, `queue`, not two. + - `Flights::start` loses its `survives_stop` parameter, since only tool calls set it and tool calls now go through `queue`. + - A withdrawn call that is already running is dropped too, matching what Cancel does to a run, and freeing its slot. + - Calls that survive stops count toward the cap like any other tool call. A waiting one keeps waiting through a Stop and starts once a slot frees. The trade-off: an operator question can wait behind slow calls, and the operator sees it only once it starts. +- Rejected alternatives: + - Waves: an `index` on every `Effect::ToolCall` (its batch slot), and a Harness wave of `base + index / cap`, with a new batch's base set to the lowest wave the Harness still held. Reason: the wave a newcomer joins never moves forward while newcomers keep joining it. Two chains issuing batches back to back, or one long call such as an open operator question, can leave an earlier batch's leftovers waiting indefinitely. It also adds an Engine field for a rare case. That was the plan's previous design. + - Waves that join the next round (base one past the wave started most recently), or a per-task floor on the base. Reason: fair, but more machinery than a case that rarely occurs needs. + - Smallest batch first. Reason: a large batch waits as long as smaller ones keep arriving, and the Harness would have to count each batch before placing it. + - One call per batch, interleaved. Reason: it needs a placement rule for each newcomer's first call, with the same starvation risk as waves, and a batch only helps its chain when complete, so every batch finishes late. + - Oldest chain first. Reason: a chain that runs forever would always win. + - The Engine computing a ready-made sort number from the cap, with `RunLimits::max_tool_calls`, `Run::limits()`, a `sort` field on each continuation, and a scan for the lowest open number. Reason: both halves would need the cap and must agree on it. + - A separate `WaitingLine` type in its own file, with new `Driver` fields. Reason: `Flights` already owns every in-flight effect and its drop paths. + - A `Flights::withdraw(ids)` method answered by its own loop in the `Driver`, this plan's earlier shape. Reason: it repeated `abort`'s take, merge, sort, and fill, and its loop skipped the teardown that `drop_in_flight` gives a cancel and a stop. + - The cap as a `drive_run` parameter or a `Performers` field. Reason: both change every call site or literal in the runner's tests, where a default trait method changes none. + - `Harness::with_limits`, or a `limits` field on `RunRequest`. Reason: the mechanism and its tests don't need it, and `RunRequest` is built by struct literal in about 19 places. See Deferred. + - A queue in the Engine. Reason: Stop can't reach calls the Harness never saw, and it needs slot bookkeeping across parking, resuming, and aborting. + - Sharing `max_concurrency`, with a chain giving its slot back while it waits. Reason: it couples task and call limits and reworks admission. No surveyed harness does it. + - Raising the limit while a chain waits. Reason: it makes the cap soft and lets limits leak through `tasks.concurrency`. + - Per-batch windows held back in the Engine. Reason: more Engine logic for what the Harness's line already does. + - A shared `Arc` cancellation flag inside `Effect`. Reason: effects stop being plain recorded data, the withdrawal becomes invisible to logs and tests, and a bool can't stop a running call. + - A `BinaryHeap` for the waiting line. Reason: it can't remove a withdrawn call from the middle. + - Letting calls that survive stops bypass the cap. Reason: `survives_stop` is a Stop flag and should not gain a second meaning. +- Assumptions, risks, and notes: + - Effect ids follow issue order (`next_effect`, `scheduler.rs` line 230, advanced in `issue`, lines 305-307), and `dispatch_tool_calls` issues a batch's calls one after another, so lowest id first finishes the oldest batch first. + - Performers do nothing until polled. `perform` wraps each tool call in an `async move` block that calls the performer inside it (`effect_loop.rs` lines 340-353), so a waiting future has started nothing. `survives_stop` is read when the effect is performed (line 347), before the call waits. + - The Harness answers each landed call before it steps, and drains any already-landed peers first (`await_answer` and `answer_landed`), and fill runs inside `poll_landed`, so freed slots refill at once. + - A big batch holds later batches behind it: with a cap of 10, a 40-call batch holds a later 1-call batch for four rounds. This is accepted, because waiting needs more than the cap of calls at once, and they come from the prompt's own fanout or rounds. + - No tool-call "started" event exists. The Engine reports `AssistantToolCalls` before dispatch and `ToolResult` on landing, so a waiting call is simply shown as running until its result arrives. + - The order in which the Harness starts calls changes landing order only, which the Engine already records as input. Replays are unaffected. + +### Deferred and Out of Scope + +- A Host setting for the cap. `HostRunContext` would answer `max_in_flight` from a Host-supplied value. Revisit when a Host needs a non-default cap. +- Changing the cap mid-run from the Host. This is Harness-only: re-read `max_in_flight` each step, or add a `RunControl` method beside `stop_round`. Revisit if operators ask. +- Changing the cap mid-run from a prompt. It needs a way for the Engine to tell the Harness the new value. Revisit if a prompt needs it. +- A fill order that lets a later batch go ahead of a big batch's leftovers. Revisit if a real prompt shows one batch holding up others. +- A stopped round keeps finished results and answers the rest with synthetic "cancelled" results, as Claude Code, Codex, Gemini CLI, and Pydantic AI do, instead of discarding the round. Model APIs such as Anthropic's and OpenAI's reject a request that leaves any requested call without a result, which is why a round today is either appended whole or not at all. Revisit at the next `models.loop` failure-semantics change. +- Tool effect labels on `ToolDescriptor` (MCP-style read-only, destructive, idempotent, and open-world, with the MCP Plugin passing server annotations through) for a future Ask mode, with calls not marked read-only running alone. Revisit when Ask mode is planned. +- Harness-reported started events, so the transcript can show waiting apart from running. Revisit if operators ask. +- Withdrawing orphaned model rounds to save tokens. Revisit if a cancelled task's model spend matters. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build --locked -p gateway`. + - `crates/gateway/app` is the workspace's only default member, so plain `cargo build` builds the same thing. The desktop app builds with `cargo workshop`, an alias for `run -p build-workshop --` in `.cargo/config.toml`, which also sets the Windows `rust-lld` linker with the static CRT. + - The `gateway-config-ui` and `workshop-server` build scripts bundle their UIs with esbuild through `build-ui`, so workspace-wide commands need Node.js plus one `npm ci --prefix crates/workshop` and one `npm ci --prefix crates/gateway/config-ui/ui` per checkout. Both installs are present on this machine, along with Node.js 24.19 (CI uses 22), cargo-nextest 0.9.128, stable Rust 1.99.0, `nightly-2026-09-05`, mdBook 0.4.44, cargo-deny 0.20.2, cargo-audit 0.22.2, and cargo-hakari 0.9.38. + - Workshop clippy and tests need the gateway sidecar staged for Tauri's `externalBin`: `cargo build --locked -p gateway --no-default-features`, then `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe`, and `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc` afterward. The staging directory `crates/workshop/desktop/binaries/` is gitignored and empty now, so nothing is staged. +- Focused test command pattern: `cargo nextest run --locked -p --all-features `, such as `-p promptforge-engine parallel_tool_calls` or `-p harness-runner effect_loop`. Drop `--all-features` for `workshop`, `workshop-server`, and `workshop-server-api`, which CI never builds with all features. +- Component test command pattern: `cargo nextest run --locked -p --all-features`, with the same three exceptions. `cargo test -p build-xtask` runs the boundary and structural checks, and `cargo xtask tidy` prints the same report. +- Full-suite test command: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` with the sidecar staged. The first run includes `build-xtask`. + - CI also runs `cargo nextest run --locked -p workshop-workspace --all-features`, `cargo nextest run --locked -p workshop-server --features headless`, and five launch-lease tests by name: one in `gateway-api-discovery` and four through `cargo test --locked -p gateway --no-default-features --features test-fixtures --test it `. + - The wording and rulebook scan: `node --test crates/workshop/ui/test/docs-claims.mjs`. It needs no `npm ci` and runs from any directory. + - The UI suites: `npm test --workspaces --if-present` in `crates/workshop` and `npm test` in `crates/gateway/config-ui/ui`. + - This plan's gates are the commands in its Testing Plan's exit criteria. Its Workshop runs need the sidecar staged, as CI's `check-workshop` job does. +- Linter command: + - `cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` and `cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, both with `CARGO_BUILD_WARNINGS=deny` (in PowerShell, `$env:CARGO_BUILD_WARNINGS='deny'`). + - Never run a standalone `cargo check --workspace` beside them. The one exception is `cargo check -p gateway --no-default-features`, the headless build shape clippy does not cover. + - UI type checks: `npm run typecheck --workspaces --if-present` in `crates/workshop` and `npm run typecheck` in `crates/gateway/config-ui/ui`. + - Supply chain, run in CI: `cargo deny check`, `cargo audit`, `cargo hakari verify`, and a `cargo tree --locked -p gateway -e normal -i ring` check that `ring` stays out of the gateway's normal closure. + - The pre-push hook runs the wording scan, the headless check, the non-Workshop clippy run, and `cargo deny` when it is installed. +- Formatter check command: `cargo fmt --all --check`, with `style_edition = "2024"` in `rustfmt.toml`. The pre-commit hook runs it, and `core.hooksPath` is set to `.githooks`. +- Docs command: + - Each with `RUSTDOCFLAGS="-D warnings"` (in PowerShell, `$env:RUSTDOCFLAGS='-D warnings'`): `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api`, `cargo doc -p promptforge --no-deps`, `cargo doc -p harness --no-deps`, `cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items`, and `cargo doc --locked --no-deps -p workshop-server --document-private-items`. The facade docs build with default features on purpose. + - The facade surface check: `cargo +nightly-2026-09-05 xtask api --check` against `crates/promptforge/public-api.txt`, and `--bless` regenerates the listing. The nightly is pinned in `crates/build-xtask/src/api/toolchain.rs`, and its fixtures run with `cargo +nightly-2026-09-05 nextest run --locked -p build-xtask --run-ignored only`. + - Public listings: only `crates/promptforge/public-api.txt` exists. The `harness` facade (`crates/harness/src/lib.rs`) re-exports `BoxFuture`, `InferenceBroker`, `Timer`, `RunControl`, and `Harness`, not `ToolPerformer`, so `RunControl`'s and `Harness::new`'s docs are where Hosts learn about the cap and waiting calls. + - The documentation site: `cargo xtask site [--books-only]`, which needs `PROMPTFORGE_DOCS` naming a `promptforge-docs` checkout and an mdBook binary (`MDBOOK`, default `mdbook`). +- Test placement and naming conventions: + - Unit tests mostly sit in a sibling file wired with `#[cfg(test)] #[path = "-tests.rs"] mod tests;`, as in 176 files, such as `crates/promptforge-internal/parser/src/build-tests.rs`. A module whose tests reach three files keeps them in a `tests/` subdirectory beside it, one file per topic. There are 18 of these, including `crates/promptforge-internal/engine/src/execute/tests/` and `crates/promptforge-internal/lua/src/protocol/tests/`. Inline `#[cfg(test)] mod tests { }` blocks remain in 141 files. + - Integration tests compile as one binary per crate: from `tests/it/main.rs` in 18 crates, and from `tests/suite/main.rs` in `harness` and `promptforge`. Each topic is one module file, and shared helpers go in `support.rs`. + - In `harness-runner`, `tests/it/main.rs` declares only top-level modules. The `effect_loop-*.rs`, `harness-*.rs`, and `prepare-*.rs` siblings are declared from `effect_loop.rs`, `harness.rs`, and `prepare.rs` with `#[path]` (so `harness-stop.rs` is the module `harness::stop`). A child module can use its parent's private helpers, such as `begun_log` and `answers` in `effect_loop.rs` and `run_beside`, `until`, and `answers_to` in `harness.rs`. Top-level `tests/` trees are exempt from the flat-directory rule. + - Private fixtures: the helpers in the Engine's `execute/tests/parallel_tool_calls.rs` are private to it. `Held::hold` (`tests/it/scripted.rs` line 164) is private, while `Held`'s `started` and `dropped` getters, `hold_host`, `with_asker`, `Operator`, and `ScriptedBroker` are `pub(crate)`. + - `harness-runner` tests run on `#[tokio::test]`'s current-thread runtime, and none sets a flavor. The `Held` fixture (`tests/it/scripted.rs` lines 145-181) counts a future as started on its first poll and as dropped on teardown, so an unstarted waiting call counts as neither. + - A few crates keep standalone test files directly in `tests/`: `gateway-stt-engine` (three files), `gateway-stt-backend-whisper` (four `native_*` files), `gateway-cloud-providers`, and `build-workshop`. There are two benches, both with `harness = false`: `benches/models_loop.rs` in `promptforge-engine` and `benches/surface.rs` in `promptforge-lua`. + - Test names are snake_case sentences stating the behavior, such as `a_rounds_catalog_calls_are_issued_in_one_step_and_append_in_call_order`. + - Doctests are banned. `build-xtask`'s `no_doctests` check fails any doc comment holding a code block rustdoc would compile, so doc examples use non-Rust fences. + - Nightly-only `build-xtask` fixtures and the native speech and CUDA tests are `#[ignore]`d. `.config/nextest.toml` puts `gateway-stt` and `gateway-stt-backend-whisper` in a `heavy` test group (eight threads, four per test) and gives the gateway's realtime STT noise and capture tests a longer timeout. +- Directory map: + - `crates/`: all 52 Cargo packages. The top level holds the public and shared layer: the `promptforge` facade, `promptforge-plugin`, `harness`, `harness-gateway-client`, `gateway-api-types`, `gateway-api-discovery`, the `plugin-*` crates (`plugin-web`, `plugin-user-input`, `plugin-mcp`), `shared-error-source`, `shared-loopback`, the `build-*` tooling (`build-ceiling`, `build-llama-cuda`, `build-ui`, `build-user-guide`, `build-workshop`, `build-xtask`), and `workspace-hack`. + - `crates/promptforge-internal/`: the Engine's private crates `types`, `engine`, `lua`, `parser`, `vfs`, and `model-client`. The scheduler this plan touches is `engine/src/execute/scheduler/`, beside `engine/src/execute/run/`. + - `crates/harness-internal/runner`: the Harness's private per-run runner, including the effect loop (`src/effect_loop.rs`), the performers (`src/performers.rs`), and the package registry (`src/packages.rs`). + - `crates/gateway/`: the gateway's private crates `app`, `cloud-providers`, `config`, `config-ui`, `local`, `logging`, `progress`, `protocol`, `routing`, and `web-search`, plus the `stt/` subsystem (`api`, `engine`, `backend-whisper`, `whisper-ffi`). + - `crates/workshop/`: Workshop's crates `desktop` (package `workshop`), `server`, `server-api`, `agents`, `gateway`, `menu`, `protocol`, `registry`, `run-log`, `status`, `support`, `user-state`, and `workspace`, plus the npm workspace packages `ui`, `look`, and `platform`. + - `crates/shared-ui`: a TypeScript and CSS package the gateway config UI uses, excluded from the Cargo workspace. + - `guide/`: the `books`, `chrome`, and `landing` pieces of the mdBook site. The chapters live in the separate `promptforge-docs` checkout. + - `prompts/`: five example prompt files. `images/`: README art. + - `tools/`: Node scripts for staging the gateway sidecar and a live TTS check, each with a `.test.mjs`. + - `vibe/`: plan records, recent ones at the top level and older ones filed by month (`2026-07` through `2026-10`), plus an ignored `scratch/`. + - `.github/workflows/`: `ci.yml`, whose `ci-green` job is the single gate, plus release, nightly, site, Miri, native-library, and installer smoke workflows. + - `.config/`: `nextest.toml` and `hakari.toml`. `.githooks/`: pre-commit and pre-push. `.cursor/rules/`: the Workshop architecture and SPA rules. Rulebooks: the root `AGENTS.md` plus 21 crate-level `AGENTS.md` files. + - Not tracked: `local/`, `target/`, and `target-msrv/` (ignored), and `cabinet/`, which holds only an empty `_trash/`. The working tree is clean at `bc49d9670` (`Close plan: easy-fixes`). +- Component boundaries, which `cargo test -p build-xtask` enforces across normal, dev, and build dependencies: + - The Engine (`promptforge*`) depends on no gateway, Workshop, Harness, or Plugin crate. Outside crates reach it only through `promptforge` and `promptforge-plugin`, the only two crates that may enter `promptforge-internal/`. + - Inside the Engine: `types` and `vfs` depend on no Engine crate, `model-client` on `types`, `lua` on `types`, `vfs`, and `model-client`, `parser` on `types` and `lua`, and `engine` on all five. The facade depends on all six, and `promptforge-plugin` names only `types` and `vfs`. `crates/promptforge/public-api.txt` pins the facade's surface. + - Inside the Engine's scheduler, withdrawals happen only inside a step: `cancel_task` (`scheduler/waits.rs`), `abandon_owned_tasks` (`scheduler/task_end.rs`), and `abort_subtree` (`scheduler/chain.rs`) run from a chain's step or end. The one path from `resume`, a fatal answer (`drive.rs` line 105), goes through `end` (`drive.rs` line 172), so clearing the buffer in `end` covers it. + - The Harness chain runs `harness-gateway-client` on `harness`, `harness` on `harness-runner`, and `harness-runner` on `promptforge` and `promptforge-plugin`. Outside its family the Harness names only `promptforge`, `promptforge-plugin`, and `workspace-hack`, except that `harness-gateway-client` depends on `plugin-web` to implement its `SearchProvider`. Outside crates reach the Harness only through `harness` and `harness-gateway-client`, and only `harness` may enter `harness-internal/`. + - Plugin crates depend only on `promptforge-plugin`, `shared-*` crates, `workspace-hack`, and outside libraries. Their only dependents are `harness-gateway-client`, `workshop-server`, and `workshop-agents`. + - The gateway (`gateway*`) depends on no Engine, Workshop, Harness, or Plugin crate. Outside crates reach it only through `gateway-api-types` and `gateway-api-discovery`. Inside the family, only `gateway-stt` is visible outside `gateway/stt/`. + - Workshop is a Host. It reaches the gateway only through that public pair, the Engine only through `promptforge` and `promptforge-plugin`, and the Harness only through `harness` and `harness-gateway-client`. Its tiers depend one way: the server on features (`agents`, `workspace`, `user-state`, `run-log`), features on services (`menu`, `gateway`, `status`), and services on vocabulary (`protocol`, `registry`, `support`). The desktop app names only `workshop-server-api`, which names only `workshop-server`. + - `shared-*` crates depend on no product crate. `build-*` crates are tooling and may depend into any container. +- Conventions summary: + - Engine, Harness, Host, and Plugin are capitalized with one meaning each. Engine crates say "the caller" and never mention the Host. `docs-claims.mjs` enforces what a scan can see in every `AGENTS.md`, every `## Invariants` crate doc, and every `.cursor/rules` file. + - 25 crate `lib.rs` files open with a `//!` doc that has a `## Invariants` section. + - Every crate build script (51 of them) calls `build_ceiling::check()`, which fails that crate's build when any `.rs` file under its `src`, `tests`, `benches`, or `examples` is over 500 lines (`LIMIT = 500`), so a 500-line file passes. `cargo test -p build-xtask` checks that every crate wires the call. Splits go into private child modules without widening visibility. + - Sizes of the files this plan touches. Engine, under `crates/promptforge-internal/engine/src/execute/`: `run/effect.rs` 403, `run.rs` 266, `scheduler.rs` 321, `scheduler/chain.rs` 306, `scheduler/drive.rs` 221, `run/tests.rs` 487, `tests/parallel_tool_calls.rs` 296. Harness, under `crates/harness-internal/runner/src/`: `effect_loop.rs` 487, `effect_loop-flight.rs` 181, `performers.rs` 129, `harness.rs` 280, `harness-control.rs` 127. Harness tests, under `crates/harness-internal/runner/tests/it/`: `effect_loop.rs` 376, `harness.rs` 344, `harness-stop.rs` 415, `support.rs` 486. + - Clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, and `allow_attributes` deny, so suppressions use `#[expect(..., reason = "...")]`. `missing_docs`, `missing_debug_implementations`, and `unreachable_pub` warn, which is fatal under the gate. `unsafe_code` is denied outside its owned boundaries, rustdoc's broken and private intra-doc links deny, and `clippy.toml` bans process-global installers outside binary entry points. + - Source directories stay flat until a group reaches three files. Below that, kebab siblings such as `foo-bar.rs` are wired with `#[path]`. + - Tool wire names: the model sees `harness/hold` as `harness_hold` (`tests/it/harness-stop.rs` line 178) and `user-input/ask` as `user-input_ask` (`crates/plugin-user-input/src/lib.rs` line 61). + - Comments explain only non-obvious constraints, and every workaround cites its upstream issue URL. + - Error messages are written for a model to read: concise, naming what was required and what was found. + - Behavior changes ship with tests in the same change. A Cargo feature gates only a real build constraint, never product shape. + - JSON that reaches a recorder round-trips exactly, with sorted keys and finite numbers. + - Workspace dependencies are declared once in the root `Cargo.toml`, with a comment on every non-obvious pin. Every member inherits `workspace-hack`, which `cargo hakari` manages. + - Commits have a one-line summary, a prose body, a bullet list of changes, optional `Design:` lines, and a `Plan: vibe/` trailer. A plan's closing commit is titled `Close plan: ` and holds only the trailer. + + + + +## Execution Instructions + + + +### Step 1: Report the tool calls each step withdraws [completed] + +- Component: `none` +- Tests: Verify runs every command below as written, in this order, from the `promptforge` repository root. Setting `$env:` variables persists for the session, so the API check below clears `CARGO_BUILD_WARNINGS` first. + - `cargo nextest run --locked -p promptforge-engine --all-features parallel_tool_calls` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --locked --no-deps --all-features -p promptforge-engine --document-private-items` + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` + - `cargo build --locked -p gateway --no-default-features` + - `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe` + - `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets` + - `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc`, always, even when an earlier command failed + - `cargo fmt --all --check` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc -p promptforge --no-deps` + - `Remove-Item Env:CARGO_BUILD_WARNINGS -ErrorAction SilentlyContinue; cargo +nightly-2026-09-05 xtask api --check`, clearing the variable first because a leftover `CARGO_BUILD_WARNINGS='deny'` makes nightly cargo's unused-`workspace-hack` manifest warnings fatal + - `cargo test -p build-xtask` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + +- Baseline is commit `bc49d9670`. Line numbers and file sizes below were verified there; search by name if one has moved. The coding sub-agent leaves `vibe/` alone. +- Behavior: each `Step::Pending` lists the tool-call effects that step orphaned while the run goes on, in effect-id order, each id at most once. A decided run's steps list none. Store effects and model rounds are never withdrawn. The Harness ignores the list until Step 2, so this step changes no run's behavior. The Engine stays sans-IO and deterministic. +- Engine, under `crates/promptforge-internal/engine/src/execute/`, with baseline sizes: + - `run/effect.rs` (403 lines): `EffectId` (line 43) adds `PartialOrd, Ord` to its derives. Its doc (lines 38-42) gains: "Ids order by issue within a run: an effect issued later has a greater id." + - `run.rs` (266): `Step::Pending` (lines 34-48) gains `withdrawn: Vec` with the field doc the Technical Design gives ("The tool-call effects this step orphaned, in effect-id order..."), and the variant's doc gains a sentence on it. + - `scheduler.rs` (321): `Scheduler` gains `withdrawn: Vec` beside `orphaned` (line 220), documented "Tool-call effects orphaned since the last step, drained into `Step::Pending`.", and set empty in `new` (lines 255-270). + - `scheduler/chain.rs` (306): `abort_effect` (lines 296-305) pushes the id onto `self.withdrawn` inside its `Continuation::ToolCall` arm only, and its doc gains a sentence saying so. It pushes only when it removes a pending entry, so no id repeats. `tasks.cancel` (`cancel_task`, `waits.rs` line 311), an owner ending with live tasks (`abandon_owned_tasks`, `task_end.rs` line 153), and an aborted owner (`abort_subtree`) all reach this arm. `abort_subtree`'s `sort_unstable_by_key(|effect| effect.get())` (line 256) becomes `sort_unstable()`. + - `scheduler/drive.rs` (221): `step` takes the buffer beside `effects` (line 56), sorts it, and returns it as `Step::Pending.withdrawn` (line 82). `end` (lines 172-186) clears the buffer after `teardown`. The run's end abandons live tasks through `settle_all_tasks`, which reaches `abort_effect` and fills the buffer, while `teardown` orphans the rest straight from the table (lines 218-219); clearing keeps the promise that a decided run lists none. A fatal answer from `resume` (`drive.rs` line 105) goes through `end`, so the clear covers it. +- Exhaustive `Step::Pending` patterns gain `withdrawn` or `..`; the compiler finds them. At baseline: + - `crates/harness-internal/runner/src/effect_loop.rs` line 164 (the Harness ignores the field in this step) + - `engine/src/test_support.rs` line 104 and `engine/src/test_support/tokio_driver.rs` line 200 + - `engine/src/execute/tests/serial_driver.rs` line 199, `execute/tests/parallel_tool_calls.rs` line 21 (the `pending` helper), `execute/tests/run_termination.rs` line 49, and `execute/run/tests.rs` line 286 (487 lines; edit in place) + - `crates/promptforge/tests/suite/event.rs` line 58 and `tests/suite/ids.rs` line 101 + - `crates/promptforge/examples/greeter.rs` line 107, which `tests/suite/greeter.rs` compiles through `#[path]` + - Every other match already uses `..`. +- `crates/promptforge/public-api.txt`: regenerate with `cargo +nightly-2026-09-05 xtask api --bless` and review the diff. It must show only `EffectId`'s derives line (line 214) gaining `Ord` and `PartialOrd`, and a new `Step::Pending::withdrawn` line beside lines 736-737. Any other change is a finding. +- New tests in `crates/promptforge-internal/engine/src/execute/tests/parallel_tool_calls.rs` (296 lines), beside `cancelling_a_task_parked_on_a_batch_orphans_every_call` (line 242). They use that file's private helpers `calls_reply`, `tool_text`, `only_effect`, and `pending`, which is why they go there and not in a new module. + - Scaffolding: + - A new `const` holds the Main and Child prompt of `cancelling_a_task_parked_on_a_batch_orphans_every_call`, shared by the three new tests; the existing test keeps its inline copy. `## Main` runs `local t = tasks.spawn('## Child')`, `models.infer('wait')`, `tasks.cancel(t)`, `models.infer('linger')`, and returns `'cancelled'`. `## Child` builds `messages.new()`, adds `msgs:user('go')`, runs `models.loop(msgs)`, and returns `'child done'`. Build each run as that test does: `parse`, `loop_context(&prompt, echo_tools())`, and `Run::from_state(state)`. + - A new helper returns a step's `withdrawn` beside its effects, since `pending` drops it. + - The module doc gains a clause on the withdrawn list. + - Each test steps the run with `run.step()`. The first step issues Main's infer round (`ReplyOrigin::Infer`) and Child's loop round (`ReplyOrigin::Chat`); answer the loop round with `calls_reply` over `[c1 echo, c2 echo]`. + - `cancelling_a_task_withdraws_its_unanswered_tool_calls`: + - After the step that issues both calls, answer Main's `wait` round. The next step cancels Child. + - The step that cancels lists both tool-call ids in `withdrawn`, in effect-id order. Every earlier step lists none. + - Answering both calls `EffectAnswer::Dropped`, then Main's `linger` round, ends the run `RunResult::Ok("cancelled")`. + - `a_call_issued_and_withdrawn_in_one_step_is_listed_twice`: + - Resume Child's round with the two calls and then Main's `wait` round before stepping, so the next step runs Child first and Main second. + - That one step lists both calls in `effects` and again in `withdrawn`. Step 2's Driver relies on this issue-then-withdraw contract. + - Answering everything outstanding ends the run `RunResult::Ok("cancelled")`. + - `a_decided_run_lists_no_withdrawn_calls`: + - Call `run.cancel()` while Child's two calls are out. + - The next step reports `run.decided()` and lists nothing in `withdrawn`, though the run's end abandoned Child through `abort_effect`. Without the clear in `end`, it would list both. + - Answering everything outstanding `EffectAnswer::Dropped` reaches `Done` with `RunResult::Cancelled`. + - Each new test must fail without the change it pins. +- Regression: every existing test passes. Existing tests change only by the mechanical `Step::Pending` pattern updates, and `parallel_tool_calls.rs`'s existing tests stay as they are. The serial and tokio drivers in `engine/src/test_support*` keep performing every effect and ignore `withdrawn`. +- Rules: + - Engine docs and comments call the Harness "the caller", and name the Harness only to say a responsibility belongs to it, never how the Harness performs it (root `AGENTS.md`). `docs-claims.mjs` catches only lowercase and compound forms, so review checks this. + - Every `.rs` file stays at or under 500 lines; the ceiling fails a file over 500. + - Clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, suppressions use `#[expect(..., reason = "...")]`, and dead code and missing docs are fatal under the gate. Doctests are banned, so doc examples use non-Rust fences. + - Comments explain only non-obvious constraints. +- `Design:` lines for the commit, from the Decision Record: + - `withdrawn` is a field on `Step::Pending`, not a `Run` query: it's per-step data, and a query valid only until the next step would tie callers to calling it in the right order. + - A decided run lists no withdrawn calls: its end reaches `abort_effect` for live tasks but orphans the rest through `teardown`, so a list there would be partial, and the caller drops a decided run's effects whole. + - `EffectId`'s order is a documented guarantee, serving the withdrawn sort, `abort_subtree`, and Step 2's waiting line. +- Commit: one `promptforge` commit, `Report the tool calls each step withdraws`, with a prose body, a bullet list of changes, the `Design:` lines, and the trailer `Plan: vibe/2026-10-11-6-tool-call-cap.md`. Fixes land in this commit. Nothing is pushed. + + + + + +### Step 2: Run at most the cap of tool calls, oldest first + +- Component: `none` +- Tests: Verify runs every command below as written, in this order, from the `promptforge` repository root. Setting `$env:` variables persists for the session, so the API check below clears `CARGO_BUILD_WARNINGS` first. + - `cargo nextest run --locked -p harness-runner --all-features effect_loop::cap:: harness::stop_cap::` + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` + - `cargo build --locked -p gateway --no-default-features` + - `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe` + - `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets` + - `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc`, always, even when an earlier command failed + - `cargo fmt --all --check` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc -p promptforge --no-deps` + - `Remove-Item Env:CARGO_BUILD_WARNINGS -ErrorAction SilentlyContinue; cargo +nightly-2026-09-05 xtask api --check`, clearing the variable first because a leftover `CARGO_BUILD_WARNINGS='deny'` makes nightly cargo's unused-`workspace-hack` manifest warnings fatal + - `cargo test -p build-xtask` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + +- Lands after Step 1, whose `Step::Pending.withdrawn` and `EffectId: Ord` this step uses. Baseline is commit `bc49d9670` plus Step 1; line numbers and sizes below were verified at `bc49d9670`, so search by name if one has moved. The coding sub-agent leaves `vibe/` alone. +- Behavior: + - At most `ToolPerformer::max_in_flight()` tool calls run at once across a run; the default is 10. Model rounds, timers, and store effects never wait and never count toward the cap. + - Oldest first: when a running tool call lands, panics, or is dropped, the waiting call with the lowest effect id starts. Ids come from one run-wide counter in issue order, and a batch's calls have consecutive ids, so the oldest batch finishes first. Every new call goes behind every call already waiting, so nothing starves. Example with a cap of 10: batch A asks for 12 calls, then batch B asks for 4 while A1 to A10 run. A1 to A10 (ids 1 to 10) start at once, A11 and A12 (ids 11 and 12) start as slots free, and B1 to B4 (ids 13 to 16) start after A's. + - Stop (`RunControl::stop_round`): running and waiting tool calls that do not survive stops are dropped and answered `Dropped`, and a waiting one never starts. A call that survives stops keeps running, or keeps waiting and starts once a slot frees. Survivors count toward the cap like any other tool call. + - Cancel, or a decided run: every waiting call is answered `Dropped` unstarted, along with everything in flight. + - Withdrawn ids: a waiting call is dropped unstarted, a running call is dropped as Stop drops one, and each is answered `Dropped`. Each effect gets exactly one answer, and an id the Harness no longer holds is ignored. + - Unchanged: what a stopped round returns (`models.loop` still discards the round), security (each call forks its own access when issued, so a waiting call already holds its identity), and replay (start order changes only landing order, which the Engine records as input). +- `crates/harness-internal/runner/src/performers.rs` (129 lines): `ToolPerformer` (lines 76-98) gains `fn max_in_flight(&self) -> NonZeroUsize`, a default method returning the private `const DEFAULT_MAX_IN_FLIGHT: NonZeroUsize = NonZeroUsize::MIN.saturating_add(9);`, so no `unwrap` is needed. Both get the docs the Technical Design gives. No implementor changes; the real performer, `HostRunContext` (`host-run.rs` line 169), keeps the default, and test tools override it. The `harness` facade doesn't re-export `ToolPerformer`, so no public listing changes. +- `crates/harness-internal/runner/src/effect_loop-flight.rs` (181), which holds the new logic and most of the prose: + - `Reach<'a>` (lines 30-37) gains `Withdrawn(&'a [EffectId])` and stays `Copy`. + - `InFlight` gains `tool_call: bool`. A new `struct Waiting { provenance, survives_stop, answer: BoxFuture }` holds an unpolled performer future. + - `Flights` gains `waiting: BTreeMap` and `cap: NonZeroUsize`, and `Flights::new(cap)` takes the cap. + - `start(id, provenance, answer)` loses its `survives_stop` parameter and serves chat and timer effects. Its body (the span, `catch_unwind`, and `Abortable` wrapping) moves into a private helper taking the `tool_call` flag, which `start` and the fill helper both call. + - `queue(id, provenance, survives_stop, answer)` inserts a tool call into `waiting`, then fills. + - A private fill helper: while fewer than `cap` tool calls are in flight and a call waits, start the lowest id (`BTreeMap::pop_first`). Counting tool calls by scanning `effects` is fine. + - `poll_landed` fills after removing a landed tool call, the panic path included. The late end of an aborted future frees nothing, because its slot was filled when it was aborted. + - `abort(reach)` keeps its signature and also takes waiting calls by the same reach: all for `Reach::All`, those that do not survive stops for `Reach::AllButSurvivors`, and the listed ones for `Reach::Withdrawn`, searched with `binary_search` inside the existing filter closure. It drops the waiting futures, aborts the reached calls in flight, returns both merged in effect order, and fills once after taking everything, so it never starts a call the same reach is about to take. Its `chosen.sort_by_key(|id| id.get())` (line 162) becomes `sort_unstable()`. + - `len` and `is_empty` count waiting calls, since each still owes an answer. `act_on_stop` compares `len` before and after a stop and needs no change. `DriveError::Stalled` must stay unreachable while calls only wait, which holds because fill leaves a call waiting only while `cap` tool calls are in flight. + - The module doc gains the cap, the waiting line, and the three reaches. +- `crates/harness-internal/runner/src/effect_loop.rs` (487 lines, may grow by at most 13; aim for about 11 net): + - `drive` reads `let cap = performers.tool.max_in_flight();` before building the `Driver`, because the struct literal moves `performers` before it builds `flights`, then sets `flights: Flights::new(cap)`. + - `perform`'s `ToolCall` arm sends its future through `self.flights.queue(...)` and returns `None`. The match's tuple becomes just the future, and chat and timer effects call `start` without the flag. Its doc says a tool call may wait. + - The `Step::Pending` arm (line 164) binds `withdrawn`. After the loop over the step's effects, when `withdrawn` is not empty, it calls `self.drop_in_flight(Reach::Withdrawn(&withdrawn))` and sets `answered`, so the loop steps again instead of awaiting. A one-line comment states the constraint: an effect can be issued and withdrawn in the same step, so withdrawal follows queuing. `drop_in_flight` answers each `Dropped` and tears down the aborted futures through `answer_landed`. A call the stop check before the effects loop already dropped is no longer held, so the reach skips it. + - The decided branch needs no change, because `abort(Reach::All)` now drains waiting calls too. + - The module doc (lines 20-35) gains one sentence: tool calls beyond the performer's cap wait in the flights, oldest first, and a cancel, a stop, or a withdrawal reaches them as well. The `drop_in_flight` doc and the `flights` field doc on `Driver` (lines 134-137) mention waiting calls. +- Docs only, which Hosts read through the `harness` facade: + - `crates/harness-internal/runner/src/harness-control.rs`: `stop_round` (lines 38-51) says a stop also drops tool calls waiting for a slot, unstarted, except those that survive stops, which start once a slot frees. `cancel` (lines 56-61) says every waiting call is answered `Dropped` too, unstarted. + - `crates/harness-internal/runner/src/harness.rs`: `Harness::new` (lines 126-132) says a run has at most 10 tool calls in flight, and the rest wait and start oldest first. +- Unchanged: the `Harness`, `RunRequest`, and `RunControl` API, `Performers`, `drive_run`, the recorder, the Workshop, `Effect::ToolCall` and its record, the Engine's batch code and `models.loop`, and the `AssistantToolCalls` and `ToolResult` events. +- Test rules: + - Every new behavior has a test that fails without it. + - A missing drop usually shows up as a hang, so every new test runs under a timeout: `run_beside`'s 10-second `PATIENCE` (`tests/it/harness.rs` lines 43 and 91-103) for `Harness::run`, or `tokio::time::timeout` around `drive_run` as `tests/it/effect_loop.rs` line 271 does. + - Before asserting that a call has not started, yield a few times, as `harness-stop.rs` does with 16 `yield_now` calls, so the check isn't vacuous. + - Tests run on `#[tokio::test]`'s current-thread runtime with no flavor set. `EffectId` has no public constructor, so `Flights` is tested through runs, not in isolation. +- New `crates/harness-internal/runner/tests/it/effect_loop-cap.rs`, through `drive_run`, declared in `tests/it/effect_loop.rs` beside its siblings (lines 28-33) as `#[path = "effect_loop-cap.rs"] mod cap;`, not in `main.rs`: + - A run with a model round builds its own prompt, model, and tool catalog, as `effect_loop-broker.rs` does (lines 40-58), with the tool offered to the model. Shared fixtures come from `support.rs` and the parent module (`begun_log`, `answers`). + - New helpers go in this file, because `support.rs` is at 486 lines: + - One test tool on the pattern of `GatedTool` (`support.rs` lines 372-400), with a `max_in_flight` override, a gate, a log of the calls it started in start order, and a teardown counter. The `Held` fixture can't serve, because its `hold` is private to `scripted.rs`. + - A scripted broker that answers each round by its last message, and can hold a round until a condition is met. + - `the_cap_holds_extra_calls_until_one_lands`: a cap of 2, and one round of three gated calls. Wait until 2 have started, and check that the third has not. Add one permit, then check that the third starts. Add the rest, and the run completes. + - `waiting_calls_start_oldest_first`: a cap of 1. Chain A's round asks for 3 calls. Then chain B's round, which the broker answers only after a1 has started, asks for 1. Wait until B's call is recorded as an effect before opening the gate; the loop queues each effect right after recording it, with no await between, so on the current-thread runtime a recorded call is already queued. Open the gate for all four. The tool records start order a1, a2, a3, b1. A wave order or a newest-first order would start b1 before a2. + - `a_call_withdrawn_in_the_step_that_issued_it_never_starts`: + - A cap of 2, and Step 1's Main and Child prompt, copied here because its `const` is private to the Engine crate: `## Main` runs `local t = tasks.spawn('## Child')`, `models.infer('wait')`, `tasks.cancel(t)`, `models.infer('linger')`, and returns `'cancelled'`; `## Child` runs `models.loop` over one user message, `go`, and returns `'child done'`. Child's loop is offered the test tool. + - The broker answers Child's round with two gated calls at once. It holds Main's `wait` round until Child's round has completed, then wakes it. Both answers then land in the same `answer_landed` drain, Child's first, so one step issues the two calls and withdraws them. + - Expect no call started, two `Dropped` answers, and the run to complete. If the Driver withdrew before queuing, the reach would find nothing, both calls would start and hold forever, and the timeout would fail the test. + - `cancelling_a_task_drops_its_waiting_and_running_calls`: a cap of 1, and a child task whose round asks for two held calls. The main chain cancels the child once one of them has started; the trigger is a main model round that the broker answers only after that call has started. Expect exactly one start, one teardown, both calls answered `Dropped`, and the run to complete. + - `a_cancel_drops_waiting_calls_without_starting_them`: a cap of 1, and one round of two held calls. Once one has started, fire the run's `CancelHandle`. Expect `RunOutcome::Cancelled`, exactly one start, and two `Dropped` answers. A `Reach::All` that missed the waiting call would let fill start it after the cancel, for two starts. +- New `crates/harness-internal/runner/tests/it/harness-stop-cap.rs`, through `Harness::run` with the real performer's default cap of 10, declared in `tests/it/harness.rs` beside `stop` and `stop_timing` (lines 31-36) as `#[path = "harness-stop-cap.rs"] mod stop_cap;`, because `harness-stop.rs` is at 415 lines and two more tests with helpers would pass 500: + - The file has its own prompt and broker, shaped like `LOOPS_ONCE` and `calls_hold_twice` in `harness-stop.rs`, which are private there. `Held` (with its `started` and `dropped` getters), `hold_host`, `with_asker`, `Operator`, and `ScriptedBroker` are `pub(crate)` in `tests/it/scripted.rs`, and `run_beside`, `until`, and `answers_to` come from the parent `harness.rs`. `Held` counts a future as started on its first poll and as dropped on teardown, so an unstarted waiting call counts as neither. + - The model sees `harness/hold` as `harness_hold` and `user-input/ask` as `user-input_ask`. + - `a_stop_drops_waiting_calls_without_starting_them`: one round of 11 `harness/hold` calls. Wait until 10 have started, then call `stop_round()`. Expect `final_text` `"false:cancelled"`, `held.started() == 10`, `held.dropped() == 10`, and 11 `Dropped` answers in `answers_to(&records, "ToolCall")`. + - `a_stop_starts_a_waiting_call_that_survives_stops`: one round of 10 `harness/hold` calls followed by one call to the fixture ask tool. The prompt offers both with `tools.always_offer`, and the Host is `with_asker(hold_host(..))` with an `Operator`. Wait until 10 holds have started, and check that no question has been asked. Call `stop_round()`; the question is asked once the stop frees the slots. Answer it. Expect the 10 holds dropped, the question's answer recorded as a non-`Dropped` `ToolCall` answer, and `final_text` `"false:cancelled"`, since the batch answers with its first failure in call order. +- Regression: every existing test passes unchanged, including `parallel_tool_calls.rs`, `batch_turn.rs`, `tool_call_access.rs`, `models_loop.rs`, and `harness-stop.rs` with its two-calls-in-flight test under the default cap of 10. At baseline no Harness, Workshop, or Plugin test uses `tasks.fanout` or builds a call list in a loop, and the largest round in `harness-stop.rs` asks for two calls. +- Rules: + - Every `.rs` file stays at or under 500 lines; the ceiling fails a file over 500. + - Clippy `all` and `pedantic` deny, `unwrap_used` and `expect_used` deny outside tests, suppressions use `#[expect(..., reason = "...")]`, and dead code and missing docs are fatal under the gate. Doctests are banned, so doc examples use non-Rust fences. + - Comments explain only non-obvious constraints. + - Out of scope: a `BinaryHeap` line (it can't remove a withdrawn call from the middle), letting survivors bypass the cap, a Host setting or mid-run change for the cap, sharing `max_concurrency`, and capping model rounds, timers, or store effects. +- `Design:` lines for the commit, from the Decision Record: + - The cap lives only in the Harness, as a default `ToolPerformer::max_in_flight()` returning 10: the performer runs the calls, and a default method changes no implementor, `Performers` literal, or `drive_run` call. The default is separate from `max_concurrency`; harnesses cluster at 8 to 10. + - Waiting calls start lowest effect id first, which finishes the oldest batch first and cannot starve. + - Waiting calls are unpolled performer futures inside `Flights`, which already owns every in-flight effect and its Stop and Cancel paths. + - A withdrawal is a third `Reach`, so `abort` and `drop_in_flight` give it the same merge, sort, fill, and teardown as a cancel or a stop. + - Calls that survive stops count toward the cap; an operator question can wait behind slow calls. +- Commit: one `promptforge` commit, `Run at most the cap of tool calls, oldest first`, with a prose body, a bullet list of changes, the `Design:` lines, and the trailer `Plan: vibe/2026-10-11-6-tool-call-cap.md`. Fixes land in this commit. Nothing is pushed. + + + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..1204f6775 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-10-11-6-tool-call-cap.md From 1b8184f7d7f69b27cdd2b60364ab12a2ff564d10 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 19:45:11 -0700 Subject: [PATCH 30/34] Run at most the cap of tool calls, oldest first A run now has at most ten tool calls running at once, unless its tool performer sets another limit, and the rest wait their turn. When a slot frees, the waiting call issued earliest starts, so the oldest batch finishes first and a newer call never jumps ahead. Model rounds and timers never wait and never count toward the limit. A waiting call has started nothing, so a cancel, a stop, or the early end of the task that issued it drops it unstarted and answers it as dropped, while a waiting call that survives stops starts once a slot frees. The driver now also drops the tool calls a run reports as withdrawn, rather than letting them run until they land. - `max_in_flight` is a default method on `ToolPerformer` that returns `DEFAULT_MAX_IN_FLIGHT`, ten. A default body means no implementor has to change, and the driver reads it once when the drive begins. - `Flights` keeps waiting tool calls as unpolled performer futures in a `BTreeMap` keyed by effect id, beside the effects in flight, and its `len` and `is_empty` count them because each still owes one answer. One owner then answers for waiting and running calls alike when a run stops, cancels, or withdraws calls. - `Reach::Withdrawn` makes a withdrawal a third reach beside a cancel and a stop, so `abort` and `drop_in_flight` give it the same merge in effect order, refill, and teardown. It looks ids up with `binary_search`, so it relies on the withdrawn list arriving in effect order. - `launch` now holds the span, panic catching, and abort wrapping, and takes flags for whether the effect is a tool call and whether it survives stops. `start` serves only chat and timer effects and loses its `survives_stop` parameter, while `queue` takes tool calls. - `fill` starts waiting calls, lowest effect id first, while fewer than the cap of tool calls are in flight, counting them by a scan of the effects in flight. - `poll_landed` refills as soon as a tool call lands, a panicking one included. An aborted call's late landing frees nothing, because `abort` already refilled its slot. - `abort` takes waiting and running calls by the same reach and refills only after taking everything, so it never starts a call it is about to drop. After a stop, that refill is what starts a waiting call that survives stops. - `withdrawn` is dropped only after the step's effects are queued, because one step can issue a call and withdraw it; withdrawing first would find nothing to drop and let the call start. The drop marks the step answered, so the loop steps again rather than waiting. - `survives_stop` calls count toward the cap like any other tool call, so an operator question can wait behind slow calls. - `effect_loop-cap.rs` pins the cap through `drive_run` with small caps: extra calls hold until one lands, waiting calls start oldest first across tasks, a call withdrawn in the step that issued it never starts, and a task cancel and a run cancel drop waiting calls unstarted. `harness-stop-cap.rs` pins a stop under the default cap of 10, for a plain waiting call and for one that survives stops. - `Harness` and `RunControl` change only in their docs, which now name the default cap of 10, oldest-first start, and how a stop and a cancel treat waiting calls. Design: new surface-growth @ crates/harness-internal/runner/src/performers.rs::ToolPerformer::max_in_flight boundary: pub Design: extends dispatch-on-tag @ crates/harness-internal/runner/src/effect_loop-flight.rs::Flights::abort Design: new flag-parameter @ crates/harness-internal/runner/src/effect_loop-flight.rs::Flights::launch Design: new temporal-coupling @ crates/harness-internal/runner/src/effect_loop.rs::Driver::drive Plan: vibe/2026-10-11-6-tool-call-cap.md --- .../runner/src/effect_loop-flight.rs | 184 +++++-- .../runner/src/effect_loop.rs | 46 +- .../runner/src/harness-control.rs | 11 +- crates/harness-internal/runner/src/harness.rs | 4 +- .../harness-internal/runner/src/performers.rs | 11 + .../runner/tests/it/effect_loop-cap.rs | 461 ++++++++++++++++++ .../runner/tests/it/effect_loop.rs | 6 +- .../runner/tests/it/harness-stop-cap.rs | 177 +++++++ .../runner/tests/it/harness.rs | 9 +- vibe/2026-10-11-6-tool-call-cap.md | 2 +- 10 files changed, 848 insertions(+), 63 deletions(-) create mode 100644 crates/harness-internal/runner/tests/it/effect_loop-cap.rs create mode 100644 crates/harness-internal/runner/tests/it/harness-stop-cap.rs diff --git a/crates/harness-internal/runner/src/effect_loop-flight.rs b/crates/harness-internal/runner/src/effect_loop-flight.rs index fcf5e4680..31b2a29f6 100644 --- a/crates/harness-internal/runner/src/effect_loop-flight.rs +++ b/crates/harness-internal/runner/src/effect_loop-flight.rs @@ -1,17 +1,35 @@ //! The effects in flight: each chat, tool-call, and timer effect's -//! performer future, polled inside the run's own future beside the rest. +//! performer future, polled inside the run's own future beside the rest, +//! and the tool calls waiting for a slot. //! -//! Every future sits in one `FuturesUnordered` as an `Abortable` future -//! under `catch_unwind`, keyed by its effect id, the provenance the effect -//! was issued under, and whether it is a tool call that survives stops. A -//! future that lands yields its effect's answer, and one that panics -//! yields `Dropped` with the panic logged. An aborted future leaves the -//! keyed set at once, since the loop answers its effect as it aborts it, -//! and is torn down the next time the set is polled; whatever it yields -//! then is discarded, so the run never sees two answers for one effect. +//! Every future in flight sits in one `FuturesUnordered` as an `Abortable` +//! future under `catch_unwind`, keyed by its effect id, the provenance the +//! effect was issued under, whether it is a tool call, and whether it +//! survives stops. A future that lands yields its effect's answer, and one +//! that panics yields `Dropped` with the panic logged. An aborted future +//! leaves the keyed set at once, since the loop answers its effect as it +//! aborts it, and is torn down the next time the set is polled; whatever +//! it yields then is discarded, so the run never sees two answers for one +//! effect. +//! +//! At most `cap` tool calls are in flight at once; chat and timer effects +//! never wait and never count toward it. A tool call beyond the cap waits +//! as its performer future, not yet polled, so a waiting call has started +//! nothing. Whenever a slot frees, because a call landed, panicked, or was +//! aborted, the waiting call with the lowest effect id starts. Ids follow +//! issue order across the run, so the oldest batch finishes first, and a +//! new call waits behind every call already waiting, so none starves. +//! +//! An abort takes waiting calls by the same reach as calls in flight: a +//! cancel takes every effect, a stop every effect but the tool calls that +//! survive stops, and a withdrawal the tool calls a step withdrew. A +//! waiting call it takes is dropped unstarted, and the loop answers it +//! `Dropped` like the rest. Every effect waiting or in flight is owed one +//! answer. use std::any::Any; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; +use std::num::NonZeroUsize; use std::panic::AssertUnwindSafe; use std::task::{Context, Poll, Waker}; @@ -27,13 +45,28 @@ use crate::performers::BoxFuture; /// the abort the loop applied. type Landing = (EffectId, Result, Aborted>); -/// Which effects in flight an abort reaches. +/// Which held effects an abort reaches. #[derive(Clone, Copy, Debug)] -pub(super) enum Reach { - /// Every effect in flight: a cancel. +pub(super) enum Reach<'a> { + /// Every effect, waiting or in flight: a cancel. All, /// Every effect but the tool calls that survive stops: a stop. AllButSurvivors, + /// The listed effects, waiting or in flight: the tool calls a step + /// withdrew, in effect order. + Withdrawn(&'a [EffectId]), +} + +impl Reach<'_> { + /// Whether this reach takes effect `id`, which survives stops when + /// `survives_stop` is set. + fn takes(self, id: EffectId, survives_stop: bool) -> bool { + match self { + Reach::All => true, + Reach::AllButSurvivors => !survives_stop, + Reach::Withdrawn(ids) => ids.binary_search(&id).is_ok(), + } + } } /// One effect in flight: what the loop needs to answer or drop it. @@ -42,45 +75,114 @@ struct InFlight { provenance: Provenance, /// Aborts the effect's future. abort: AbortHandle, + /// Whether the effect is a tool call, which counts toward the cap. + tool_call: bool, /// Whether the effect is a tool call that survives stops, which a /// stop leaves alone. survives_stop: bool, } -/// The effects in flight and their futures. +/// A tool call waiting for a slot: its performer future, not yet polled. +struct Waiting { + provenance: Provenance, + survives_stop: bool, + answer: BoxFuture, +} + +/// The effects in flight and their futures, and the tool calls waiting +/// for a slot. pub(super) struct Flights { futures: FuturesUnordered>, - /// The effects still owed an answer. A landing for an id not here is - /// an aborted future's late end and is discarded. + /// The effects in flight still owed an answer. A landing for an id not + /// here is an aborted future's late end and is discarded. effects: HashMap, + /// Tool calls waiting for a slot; the lowest id starts first. + waiting: BTreeMap, + /// The most tool calls in flight at once. + cap: NonZeroUsize, } impl Flights { - pub(super) fn new() -> Self { + pub(super) fn new(cap: NonZeroUsize) -> Self { Self { futures: FuturesUnordered::new(), effects: HashMap::new(), + waiting: BTreeMap::new(), + cap, } } /// Whether no effect is owed an answer. pub(super) fn is_empty(&self) -> bool { - self.effects.is_empty() + self.effects.is_empty() && self.waiting.is_empty() } - /// How many effects are owed an answer. + /// How many effects are owed an answer, waiting calls included. pub(super) fn len(&self) -> usize { - self.effects.len() + self.effects.len() + self.waiting.len() + } + + /// Puts `answer`, the performer future of chat or timer effect `id`, + /// in flight under `provenance`. It never waits and never counts + /// toward the cap. + pub(super) fn start( + &mut self, + id: EffectId, + provenance: Provenance, + answer: BoxFuture, + ) { + self.launch(id, provenance, false, false, answer); + } + + /// Queues `answer`, the performer future of tool call `id`, under + /// `provenance`, then starts waiting calls, lowest id first, while + /// fewer than `cap` tool calls are in flight. `survives_stop` marks a + /// call that survives stops. + pub(super) fn queue( + &mut self, + id: EffectId, + provenance: Provenance, + survives_stop: bool, + answer: BoxFuture, + ) { + self.waiting.insert( + id, + Waiting { + provenance, + survives_stop, + answer, + }, + ); + self.fill(); + } + + /// Starts waiting calls, lowest id first, while fewer than `cap` tool + /// calls are in flight. + fn fill(&mut self) { + let mut running = self + .effects + .values() + .filter(|in_flight| in_flight.tool_call) + .count(); + while running < self.cap.get() { + let Some((id, call)) = self.waiting.pop_first() else { + return; + }; + self.launch(id, call.provenance, true, call.survives_stop, call.answer); + running += 1; + } } /// Puts `answer`, the performer future of effect `id`, in flight under /// `provenance`, inside a span that records the effect id, the task /// path, and the task-local sequence, so a run's effects trace as a - /// group. `survives_stop` marks a tool call that survives stops. - pub(super) fn start( + /// group. `tool_call` marks a tool call, and `survives_stop` one that + /// survives stops. + fn launch( &mut self, id: EffectId, provenance: Provenance, + tool_call: bool, survives_stop: bool, answer: BoxFuture, ) { @@ -99,6 +201,7 @@ impl Flights { InFlight { provenance, abort, + tool_call, survives_stop, }, ); @@ -106,7 +209,8 @@ impl Flights { /// Polls for the next effect whose future landed, with its provenance /// and answer. A future that panicked lands as `Dropped`, and the - /// panic is logged against its effect. + /// panic is logged against its effect. A tool call that lands frees + /// its slot for the next waiting call. pub(super) fn poll_landed( &mut self, cx: &mut Context<'_>, @@ -118,6 +222,9 @@ impl Flights { let Some(in_flight) = self.effects.remove(&id) else { continue; }; + if in_flight.tool_call { + self.fill(); + } let answer = match landed { Ok(Ok(answer)) => answer, Ok(Err(panic)) => { @@ -146,27 +253,34 @@ impl Flights { } } - /// Aborts every effect in flight that `reach` takes and returns each - /// one's id and provenance in effect order, for the loop to answer - /// `Dropped`. - pub(super) fn abort(&mut self, reach: Reach) -> Vec<(EffectId, Provenance)> { - let mut chosen: Vec = self + /// Takes every effect waiting or in flight that `reach` takes: + /// drops each waiting call's future unstarted, aborts each effect in + /// flight, and returns each one's id and provenance in effect order, + /// for the loop to answer `Dropped`. Then starts waiting calls in the + /// slots it freed, so it never starts a call the same reach takes. + pub(super) fn abort(&mut self, reach: Reach<'_>) -> Vec<(EffectId, Provenance)> { + let in_flight = self .effects .iter() - .filter(|(_, in_flight)| match reach { - Reach::All => true, - Reach::AllButSurvivors => !in_flight.survives_stop, - }) - .map(|(id, _)| *id) - .collect(); - chosen.sort_by_key(|id| id.get()); + .filter(|(id, in_flight)| reach.takes(**id, in_flight.survives_stop)) + .map(|(id, _)| *id); + let waiting = self + .waiting + .iter() + .filter(|(id, call)| reach.takes(**id, call.survives_stop)) + .map(|(id, _)| *id); + let mut chosen: Vec = in_flight.chain(waiting).collect(); + chosen.sort_unstable(); let mut aborted = Vec::with_capacity(chosen.len()); for id in chosen { if let Some(in_flight) = self.effects.remove(&id) { in_flight.abort.abort(); aborted.push((id, in_flight.provenance)); + } else if let Some(call) = self.waiting.remove(&id) { + aborted.push((id, call.provenance)); } } + self.fill(); aborted } } diff --git a/crates/harness-internal/runner/src/effect_loop.rs b/crates/harness-internal/runner/src/effect_loop.rs index d3a8b937b..92cee6a46 100644 --- a/crates/harness-internal/runner/src/effect_loop.rs +++ b/crates/harness-internal/runner/src/effect_loop.rs @@ -32,7 +32,8 @@ //! loop steps again instead of awaiting an answer. A Vfs effect is never in //! flight at a cancel or a stop: it is answered before the loop waits //! again. A drop is an answer, recorded like any other, so every effect -//! record has exactly one answer record. +//! record has exactly one answer record. Tool calls beyond the performer's +//! cap wait, oldest first, and a cancel, a stop, or a withdrawal reaches them. //! //! A performer that panics is caught where its future is polled: its //! effect is answered `Dropped` and the panic is logged, so a lost @@ -113,12 +114,13 @@ pub(crate) async fn drive( cancel: CancelHandle, stop: Arc, ) -> Result { + let cap = performers.tool.max_in_flight(); let mut driver = Driver { run, performers, recorder, run_id, - flights: Flights::new(), + flights: Flights::new(cap), cancel, stop, }; @@ -131,9 +133,9 @@ struct Driver { performers: Performers, recorder: Arc, run_id: RunId, - /// The chat, tool-call, and timer effects in flight. Dropping the - /// driver drops their futures, so nothing a performer holds outlives - /// the drive however it ends. + /// The chat, tool-call, and timer effects in flight and the tool calls + /// waiting for a slot. Dropping the driver drops their futures, so + /// nothing a performer holds outlives the drive however it ends. flights: Flights, cancel: CancelHandle, stop: Arc, @@ -162,7 +164,9 @@ impl Driver { return Ok(outcome); } Step::Pending { - effects, events, .. + effects, + events, + withdrawn, } => { // The run's own word, not a scan of its events: the // events are a report, not a control channel. A cancel @@ -201,6 +205,11 @@ impl Driver { answered = true; } } + // A step can issue a call and withdraw it, so withdrawal follows queuing. + if !withdrawn.is_empty() { + self.drop_in_flight(Reach::Withdrawn(&withdrawn)).await?; + answered = true; + } if answered { // Stepping again, not awaiting: the answers just // resumed, a stop's drops as well as the inline @@ -277,11 +286,11 @@ impl Driver { Ok(()) } - /// Aborts the effects in flight that `reach` takes and answers each - /// `Dropped`, in effect order. The aborted futures are then torn down, + /// Answers `Dropped`, in effect order, each effect waiting or in + /// flight that `reach` takes. The aborted futures are then torn down, /// so whatever a dropped performer held is gone before the run steps /// on; an effect `reach` spared that landed meanwhile is answered too. - async fn drop_in_flight(&mut self, reach: Reach) -> Result<(), RecorderError> { + async fn drop_in_flight(&mut self, reach: Reach<'_>) -> Result<(), RecorderError> { for (id, provenance) in self.flights.abort(reach) { self.drop_effect(id, &provenance).await?; } @@ -311,9 +320,9 @@ impl Driver { } /// Puts the performer future of a chat, tool-call, or timer effect in - /// flight and returns `None`. The performer is called inside its - /// future, so a performer that panics as it is called is caught like - /// one that panics as it runs. + /// flight, where a tool call may wait for a slot, and returns `None`. + /// The performer is called inside its future, so a performer that + /// panics as it is called is caught like one that panics as it runs. /// /// A Vfs effect goes nowhere and is handed back as the access and /// operation it carries, for the caller to answer inline. @@ -323,7 +332,7 @@ impl Driver { provenance: &Provenance, effect: Effect, ) -> Option<(Arc, VfsOp)> { - let (answer, survives_stop): (BoxFuture, bool) = match effect { + let answer: BoxFuture = match effect { Effect::Chat { binding, messages, @@ -337,7 +346,7 @@ impl Driver { let completion = broker.chat(binding, messages, tools, options, round).await; EffectAnswer::Chat(completion) }; - (Box::pin(round), false) + Box::pin(round) } Effect::ToolCall { tool, @@ -351,7 +360,9 @@ impl Driver { let call = async move { EffectAnswer::ToolCall(performer.call(tool, alias, access, origin, args).await) }; - (Box::pin(call), survives_stop) + self.flights + .queue(id, provenance.clone(), survives_stop, Box::pin(call)); + return None; } Effect::Vfs { access, op } => return Some((access, op)), Effect::Timer { seconds } => { @@ -360,11 +371,10 @@ impl Driver { timer.sleep(seconds).await; EffectAnswer::Timer }; - (Box::pin(sleep), false) + Box::pin(sleep) } }; - self.flights - .start(id, provenance.clone(), survives_stop, answer); + self.flights.start(id, provenance.clone(), answer); None } diff --git a/crates/harness-internal/runner/src/harness-control.rs b/crates/harness-internal/runner/src/harness-control.rs index 998cbbeb7..33e5f87ab 100644 --- a/crates/harness-internal/runner/src/harness-control.rs +++ b/crates/harness-internal/runner/src/harness-control.rs @@ -45,6 +45,9 @@ impl RunControl { /// A `pcall` around a dropped call catches its cancelled error. /// Otherwise, the error ends the run cancelled. /// + /// A stop also drops the tool calls waiting for a slot, unstarted, + /// except those that survive stops, which start once a slot frees. + /// /// A stop reaches only the effects in flight when the loop that drives /// the run sees it. The loop looks for a stop while it waits for an /// answer and before it starts a step's effects. It drops the effects @@ -56,9 +59,11 @@ impl RunControl { /// Cancels the run. /// /// Every effect in flight, tool calls that survive stops included, is - /// answered `Dropped`, and the run ends cancelled. A cancel raised - /// before the run begins ends the run before it reaches the recorder. - /// Calling `cancel` more than once is the same as calling it once. + /// answered `Dropped`, and the run ends cancelled. Every tool call + /// waiting for a slot is answered `Dropped` too, unstarted. A cancel + /// raised before the run begins ends the run before it reaches the + /// recorder. Calling `cancel` more than once is the same as calling it + /// once. pub fn cancel(&self) { self.cancel.cancel(); } diff --git a/crates/harness-internal/runner/src/harness.rs b/crates/harness-internal/runner/src/harness.rs index c899cb59d..2459cf075 100644 --- a/crates/harness-internal/runner/src/harness.rs +++ b/crates/harness-internal/runner/src/harness.rs @@ -129,7 +129,9 @@ impl Harness { /// run's model and gets the model's replies through `broker`. It sleeps /// through `timer`. Its tool calls go to the Plugins installed in /// `host`, and each call is lent `services`, the run's own, such as - /// its input broker. All of this work happens in [`Harness::run`]. + /// its input broker. A run has at most 10 tool calls in flight; the + /// rest wait and start oldest first. All of this work happens in + /// [`Harness::run`]. #[must_use] pub fn new( recorder: Arc, diff --git a/crates/harness-internal/runner/src/performers.rs b/crates/harness-internal/runner/src/performers.rs index ef32a9086..c2fbc9050 100644 --- a/crates/harness-internal/runner/src/performers.rs +++ b/crates/harness-internal/runner/src/performers.rs @@ -25,6 +25,7 @@ //! [`InferenceBroker`] and the [`Timer`]. use std::future::Future; +use std::num::NonZeroUsize; use std::pin::Pin; use std::sync::Arc; @@ -73,6 +74,10 @@ pub trait InferenceBroker: Send + Sync { ) -> BoxFuture, CompletionError>>; } +/// The tool calls a run has in flight at once unless its performer says +/// otherwise. +const DEFAULT_MAX_IN_FLIGHT: NonZeroUsize = NonZeroUsize::MIN.saturating_add(9); + /// Performs a `ToolCall` effect: resolves `tool` to an implementation and /// calls it with `args`, lending it the call's `access` and `origin`. pub trait ToolPerformer: Send + Sync { @@ -95,6 +100,12 @@ pub trait ToolPerformer: Send + Sync { /// `survives_stop` flag. A stop drops every other call in flight; a /// cancel drops this one too. fn survives_stop(&self, tool: &ToolId) -> bool; + + /// The most tool calls the Harness runs at once across the run; calls + /// beyond it wait and start oldest first. The default is 10. + fn max_in_flight(&self) -> NonZeroUsize { + DEFAULT_MAX_IN_FLIGHT + } } /// The Host's clock, which performs each `Timer` effect as one sleep. diff --git a/crates/harness-internal/runner/tests/it/effect_loop-cap.rs b/crates/harness-internal/runner/tests/it/effect_loop-cap.rs new file mode 100644 index 000000000..5c905c232 --- /dev/null +++ b/crates/harness-internal/runner/tests/it/effect_loop-cap.rs @@ -0,0 +1,461 @@ +//! The tool-call cap: at most the performer's `max_in_flight()` tool calls +//! run at once, the rest wait and start lowest effect id first, and a +//! waiting call that a withdrawal, a task cancel, or a run cancel reaches +//! is answered `Dropped` without ever starting. + +use std::future::Future; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use harness_runner::effect_loop::drive_run; +use harness_runner::performers::{BoxFuture, ToolPerformer}; +use harness_runner::recorder::{MemoryRecorder, RecordKind, RunId, RunOutcome}; +use promptforge::cancel::CancelHandle; +use promptforge::effect::ToolCallOrigin; +use promptforge::model::{ + Completion, CompletionError, CompletionResult, ModelDescriptor, ModelId, ThinkingMode, ToolCall, +}; +use promptforge::timestamp::Timestamp; +use promptforge::tools::{ToolCatalog, ToolDescriptor, ToolError, ToolId, ToolOutput}; +use promptforge::vfs::Access; +use promptforge::{Environment, Prompt, Run, RunContext}; +use serde_json::{Value, json}; +use tokio::sync::{Notify, Semaphore}; + +use super::begun_log; +use crate::scripted::{Kept, MODEL, ScriptedBroker, reply}; +use crate::support::{EXECUTION, unused}; + +/// How long a test waits for a run before it fails. +const PATIENCE: Duration = Duration::from_secs(10); + +/// The fixture tool every run here offers its model. +const TOOL: &str = "tests/cap/park"; + +/// [`TOOL`]'s wire name, as the model calls it. +const WIRE: &str = "tests_cap_park"; + +/// The frontmatter and setup every run here shares: one model role bound +/// as the default, and [`TOOL`] offered to every model round. +const PREAMBLE: &str = "---\nname: runner-cap\ndescription: caps tool calls\npromptforge: 0\n\ + models:\n writer: {}\n---\n\n\ + # Cap\n\n```lua\nmodels.default('writer')\ntools.always_offer('tests/cap/park')\n```\n\n"; + +/// Main spawns Child and waits on a `wait` round while Child's loop asks +/// its round, then cancels Child and lingers on a second round. +const MAIN_CANCELS: &str = "## Main\n\n```lua\n\ + local t = tasks.spawn('## Child')\n\ + models.infer('wait')\n\ + tasks.cancel(t)\n\ + models.infer('linger')\n\ + return 'cancelled'\n```\n\n"; + +/// Main spawns A, then B, and joins both. +const MAIN_JOINS: &str = "## Main\n\n```lua\n\ + local a = tasks.spawn('## A')\n\ + local b = tasks.spawn('## B')\n\ + tasks.join({ a, b })\n\ + return 'joined'\n```\n\n"; + +/// A section that runs a model loop over one user message, `user`, and +/// returns `result`. +fn looping(section: &str, user: &str, result: &str) -> String { + format!( + "## {section}\n\n```lua\nlocal msgs = messages.new()\nmsgs:user('{user}')\n\ + models.loop(msgs)\nreturn '{result}'\n```\n\n" + ) +} + +/// A run over [`PREAMBLE`] and `sections`, with the one role bound to one +/// current model and [`TOOL`] in its catalog. +fn run_of(sections: &str) -> Run { + let source = format!("{PREAMBLE}{sections}"); + let (prompt, _parse_events) = Prompt::parse(&source, EXECUTION); + let prompt = Arc::new(prompt.expect("the cap fixture parses")); + let model = ModelDescriptor::new( + ModelId::gateway(MODEL).expect("the model id is valid"), + "The host's current model", + NonZeroU32::new(131_072).expect("131072 is non-zero"), + ThinkingMode::Never, + ); + let tool = ToolDescriptor::new( + ToolId::parse(TOOL).expect("the tool id is valid"), + "Park until the test opens the gate.", + json!({ "type": "object", "properties": { "name": { "type": "string" } } }), + ); + let catalog = ToolCatalog::new(&[tool]).expect("the one-tool catalog is valid"); + let ctx = RunContext::new(EXECUTION, 7, Timestamp::UNIX_EPOCH).model(model); + let (ctx, requirements) = Environment::new().tools(catalog).prepare(&prompt, ctx); + assert!( + requirements.is_satisfied(), + "the bound model and the catalog satisfy the fixture: {requirements:?}" + ); + Run::new(prompt, "", ctx) +} + +/// Parks every call until the test adds a permit to `gate`, then answers +/// it, with at most `cap` calls in flight. Logs each call's `name` +/// argument as the call starts, on its first poll, and counts each +/// started call torn down before it answered. +struct ParkingTool { + cap: NonZeroUsize, + gate: Arc, + started: Kept, + torn_down: Arc, +} + +impl ParkingTool { + fn with_cap(cap: usize) -> Arc { + Arc::new(Self { + cap: NonZeroUsize::new(cap).expect("the cap is not zero"), + gate: Arc::new(Semaphore::new(0)), + started: Kept::default(), + torn_down: Arc::default(), + }) + } + + /// The `name` of every call started, in start order. + fn started(&self) -> Vec { + self.started.lock().unwrap().clone() + } + + /// Lets `calls` more parked calls answer. + fn open(&self, calls: usize) { + self.gate.add_permits(calls); + } + + fn torn_down(&self) -> usize { + self.torn_down.load(Ordering::SeqCst) + } +} + +impl ToolPerformer for ParkingTool { + fn call( + &self, + _tool: ToolId, + _alias: String, + _access: Arc, + _origin: ToolCallOrigin, + args: Value, + ) -> BoxFuture> { + let name = args["name"].as_str().unwrap_or_default().to_owned(); + let gate = Arc::clone(&self.gate); + let started = Arc::clone(&self.started); + let torn_down = Arc::clone(&self.torn_down); + Box::pin(async move { + started.lock().unwrap().push(name); + let mut torn_down = TornDown(Some(torn_down)); + gate.acquire() + .await + .expect("the gate semaphore is never closed") + .forget(); + torn_down.0 = None; + Ok(ToolOutput::trusted("opened")) + }) + } + + fn survives_stop(&self, _tool: &ToolId) -> bool { + false + } + + fn max_in_flight(&self) -> NonZeroUsize { + self.cap + } +} + +/// Counts a call torn down before it answered. +struct TornDown(Option>); + +impl Drop for TornDown { + fn drop(&mut self) { + if let Some(count) = &self.0 { + count.fetch_add(1, Ordering::SeqCst); + } + } +} + +/// Plays each round through `script`, handed the round's last message. +fn answering( + script: impl Fn(&str) -> BoxFuture, CompletionError>> + Send + Sync + 'static, +) -> ScriptedBroker { + ScriptedBroker::new(move |_round, messages| { + let last = messages + .last() + .map(|message| message.content().to_owned()) + .unwrap_or_default(); + script(&last) + }) +} + +/// A completion asking for [`TOOL`] once per entry of `names`, each +/// call's id and `name` argument the entry. +fn parks(names: &[&str]) -> Result, CompletionError> { + let calls = names + .iter() + .map(|name| ToolCall::from_parts(*name, WIRE, json!({ "name": name }))) + .collect::, _>>()?; + Completion::from_result(CompletionResult::ToolCalls(calls), MODEL).map(Box::new) +} + +/// Yields until `condition` holds, or fails naming `what`. Every +/// performer here is in memory, so the run reaches each state within a +/// few polls. +async fn until(what: &str, condition: impl Fn() -> bool) { + for _ in 0..10_000 { + if condition() { + return; + } + tokio::task::yield_now().await; + } + panic!("{what}"); +} + +/// Yields enough for the run beside the test to settle, so a check that +/// a call has not started is not vacuous. +async fn settle() { + for _ in 0..16 { + tokio::task::yield_now().await; + } +} + +/// One run's recorder, parking tool, and cancel flag, shared by the run +/// and the test that steers it. +struct Rig { + recorder: Arc, + run_id: RunId, + tool: Arc, + cancel: CancelHandle, +} + +impl Rig { + /// A rig whose tool runs at most `cap` calls at once. + async fn with_cap(cap: usize) -> Self { + let (recorder, run_id) = begun_log().await; + Self { + recorder, + run_id, + tool: ParkingTool::with_cap(cap), + cancel: CancelHandle::new(), + } + } + + /// Drives `run` with `broker` and the rig's tool beside `steer`, and + /// returns the run's outcome. Fails when the two outlast [`PATIENCE`]. + async fn drive( + &self, + run: Run, + broker: ScriptedBroker, + steer: impl Future, + ) -> RunOutcome { + let mut performers = unused(); + performers.broker = Arc::new(broker); + performers.tool = self.tool.clone(); + let drive = drive_run( + run, + performers, + self.recorder.clone(), + self.run_id, + self.cancel.clone(), + ); + let (outcome, ()) = tokio::time::timeout(PATIENCE, async { tokio::join!(drive, steer) }) + .await + .expect("the run ends in time"); + outcome.expect("the drive reaches an outcome") + } + + /// Whether the run has recorded a tool call whose `name` is `name`. + fn recorded(&self, name: &str) -> bool { + self.recorder.records(self.run_id).iter().any(|record| { + record.kind == RecordKind::Effect && record.payload["ToolCall"]["args"]["name"] == name + }) + } + + /// The answer payloads of the run's tool calls, in effect order. + fn tool_call_answers(&self) -> Vec { + let records = self.recorder.records(self.run_id); + records + .iter() + .filter(|record| { + record.kind == RecordKind::Effect && record.payload.get("ToolCall").is_some() + }) + .map(|effect| { + records + .iter() + .find(|record| { + record.kind == RecordKind::Answer && record.effect_id == effect.effect_id + }) + .expect("every tool call is answered") + .payload + .clone() + }) + .collect() + } +} + +fn completed(final_text: &str) -> RunOutcome { + RunOutcome::Completed { + final_text: final_text.to_owned(), + } +} + +#[tokio::test] +async fn the_cap_holds_extra_calls_until_one_lands() { + let rig = Rig::with_cap(2).await; + let broker = answering(|last| { + let answer = if last == "go" { + parks(&["c1", "c2", "c3"]) + } else { + reply("done") + }; + Box::pin(async move { answer }) + }); + + let outcome = rig + .drive(run_of(&looping("Only", "go", "done")), broker, async { + until("two calls start", || rig.tool.started().len() >= 2).await; + settle().await; + assert_eq!( + rig.tool.started(), + ["c1", "c2"], + "the third call waits for a slot" + ); + rig.tool.open(1); + until("the third call starts", || rig.tool.started().len() >= 3).await; + rig.tool.open(2); + }) + .await; + + assert_eq!(outcome, completed("done")); + assert_eq!(rig.tool.started(), ["c1", "c2", "c3"]); + assert_eq!(rig.tool_call_answers().len(), 3); +} + +#[tokio::test] +async fn waiting_calls_start_oldest_first() { + let rig = Rig::with_cap(1).await; + let tool = Arc::clone(&rig.tool); + let broker = answering(move |last| match last { + "a" => Box::pin(async { parks(&["a1", "a2", "a3"]) }), + "b" => { + let tool = Arc::clone(&tool); + Box::pin(async move { + until("a1 starts", || tool.started().contains(&"a1".to_owned())).await; + parks(&["b1"]) + }) + } + _ => Box::pin(async { reply("done") }), + }); + let sections = format!( + "{MAIN_JOINS}{}{}", + looping("A", "a", "a done"), + looping("B", "b", "b done") + ); + + let outcome = rig + .drive(run_of(§ions), broker, async { + until("B's call is issued", || rig.recorded("b1")).await; + settle().await; + assert_eq!(rig.tool.started(), ["a1"], "the cap holds the rest"); + rig.tool.open(4); + }) + .await; + + assert_eq!(outcome, completed("joined")); + assert_eq!( + rig.tool.started(), + ["a1", "a2", "a3", "b1"], + "A's waiting calls start before B's newer one" + ); +} + +#[tokio::test] +async fn a_call_withdrawn_in_the_step_that_issued_it_never_starts() { + let rig = Rig::with_cap(2).await; + let child_answered = Arc::new(Notify::new()); + let broker = answering(move |last| { + let child_answered = Arc::clone(&child_answered); + match last { + "go" => Box::pin(async move { + let answer = parks(&["c1", "c2"]); + child_answered.notify_one(); + answer + }), + "wait" => Box::pin(async move { + child_answered.notified().await; + reply("go") + }), + _ => Box::pin(async { reply("done") }), + } + }); + let sections = format!("{MAIN_CANCELS}{}", looping("Child", "go", "child done")); + + let outcome = rig.drive(run_of(§ions), broker, async {}).await; + + assert_eq!(outcome, completed("cancelled")); + assert!( + rig.tool.started().is_empty(), + "a withdrawn call never starts: {:?}", + rig.tool.started() + ); + assert_eq!( + rig.tool_call_answers(), + [json!("Dropped"), json!("Dropped")] + ); +} + +#[tokio::test] +async fn cancelling_a_task_drops_its_waiting_and_running_calls() { + let rig = Rig::with_cap(1).await; + let tool = Arc::clone(&rig.tool); + let broker = answering(move |last| match last { + "go" => Box::pin(async { parks(&["c1", "c2"]) }), + "wait" => { + let tool = Arc::clone(&tool); + Box::pin(async move { + until("one of Child's calls starts", || !tool.started().is_empty()).await; + reply("go") + }) + } + _ => Box::pin(async { reply("done") }), + }); + let sections = format!("{MAIN_CANCELS}{}", looping("Child", "go", "child done")); + + let outcome = rig.drive(run_of(§ions), broker, async {}).await; + + assert_eq!(outcome, completed("cancelled")); + assert_eq!(rig.tool.started(), ["c1"], "the waiting call never started"); + assert_eq!(rig.tool.torn_down(), 1, "the running call was torn down"); + assert_eq!( + rig.tool_call_answers(), + [json!("Dropped"), json!("Dropped")] + ); +} + +#[tokio::test] +async fn a_cancel_drops_waiting_calls_without_starting_them() { + let rig = Rig::with_cap(1).await; + let broker = answering(|last| { + let answer = if last == "go" { + parks(&["c1", "c2"]) + } else { + reply("done") + }; + Box::pin(async move { answer }) + }); + + let outcome = rig + .drive(run_of(&looping("Only", "go", "done")), broker, async { + until("one call starts", || !rig.tool.started().is_empty()).await; + rig.cancel.cancel(); + }) + .await; + + assert_eq!(outcome, RunOutcome::Cancelled); + assert_eq!(rig.tool.started(), ["c1"], "the waiting call never started"); + assert_eq!( + rig.tool_call_answers(), + [json!("Dropped"), json!("Dropped")] + ); +} diff --git a/crates/harness-internal/runner/tests/it/effect_loop.rs b/crates/harness-internal/runner/tests/it/effect_loop.rs index ebf064641..211f70b4b 100644 --- a/crates/harness-internal/runner/tests/it/effect_loop.rs +++ b/crates/harness-internal/runner/tests/it/effect_loop.rs @@ -4,8 +4,8 @@ //! performer that panics drops its effect rather than stranding the run; //! and a refused recorder write ends the drive with the recorder's error //! and tears down the performers still out. The Vfs effect the loop -//! answers inline has its own module, `vfs`, and the `Chat` effect the -//! inference broker answers has `broker`. +//! answers inline has its own module, `vfs`, the `Chat` effect the +//! inference broker answers has `broker`, and the tool-call cap has `cap`. use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -27,6 +27,8 @@ use crate::support::{ #[path = "effect_loop-broker.rs"] mod broker; +#[path = "effect_loop-cap.rs"] +mod cap; #[path = "effect_loop-recorder.rs"] mod recorder_failures; #[path = "effect_loop-vfs.rs"] diff --git a/crates/harness-internal/runner/tests/it/harness-stop-cap.rs b/crates/harness-internal/runner/tests/it/harness-stop-cap.rs new file mode 100644 index 000000000..21f9546ae --- /dev/null +++ b/crates/harness-internal/runner/tests/it/harness-stop-cap.rs @@ -0,0 +1,177 @@ +//! A stop under the real performer's default cap of 10: the stop drops +//! the calls waiting for a slot without starting them, and a waiting call +//! that survives stops starts once the stop frees a slot. + +use std::fmt::Write as _; +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use harness_runner::Harness; +use harness_runner::recorder::MemoryRecorder; +use promptforge::model::{Completion, CompletionResult, ToolCall}; +use promptforge_plugin::HostServices; +use serde_json::json; + +use super::{ + PendingTimer, RunOutcome, answers_to, answers_where, calls, request, run_beside, until, +}; +use crate::scripted::{Held, MODEL, Operator, ScriptedBroker, hold_host, reply, with_asker}; + +/// The held tool's id. +const HOLD_TOOL: &str = "harness/hold"; + +/// The fixture ask tool's id, under the name the asker is installed as. +const ASK_TOOL: &str = "user-input/ask"; + +/// A model loop under a `pcall` that offers each of `tools` and names +/// each one's Plugin, reporting how the loop call ended. +fn loops_offering(tools: &[&str]) -> String { + let mut plugins = String::new(); + let mut offers = String::new(); + for tool in tools { + let plugin = tool.split('/').next().unwrap_or_default(); + writeln!(plugins, " - {plugin}").unwrap(); + writeln!(offers, "tools.always_offer('{tool}')").unwrap(); + } + format!( + "---\nname: loops\ndescription: d\npromptforge: 0\nplugins:\n{plugins}\ + models:\n writer: {{}}\n---\n\n\ + # Loops\n\n```lua\nmodels.default('writer')\n{offers}```\n\n\ + ## Only\n\n```lua\nlocal msgs = messages.new()\nmsgs:user('hold')\n\ + local ok, err = pcall(models.loop, msgs)\n\ + return tostring(ok) .. ':' .. err.kind\n```\n" + ) +} + +/// A broker whose first round calls each of `wire_names` in order, as +/// `c1` onward, and whose every later round replies with text. +fn calls_once(wire_names: Vec<&'static str>) -> ScriptedBroker { + ScriptedBroker::new(move |round, _messages| { + let wire_names = wire_names.clone(); + Box::pin(async move { + if round.id.get() > 0 { + return reply("done"); + } + let calls = wire_names + .into_iter() + .enumerate() + .map(|(index, name)| { + ToolCall::from_parts(format!("c{}", index + 1), name, json!({})) + }) + .collect::, _>>()?; + Completion::from_result(CompletionResult::ToolCalls(calls), MODEL).map(Box::new) + }) + }) +} + +/// Yields enough for the run beside the test to settle, so a check that +/// a call has not started is not vacuous. +async fn settle() { + for _ in 0..16 { + tokio::task::yield_now().await; + } +} + +#[tokio::test] +async fn a_stop_drops_waiting_calls_without_starting_them() { + let held = Arc::new(Held::default()); + let recorder = Arc::new(MemoryRecorder::new()); + let harness = Harness::new( + recorder.clone(), + Arc::new(calls_once(vec!["harness_hold"; 11])), + Arc::new(PendingTimer::default()), + Arc::new(hold_host(&held)), + HostServices::new(), + ); + let watched = Arc::clone(&held); + + let report = run_beside( + harness, + request(&loops_offering(&[HOLD_TOOL])), + |control| async move { + until("ten calls are in flight", || watched.started() >= 10).await; + settle().await; + assert_eq!(watched.started(), 10, "the eleventh call waits for a slot"); + control.stop_round(); + }, + ) + .await + .expect("the run reaches an outcome"); + + assert_eq!( + report.outcome, + RunOutcome::Completed { + final_text: "false:cancelled".to_owned() + }, + "the loop's pcall caught the dropped batch" + ); + assert_eq!(held.started(), 10, "the waiting call never started"); + assert_eq!(held.dropped(), 10, "the stop tore down the ten in flight"); + let records = recorder.records(report.run_id.expect("the run began")); + assert_eq!( + answers_to(&records, "ToolCall"), + vec![json!("Dropped"); 11], + "the stop answered every call Dropped, the waiting one included" + ); +} + +#[tokio::test] +async fn a_stop_starts_a_waiting_call_that_survives_stops() { + let held = Arc::new(Held::default()); + let (operator, answers) = Operator::new(); + let asked = Arc::clone(&operator.asked); + let recorder = Arc::new(MemoryRecorder::new()); + let mut wire_names = vec!["harness_hold"; 10]; + wire_names.push("user-input_ask"); + let harness = Harness::new( + recorder.clone(), + Arc::new(calls_once(wire_names)), + Arc::new(PendingTimer::default()), + Arc::new(with_asker(hold_host(&held))), + operator.services(), + ); + let watched = Arc::clone(&held); + + let report = run_beside( + harness, + request(&loops_offering(&[HOLD_TOOL, ASK_TOOL])), + |control| async move { + until("ten holds are in flight", || watched.started() >= 10).await; + settle().await; + assert_eq!( + asked.load(Ordering::SeqCst), + 0, + "the question waits behind the ten holds" + ); + control.stop_round(); + until("the stop frees a slot for the question", || { + asked.load(Ordering::SeqCst) == 1 + }) + .await; + answers.send("kept".to_owned()).unwrap(); + }, + ) + .await + .expect("the run reaches an outcome"); + + assert_eq!( + report.outcome, + RunOutcome::Completed { + final_text: "false:cancelled".to_owned() + }, + "the batch answers with its first failure in call order" + ); + assert_eq!(held.started(), 10); + assert_eq!(held.dropped(), 10, "the stop tore down the ten holds"); + let records = recorder.records(report.run_id.expect("the run began")); + assert_eq!( + answers_where(&records, "ToolCall", calls(HOLD_TOOL)), + vec![json!("Dropped"); 10] + ); + let asks = answers_where(&records, "ToolCall", calls(ASK_TOOL)); + assert_eq!(asks.len(), 1, "one question: {asks:?}"); + assert_eq!( + asks[0]["ToolCall"]["Ok"]["text"], "kept", + "the question survived the stop and was answered" + ); +} diff --git a/crates/harness-internal/runner/tests/it/harness.rs b/crates/harness-internal/runner/tests/it/harness.rs index 8787b68d1..1a0dcd6ae 100644 --- a/crates/harness-internal/runner/tests/it/harness.rs +++ b/crates/harness-internal/runner/tests/it/harness.rs @@ -4,9 +4,10 @@ //! recorded failed run; a Vfs effect is answered inline as it is issued; //! and one run completes under a thread-parking executor with no runtime //! behind it. How a stop and a cancel reach the effects in flight sits in -//! the `stop` child module, how what a stop reaches depends on when the -//! loop sees it in the `stop_timing` child module, and every error `run` -//! returns in the `errors` child module. +//! the `stop` child module, how a stop reaches the tool calls waiting for +//! a slot in the `stop_cap` child module, how what a stop reaches depends +//! on when the loop sees it in the `stop_timing` child module, and every +//! error `run` returns in the `errors` child module. use std::future::Future; use std::pin::Pin; @@ -32,6 +33,8 @@ use crate::support::{HookedBackend, PendingTimer}; mod errors; #[path = "harness-stop.rs"] mod stop; +#[path = "harness-stop-cap.rs"] +mod stop_cap; #[path = "harness-stop-timing.rs"] mod stop_timing; diff --git a/vibe/2026-10-11-6-tool-call-cap.md b/vibe/2026-10-11-6-tool-call-cap.md index e35ba84ac..f90a96391 100644 --- a/vibe/2026-10-11-6-tool-call-cap.md +++ b/vibe/2026-10-11-6-tool-call-cap.md @@ -569,7 +569,7 @@ The design moved from an Engine queue, to an Engine-ordered line with the cap in -### Step 2: Run at most the cap of tool calls, oldest first +### Step 2: Run at most the cap of tool calls, oldest first [completed] - Component: `none` - Tests: Verify runs every command below as written, in this order, from the `promptforge` repository root. Setting `$env:` variables persists for the session, so the API check below clears `CARGO_BUILD_WARNINGS` first. From 9cdef938971aaaf97745fb35620e4bf71e27ccc6 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 20:00:58 -0700 Subject: [PATCH 31/34] Close plan: tool-call-cap Plan: vibe/2026-10-11-6-tool-call-cap.md --- vibe/2026-10-11-6-tool-call-cap.md | 10 +++++----- vibe/ACTIVE | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/2026-10-11-6-tool-call-cap.md b/vibe/2026-10-11-6-tool-call-cap.md index f90a96391..62fc3c11e 100644 --- a/vibe/2026-10-11-6-tool-call-cap.md +++ b/vibe/2026-10-11-6-tool-call-cap.md @@ -4,19 +4,19 @@ overview: Bound the Plugin tool calls running at once across a run, with the cap todos: - id: engine-withdrawn content: "Step 1: EffectId derives PartialOrd and Ord; Step::Pending gains withdrawn, filled by abort_effect for tool-call continuations and cleared when the run is decided; update every exhaustive Step::Pending pattern; bless and review public-api.txt; three Engine tests in parallel_tool_calls.rs" - status: pending + status: completed - id: harness-cap content: "Step 2: ToolPerformer::max_in_flight, a default method returning 10; Flights keeps waiting futures in a BTreeMap by EffectId, caps tool calls in flight, starts the lowest id when a slot frees, and drains waiting calls in abort(reach), which gains Reach::Withdrawn; start drops its survives_stop parameter" - status: pending + status: completed - id: harness-driver content: "Step 2: Driver builds Flights with the performer's cap, sends tool calls through Flights::queue, and drops the step's withdrawn ids through drop_in_flight after starting its new effects; RunControl and Harness::new docs name the cap and waiting calls" - status: pending + status: completed - id: harness-tests content: "Step 2: end-to-end tests: the cap holds extra calls, waiting calls start oldest first, a call withdrawn in the step that issued it never starts, cancelling a task drops its waiting and running calls, a cancel and a stop drop waiting calls unstarted, a stop starts a waiting call that survives stops" - status: pending + status: completed - id: verify content: "Each step: both nextest runs, both clippy runs, fmt, both docs runs, xtask api --check, build-xtask, docs-claims" - status: pending + status: completed isProject: false --- diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 1204f6775..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-10-11-6-tool-call-cap.md From 0ab627762102f1a32c1f490f88fd0ff7dbfd0a58 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 22:28:07 -0700 Subject: [PATCH 32/34] Document every occasion that drops a Plugin call The public docs said only a cancel could drop a call whose tool survives stops, which left out the run ending for other reasons and the early end of the task that made the call. The Plugin contract now holds the one full list of drop occasions, and the other public copies state only their own fact and point to it, so the next occasion needs one edit. The same contract now tells Plugin authors that a call must not wait for another call of the same run, because the Harness runs a bounded number at once and holds the rest. The doc for stopping a round now says a stop reaches waiting effects as well as those in flight. Behavior is unchanged. - `Plugin::call` now lists every occasion the Harness drops its future: a stop, the run's end with a cancel included, and the early end of the task that made the call, when it is cancelled, its owner ends, or an ancestor is aborted. Only a stop spares a call whose tool sets `survives_stop`, and the run goes on after a stop or a task's early end. - `crates/promptforge-plugin/src/lib.rs` and `crates/plugin-user-input/src/lib.rs` stop restating the occasions and point to the list on `Plugin::call`. The user-input docs keep only their own fact, that a stop leaves the operator's question open. - `ToolDescriptor` keeps its own accurate sentence on `survives_stop` instead of a link, because the types crate does not depend on the Plugin crate. It now says the run's end and the early end of the task that made the call still end a stop-surviving call. - `Plugin::call` gains a rule that a call must not wait for another call of the same run to finish. The Harness runs a bounded number of a run's calls at once and holds the rest, so such a wait can stall the run until a stop or a cancel, and the rule names no number. - `stop_round` now says a stop reaches the effects in flight or waiting when the loop sees it. The rest of that doc is only rewrapped. Plan: vibe/2026-10-11-7-tool-call-cap-debt.md --- .../runner/src/harness-control.rs | 8 +- crates/plugin-user-input/src/lib.rs | 7 +- .../types/src/tools/descriptor.rs | 5 +- crates/promptforge-plugin/src/lib.rs | 5 +- crates/promptforge-plugin/src/plugin.rs | 11 +- vibe/2026-10-11-7-tool-call-cap-debt.md | 369 ++++++++++++++++++ vibe/ACTIVE | 1 + 7 files changed, 393 insertions(+), 13 deletions(-) create mode 100644 vibe/2026-10-11-7-tool-call-cap-debt.md create mode 100644 vibe/ACTIVE diff --git a/crates/harness-internal/runner/src/harness-control.rs b/crates/harness-internal/runner/src/harness-control.rs index 33e5f87ab..6e5208b33 100644 --- a/crates/harness-internal/runner/src/harness-control.rs +++ b/crates/harness-internal/runner/src/harness-control.rs @@ -48,10 +48,10 @@ impl RunControl { /// A stop also drops the tool calls waiting for a slot, unstarted, /// except those that survive stops, which start once a slot frees. /// - /// A stop reaches only the effects in flight when the loop that drives - /// the run sees it. The loop looks for a stop while it waits for an - /// answer and before it starts a step's effects. It drops the effects - /// the stop reaches, if any, then lowers the stop. + /// A stop reaches only the effects in flight or waiting when the loop + /// that drives the run sees it. The loop looks for a stop while it + /// waits for an answer and before it starts a step's effects. It drops + /// the effects the stop reaches, if any, then lowers the stop. pub fn stop_round(&self) { self.stop.raise(); } diff --git a/crates/plugin-user-input/src/lib.rs b/crates/plugin-user-input/src/lib.rs index b8327ae56..d33b52dcf 100644 --- a/crates/plugin-user-input/src/lib.rs +++ b/crates/plugin-user-input/src/lib.rs @@ -30,7 +30,9 @@ //! The operator's answer is trusted, so the ask tool answers with //! [`ToolOutput::trusted`](promptforge_plugin::ToolOutput::trusted) and //! the text is never guard-wrapped. The ask tool's descriptor survives -//! stops: a stop leaves the question open, and only a cancel drops it. +//! stops: a stop leaves the question open. The other occasions +//! [`Plugin::call`](promptforge_plugin::Plugin::call) lists drop it, the +//! early end of the task that asked included. //! //! # A run without an operator //! @@ -70,7 +72,8 @@ //! inside the run's own future, so an [`InputBroker`] must not block //! while polled. //! - The ask tool's descriptor sets `survives_stop`, so a stop leaves the -//! operator's question open, and only a cancel drops it. +//! operator's question open; every other occasion `Plugin::call` lists +//! drops it. mod ask; diff --git a/crates/promptforge-internal/types/src/tools/descriptor.rs b/crates/promptforge-internal/types/src/tools/descriptor.rs index 052236ca7..8d775ea8c 100644 --- a/crates/promptforge-internal/types/src/tools/descriptor.rs +++ b/crates/promptforge-internal/types/src/tools/descriptor.rs @@ -33,8 +33,9 @@ pub struct ToolDescriptor { /// script receives it as a string. pub structured_output: bool, /// Whether a stop leaves the tool's calls in flight. A stop cancels - /// every other call in flight and lets the run go on; a cancel still - /// ends this one. A question to the operator sets it, so the question + /// every other call in flight and lets the run go on; the run's end + /// and the early end of the task that made the call still end this + /// one. A question to the operator sets it, so the question /// stays open. A descriptor serialized without the field reads as /// `false`. #[serde(default)] diff --git a/crates/promptforge-plugin/src/lib.rs b/crates/promptforge-plugin/src/lib.rs index 37e7e7f6b..1e7447557 100644 --- a/crates/promptforge-plugin/src/lib.rs +++ b/crates/promptforge-plugin/src/lib.rs @@ -51,9 +51,8 @@ //! - [`Plugin::call`] marks the trust of every output correctly: output //! that embeds data an attacker can influence is //! [`ToolOutput::untrusted`]. -//! - [`Plugin::call`] is cancellation-aware. Its future is dropped on a -//! stop or a cancel, except that a stop spares a call whose tool's -//! descriptor sets `survives_stop`. +//! - [`Plugin::call`] is cancellation-aware: its future may be dropped +//! before it finishes, on the occasions its invariants list. mod context; mod plugin; diff --git a/crates/promptforge-plugin/src/plugin.rs b/crates/promptforge-plugin/src/plugin.rs index 7ec614c10..95f010e48 100644 --- a/crates/promptforge-plugin/src/plugin.rs +++ b/crates/promptforge-plugin/src/plugin.rs @@ -82,8 +82,15 @@ pub trait Plugin: Send + Sync { /// embeds data an attacker can influence is /// [`ToolOutput::untrusted`]. /// - `call` is cancellation-aware. The Harness drops the returned - /// future on a stop or a cancel, unless the tool's descriptor sets - /// `survives_stop`, in which case only a cancel drops it. + /// future on a stop, when the run ends (a cancel included), and when + /// the task that made the call ends early: it is cancelled, its owner + /// ends, or an ancestor is aborted. Only a stop spares a call whose + /// tool's descriptor sets `survives_stop`. After a stop or a task's + /// early end, the run goes on. + /// - `call` must not wait for another call of the same run to finish. + /// The Harness runs a bounded number of a run's tool calls at once + /// and holds the rest, so a call that waits on one held behind it can + /// stall the run until a stop or a cancel. /// /// # Errors /// Returns a [`ToolError`], whose message is safe to show the model, diff --git a/vibe/2026-10-11-7-tool-call-cap-debt.md b/vibe/2026-10-11-7-tool-call-cap-debt.md new file mode 100644 index 000000000..b10c14e3d --- /dev/null +++ b/vibe/2026-10-11-7-tool-call-cap-debt.md @@ -0,0 +1,369 @@ +--- +name: Tool-call cap debt +overview: "Remove the one debt the tool-call-cap work added: the public Plugin contract still says only a cancel drops a stop-surviving call, but a withdrawal now drops it while the run goes on. Make Plugin::call the single statement of when the Harness drops a call, point the other public copies at it, and pin the changed occasion with one behavior test. Two doc ride-alongs join the same pass: a Plugin::call rule against waiting on another call of the same run (D1-14), and 'in flight or waiting' in RunControl::stop_round (D1-16)." +todos: + - id: d1-1-docs + content: "D1-1: make Plugin::call's # Invariants the full drop-occasion list; point the promptforge-plugin crate doc and plugin-user-input (lines 33, 72-73) at it; make ToolDescriptor::survives_stop's field doc accurate. D1-14: add the no-waiting-on-another-call bullet to the same # Invariants. D1-16: 'in flight or waiting' in RunControl::stop_round (harness-control.rs line 51)" + status: pending + - id: d1-1-pin + content: "D1-1: add cancelling_a_task_drops_its_question_to_the_operator to tests/it/harness-stop-cap.rs, holding Main's after round until the question is torn down" + status: pending + - id: verify + content: Focused stop_cap tests, contract read-through, git grep for 'only a cancel', docs builds, docs-claims, API check with CARGO_BUILD_WARNINGS cleared, and the full exit gates + status: pending +isProject: false +--- + +# Debt removal: the tool-call cap's drop contract + + + +## Product Requirements + +- Scope and target work: + - Repository: `c:\Users\Vinnie\cursor\promptforge`, branch `master`. Paths are relative to it, and line numbers were read at `9cdef9389`; search by name if they have moved. + - Target: the three commits whose trailer is `Plan: vibe/2026-10-11-6-tool-call-cap.md`: + - `1f7f88dd7` Report the tool calls each step withdraws + - `1b8184f7d` Run at most the cap of tool calls, oldest first + - `9cdef9389` Close plan: tool-call-cap + - Baseline `bc49d9670`, the parent of the oldest target commit. Endpoint and disposition `9cdef9389`, which `origin/master` also points at. The worktree was clean and excluded. + - The operator named no target. A repository-wide run would have covered no new work, because `origin/master` is at `HEAD`, so the target is the plan just executed. + - Removal scope: D1-1, plus D1-14 and D1-16. Analysis rejected those two, but the user brought them into scope as ride-alongs to D1-1's doc pass. +- Cleanup goals: + - Every public document that says when the Harness drops a Plugin call's future matches what the code does, so a Plugin author can rely on it. + - The full list of drop occasions lives in one place, `Plugin::call`'s `# Invariants`. The other public copies state only their own fact and point there, so the next drop occasion needs one edit. + - A behavior test pins the occasion that changed: the early end of a task drops its stop-surviving call while the run goes on. + - D1-14: a Plugin author learns from `Plugin::call`'s invariants that a call must not wait for another call of the same run to finish, since the Harness now holds calls beyond its cap. + - D1-16: a Host reads `RunControl::stop_round` without doubt about whether a stop reaches waiting calls. +- Non-goals: + - No change to Harness or Engine behavior. A withdrawal keeps dropping stop-surviving calls. + - No change to Stop, the cap, or what a stopped round returns. + - No edit to `../promptforge-docs`. + - No deadlock detection for calls that wait on each other (D1-14 is a documented rule, not a mechanism). + - The other rejected candidates stay as they are (Debt Inventory). +- Success criteria: + - `Plugin::call`'s invariant names every drop occasion and says only a stop spares a stop-surviving call. The occasions are: + - a stop; + - the run's end, a cancel included; + - the early end of the task that made the call: the task is cancelled, its owner ends, or an ancestor is aborted. + - No public doc says "only a cancel drops it", or lists stop and cancel as the only occasions. + - The new behavior test passes, and it would fail if a withdrawal spared stop-surviving calls. + - D1-14: `Plugin::call`'s `# Invariants` has a bullet saying a call must not wait for another call of the same run to finish, because the Harness runs a bounded number of a run's calls at once and holds the rest. + - D1-16: `RunControl::stop_round`'s doc says a stop reaches only the effects in flight or waiting when the loop sees it. + - Every exit gate in the Testing Plan passes. + +## Functional Specification + +### Debt Inventory + +- Debt added: + - D1-1, introduced by `1b8184f7d` on top of `1f7f88dd7`: a withdrawal drops a stop-surviving call while the run goes on, which contradicts the public Plugin contract. + - Evidence, at `9cdef9389`: + - `Reach::takes` (`crates/harness-internal/runner/src/effect_loop-flight.rs` lines 60-70) accepts any listed id for `Reach::Withdrawn` and ignores `survives_stop`. `Flights::abort` (lines 261-285) aborts each matching call in flight and drops each matching waiting call. + - The Driver calls `drop_in_flight(Reach::Withdrawn(&withdrawn))` on every step that is not decided (`effect_loop.rs` lines 208-212), so the run goes on after the drop. + - The Engine lists every tool call that `abort_effect` orphans (`crates/promptforge-internal/engine/src/execute/scheduler/chain.rs` lines 297-307). That covers `tasks.cancel` (`cancel_task` to `abort_subtree`, `waits.rs` line 343), an owner ending with live tasks (`settle_owned_tasks` to `abandon_owned_tasks` to `abort_subtree`, `task_end.rs` lines 94-109 and 189), and an ancestor's abort (`abort_subtree`, `chain.rs` line 258). Script `tools.call` and model batches share the one `Continuation::ToolCall`, so `input.ask()` is listed too. + - The ask tool sets `survives_stop(true)` (`crates/plugin-user-input/src/ask.rs` line 40). + - Unchanged contract text: + - `crates/promptforge-plugin/src/plugin.rs` lines 84-86: "The Harness drops the returned future on a stop or a cancel, unless the tool's descriptor sets `survives_stop`, in which case only a cancel drops it." + - `crates/plugin-user-input/src/lib.rs` line 33, and again at lines 72-73 under `## Invariants`: "only a cancel drops it." + - Relationship to target work: the design record required "A withdrawn call never starts, and one already running is dropped", but it never listed the Plugin-facing docs among the files to update, and neither target commit touched them. + - Impact: + - A Plugin built outside this repository that marks a tool `survives_stop` and trusts "only a cancel drops it" may treat the drop as the run's end. It could then report a cancelled run to a person, or release run-wide state that later calls still need. + - Nothing in the tree misbehaves today. Workshop's `WaitGuard` (`crates/workshop/agents/src/input-tool.rs` lines 20-47) closes each wait on drop, and the `InputBroker` invariant already requires that. + - Reversal cost: low. Documentation in four files plus one test, with no behavior change. + - Target state: the full list lives on `Plugin::call`, the other public copies point to it or state only their own fact, and a test pins the task's-early-end occasion for a stop-surviving call. +- Brought into scope by the user. Analysis rejected both, but each is a one-line doc edit in D1-1's pass: + - D1-14, classed weak or speculative: tool calls that wait on each other can stall a run under the cap. + - Evidence, at `9cdef9389`: before the target, every issued call ran at once. Now `Flights`' fill (`crates/harness-internal/runner/src/effect_loop-flight.rs` lines 161-174) runs at most the cap and holds the rest, oldest first. + - Consequence: suppose a call can finish only after a later, waiting call of the same run makes progress, and the cap's worth of earlier calls all block that way. Then the run waits until a stop or a cancel. + - Reach: no tool in the tree does this, but an MCP server with a producer and consumer tool pair could. + - Why a doc: `ToolPerformer` is internal (the `harness` facade doesn't re-export it), so Plugin authors can't see the cap. `Plugin::call`'s `# Invariants` is where they read the rules. + - Target state: one more bullet in that `# Invariants`. + - D1-16, classed residual but acceptable: "only the effects in flight" in `RunControl::stop_round`. + - Evidence: `crates/harness-internal/runner/src/harness-control.rs` lines 48-49, added by `1b8184f7d`, say a stop also drops waiting calls. Line 51, unchanged, says "A stop reaches only the effects in flight when the loop that drives the run sees it". The same commit made "in flight" and "waiting" separate states. + - Target state: "only the effects in flight or waiting". The `harness` facade re-exports `RunControl`, so Hosts read this. +- Cheap fixes: none. +- Exposed pre-existing debt: none. + - D1-19 is unrelated pre-existing debt: a run that ends without a cancel (a fatal answer, a stall, or a root that returns while a task waits) already dropped stop-surviving calls before the target. + - D1-1 needs accurate wording anyway, and that wording covers D1-19 at no extra cost, so D1-19 gets no work item of its own. +- Rejected candidates: 18 more, beyond the two above that the user brought into scope. + - Residual but acceptable, 8: + - D1-2: the `withdrawn` field on the exhaustive `Step::Pending` variant. Its shape predates the target, and the user accepted public format changes. + - D1-3: `EffectId`'s documented order, which codifies what the allocator already does. + - D1-7: callers must handle `withdrawn` after `effects`. The order is documented and pinned on both sides. + - D1-8: `Reach::Withdrawn` depends on a sorted list. The sort is documented and tested, and a miss would be harmless. + - D1-13: stop-surviving calls count toward the cap. That's a chosen, tested trade-off. + - D1-15: the task-cancel test is weaker than its name, but the same-step test catches the realistic regression. + - D1-17: `effect_loop.rs` is at 499 lines. That's a size lead with no behavior consequence. + - X1: three more copies of the occasion list are incomplete but say no "only". The public one, on `ToolDescriptor::survives_stop`, is updated in D1-1's pass anyway. + - Weak or speculative, 2: + - D1-6: two booleans on private `Flights::launch`, with two call sites and no bug. + - D1-18: the default of 10 restated in prose and tests. They all agree today. + - False, 7: + - D1-4: `max_in_flight` is public only inside an internal crate. + - D1-5: `Reach` is a private enum with one match. + - D1-9: the Engine's withdrawn list is complete, has no repeats, and is empty once the run is decided. + - D1-10: there's no stall or stranded waiting call, because waiting implies the cap's worth of calls are running. + - D1-11: there's no lost wakeup for a call started inside `landed_now`. + - D1-12: no performer future makes progress beyond the cap. + - X2: nothing in the tree reads a dropped ask as a run-level event. + - Unrelated pre-existing, 1: D1-19 (above). + + + + +## Technical Design + +Documentation in five files plus one behavior test; no Harness, Engine, or Plugin behavior changes. The drop-occasion list moves to a single home, because these copies were written for two occasions (stop and cancel) and fell behind as occasions grew: a decided run without a cancel, waiting calls, and now a withdrawal. + +- [crates/promptforge-plugin/src/plugin.rs](crates/promptforge-plugin/src/plugin.rs), the `# Invariants` bullet on `Plugin::call` (lines 84-86), becomes the full list. Suggested wording: + +```text +- `call` is cancellation-aware. The Harness drops the returned future on + a stop, when the run ends (a cancel included), and when the task that + made the call ends early: it is cancelled, its owner ends, or an + ancestor is aborted. Only a stop spares a call whose tool's descriptor + sets `survives_stop`. After a stop or a task's early end, the run goes + on. +``` + + The same `# Invariants` gains one bullet for D1-14. It names no number, because the cap is the Harness's to set. Suggested wording: + +```text +- `call` must not wait for another call of the same run to finish. The + Harness runs a bounded number of a run's tool calls at once and holds + the rest, so a call that waits on one held behind it can stall the run + until a stop or a cancel. +``` + +- [crates/harness-internal/runner/src/harness-control.rs](crates/harness-internal/runner/src/harness-control.rs), for D1-16: in `RunControl::stop_round`'s doc, line 51 "A stop reaches only the effects in flight when the loop that drives the run sees it" becomes "A stop reaches only the effects in flight or waiting when the loop that drives the run sees it". Nothing else in that doc changes. +- [crates/promptforge-plugin/src/lib.rs](crates/promptforge-plugin/src/lib.rs), the crate-level bullet (lines 54-56), stops restating the list, for example: "[`Plugin::call`] is cancellation-aware: its future may be dropped before it finishes, on the occasions its invariants list." +- [crates/plugin-user-input/src/lib.rs](crates/plugin-user-input/src/lib.rs): + - The overview sentence (line 33) becomes, for example: "The ask tool's descriptor survives stops: a stop leaves the question open. The other occasions [`Plugin::call`](promptforge_plugin::Plugin::call) lists drop it, the early end of the task that asked included." + - The `## Invariants` bullet (lines 72-73) becomes, for example: "The ask tool's descriptor sets `survives_stop`, so a stop leaves the operator's question open; every other occasion `Plugin::call` lists drops it." +- [crates/promptforge-internal/types/src/tools/descriptor.rs](crates/promptforge-internal/types/src/tools/descriptor.rs), the `survives_stop` field doc (lines 35-39), keeps its own sentence, made accurate: + - Suggested: "A stop cancels every other call in flight and lets the run go on; the run's end and the early end of the task that made the call still end this one." + - It can't link to `Plugin::call`, because `promptforge-types` doesn't depend on `promptforge-plugin`. + - It is an Engine crate doc, so it names neither the Host nor how the Harness works (root `AGENTS.md`). +- [crates/harness-internal/runner/tests/it/harness-stop-cap.rs](crates/harness-internal/runner/tests/it/harness-stop-cap.rs) (177 lines) gains `cancelling_a_task_drops_its_question_to_the_operator`, through `Harness::run`: + - Host and broker: + - The Host is `with_asker(hold_host(&Arc::default()))`, with `operator.services()` from an `Operator` (`tests/it/scripted.rs`), a `PendingTimer`, and a `ScriptedBroker`. + - The file's existing `calls_once` and `settle` helpers show the shape. `run_beside`, `until`, `answers_where`, and `calls` come from the parent `tests/it/harness.rs`. + - Prompt: + - The frontmatter declares `plugins: - user-input` and a `writer` model, with `models.default('writer')` in the H1 block. + - `## Main` runs `local t = tasks.spawn('## Child')`, `models.infer('wait')`, `tasks.cancel(t)`, and `models.infer('after')`, then returns `'cancelled'`. + - `## Child` runs `return (input.ask())`. + - The broker holds back two of Main's rounds: + - It answers the `wait` round only once the operator has been asked (`operator.asked == 1`). + - It answers the `after` round only once the question has been torn down (`operator.abandoned == 1`). + - Expect: + - `RunOutcome::Completed { final_text: "cancelled" }`; + - `operator.abandoned == 1`; + - the ask's `ToolCall` answer recorded `Dropped` (`answers_where(&records, "ToolCall", calls(ASK_TOOL))`). + - Why it pins the contract: + - If a withdrawal spared stop-surviving calls, the question would stay open, and Main's `after` round would never be answered. The run would then never end, and `run_beside`'s 10-second `PATIENCE` would fail the test. + - Holding `after` back is what makes the test meaningful. Without it, the run's end would drop the question through the decided branch's `Reach::All` and hide a regression. + - The test passes at `9cdef9389` by design, because it pins the behavior being documented rather than a bug. + + + + +## Testing Plan + +- Focused, D1-1: `cargo nextest run --locked -p harness-runner --all-features harness::stop_cap::`. All three tests pass, the new one included. +- Contract check, D1-1: + - Read the new `Plugin::call` bullet against `Reach::takes` (`effect_loop-flight.rs` lines 60-70), the Driver's withdrawal block (`effect_loop.rs` lines 208-212), and the decided branch's `drop_in_flight(Reach::All)` (`effect_loop.rs` around line 189). + - Each occasion the bullet names must have a code path, and no code path may be left out. + - `git grep -n "only a cancel" -- . ':(exclude)vibe'` returns nothing (exit code 1). It excludes `vibe/` because dated plan records, this plan's own copy included, quote the old text. +- Contract check, D1-14: read the new bullet against fill (`effect_loop-flight.rs` lines 161-174). The Harness holds calls beyond the cap and starts one only when a running call lands or is dropped, so a call that blocks on a held one keeps its slot. +- Contract check, D1-16: read the edited sentence against `act_on_stop` and `abort(Reach::AllButSurvivors)`. A stop seen by the loop takes the waiting calls that don't survive stops as well as those in flight, and nothing started after the loop saw it. +- Docs and wording, D1-1: + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api`. Rustdoc's `broken_intra_doc_links` denies, so this checks the new `Plugin::call` links. + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc -p promptforge --no-deps`. + - `node --test crates/workshop/ui/test/docs-claims.mjs`, which scans `## Invariants` wording and the Engine wording rule. + - `Remove-Item Env:CARGO_BUILD_WARNINGS -ErrorAction SilentlyContinue; cargo +nightly-2026-09-05 xtask api --check`, because `ToolDescriptor` is facade surface. Clear the variable first: if `CARGO_BUILD_WARNINGS='deny'` is still set from a clippy run, nightly cargo's `workspace-hack` manifest warnings become fatal. +- Regression, unchanged behavior: every existing test passes untouched. That includes `tests/it/effect_loop-cap.rs`, `tests/it/harness-stop.rs`, `crates/plugin-user-input/tests/it/ask.rs`, and the Workshop `server/tests/it/agents/turns.rs` ("a stop cancels no question"). +- Exit gates: + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` + - The Workshop crates, with the gateway sidecar staged: + - Stage it: `cargo build --locked -p gateway --no-default-features`, then `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe`. + - Run `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` and `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`. + - Always run `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc` afterward. + - `cargo fmt --all --check` + - `cargo test -p build-xtask` + - The docs, wording, and API checks above. + + + + +## Decision Record + +- Reversible decisions and consequences: + - Fix the documents, not the behavior. The tool-call-cap design record already decided that "A withdrawn call never starts, and one already running is dropped". The documents are catching up, and undoing the edit would be another doc edit, so this is not treated as a hard-to-reverse public-interface change. + - Give the occasion list one home, `Plugin::call`'s `# Invariants`. + - The `promptforge-plugin` crate doc and `plugin-user-input` keep only their own fact (a stop leaves the question open) and point to `Plugin::call`. + - The consequence is that one edit covers the next drop occasion. + - The record shows the cost of the old way: D1-1, D1-19, and X1 are three copies that fell behind the same list. + - `ToolDescriptor::survives_stop` keeps its own accurate sentence, because `promptforge-types` can't depend on `promptforge-plugin`. + - Pin the changed occasion with a behavior test, in `harness-stop-cap.rs` (177 lines) rather than `effect_loop-cap.rs` (461 lines). It uses the real fixture ask tool, so it pins exactly what the operator-question docs promise. + - D1-19's case is covered by the accurate wording, with no work item of its own. + - D1-14 is a documented rule, not a Harness mechanism. No tool in the tree waits on another call of the same run, and the Harness can't see what a call waits on. +- User-resolved choices: + - No architecture choices were needed. The behavior was settled in the tool-call-cap plan. + - The user brought D1-14 and D1-16 into scope as ride-alongs to D1-1's doc pass, after analysis had rejected them. +- Rejected alternatives: + - Make `Reach::Withdrawn` spare stop-surviving calls, so the old "only a cancel drops it" stays true. + - A cancelled task's question would then stay open with nobody to use the answer, and it would hold a slot under the cap. + - It would also contradict the tool-call-cap design record. + - Restate the full list in every copy. Reason: that's the drift that produced this finding. + - Leave the documents as they are. Reason: the public contract would stay contradicted. + - Ride-alongs left out: the internal `ToolPerformer::survives_stop` doc and the `InputBroker` example sentence. Neither says "only", and neither misleads. + - Detecting or breaking waits between calls of one run inside the Harness (D1-14). Reason: no tool in the tree does it, and the Harness can't see what a call waits on. +- Assumptions and risks: + - The impact on Plugins outside this repository is inferred, not observed. The user said earlier that no one depends on the public formats yet. + - The `../promptforge-docs` book was not examined for further copies of the contract. + - Two rejections, D1-11 and D1-12, rest on `futures-util`'s documented `FuturesUnordered` behavior. The pinned source was not re-read. + +### Deferred and Out of Scope + +- Copies of the drop contract in `../promptforge-docs`. Revisit at the next docs sync. +- D1-15, a stronger task-cancel test in `effect_loop-cap.rs`, where the main chain keeps working after the cancel so the run's end can't hide a missed drop. Revisit the next time that file changes. +- D1-17, splitting `crates/harness-internal/runner/src/effect_loop.rs` (499 lines) before its next change. Revisit with the next Driver change. +- D1-2, `#[non_exhaustive]` on `Step::Pending`. That choice predates the target. Revisit before the facade's first stable release. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build --locked` builds only the gateway (the workspace default member); the desktop app is `cargo build --locked -p workshop`. Crate build scripts bundle the TypeScript UIs, so a fresh checkout first needs `npm ci --prefix crates/workshop` and `npm ci --prefix crates/gateway/config-ui/ui`. +- Focused test command pattern: `cargo nextest run --locked -p --all-features `; drop `--all-features` for `workshop`, `workshop-server`, and `workshop-server-api`. A single JS test file: `node --test `. +- Component test command pattern: `cargo nextest run --locked -p --all-features`, with the same `--all-features` exception for the three workshop crates. Workshop UI packages: `npm test --workspaces --if-present` from `crates/workshop`; gateway config UI: `npm test` from `crates/gateway/config-ui/ui`. +- Full-suite test command: `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features`, then `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api`, then `node --test crates/workshop/ui/test/docs-claims.mjs`. The workspace run includes `build-xtask`, which holds the boundary and structural checks. +- Linter command: `CARGO_BUILD_WARNINGS=deny cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features`, then `CARGO_BUILD_WARNINGS=deny cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets`, plus `cargo check -p gateway --no-default-features` for the headless build shape. On PowerShell set `$env:CARGO_BUILD_WARNINGS='deny'` first. Never run a standalone `cargo check --workspace`. +- Formatter check command: `cargo fmt --all --check` (rustfmt `style_edition = "2024"`). +- Docs command: `RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api`, plus `RUSTDOCFLAGS="-D warnings" cargo doc -p promptforge --no-deps` for the facade with default features. When the `promptforge` facade's public surface changes, also run `cargo + xtask api --check`, which compares against `crates/promptforge/public-api.txt`. +- Test placement and naming conventions: + - Unit tests live in a sibling file wired from the module as `#[cfg(test)] #[path = "-tests.rs"] mod tests;` (for example `effect.rs` beside `effect-tests.rs`). A module with three or more test files uses a `tests/` subdirectory instead (for example `promptforge-internal/vfs/src/handle/tests/`). Some older files keep an inline `mod tests { }`. + - Integration tests compile as one binary per crate: `tests/suite/main.rs` for the `promptforge` and `harness` facades, `tests/it/main.rs` for the gateway app, workshop server, and `plugin-mcp`. Each topic is a `mod` file; shared helpers sit in `support.rs` or `common/`. Prompt fixtures live under `tests/prompts/` (`valid`, `invalid`, `execution`). + - Test functions are snake_case sentences stating the behavior, such as `a_run_cancelled_before_it_begins_ends_cancelled_and_is_never_recorded`; a `test_` prefix is rare. + - Test-only hooks sit behind a `test-fixtures` Cargo feature (gateway and workshop crates) or `test-support` (Engine crates and `promptforge-plugin`); `build-xtask` checks they do not leak into normal builds. + - JS tests are `*.test.mjs` beside the source or `*.mjs` under `test/`, run by `node --test`. +- Directory map: + - `crates/`: all code. Top-level public crates (`promptforge`, `promptforge-plugin`, `harness`, `harness-gateway-client`, `gateway-api-types`, `gateway-api-discovery`), Plugins (`plugin-web`, `plugin-mcp`, `plugin-user-input`), shared utilities (`shared-error-source`, `shared-loopback`, and `shared-ui`, a TypeScript and CSS package rather than a crate), tooling (`build-xtask` for structural checks and `cargo xtask`, `build-workshop` behind the `cargo workshop` alias, `build-ui`, `build-user-guide`, `build-ceiling`, `build-llama-cuda`), and `workspace-hack` for cargo-hakari. + - `crates/promptforge-internal/`: private Engine crates `types`, `parser`, `lua`, `engine`, `vfs`, `model-client`. + - `crates/harness-internal/runner`: the private Harness crate `harness-runner`. + - `crates/gateway/`: private gateway crates; `app` is the `promptforge-gateway` binary, beside `config`, `config-ui`, `local`, `logging`, `progress`, `protocol`, `routing`, `cloud-providers`, `web-search`, and the `stt/` speech subsystem whose public member is `api` (`gateway-stt`). + - `crates/workshop/`: the Workshop Host; `desktop` (Tauri app, package `workshop`), `server` and `server-api`, tier crates `agents`, `gateway`, `menu`, `protocol`, `registry`, `run-log`, `status`, `support`, `user-state`, `workspace`, and the npm workspaces `ui`, `look`, `platform`. + - `guide/`: user guide and docs site sources. `prompts/`: example prompts. `tools/`: Node scripts for gateway sidecar staging and live TTS checks, each with a `.test.mjs`. `vibe/`: dated plan and run records. `local/`: gitignored machine-local gateway config and fixtures. + - `.github/workflows/`: CI, release, and nightly jobs. `.githooks/`: pre-commit runs fmt; pre-push runs docs-claims, the headless check, workspace clippy, and cargo-deny. `.config/`: nextest and hakari config. `.cursor/rules/`: Workshop architecture and SPA rules. +- Component boundaries (enforced by `cargo test -p build-xtask`): + - Engine crates (`promptforge*`) depend on no gateway, workshop, Harness, or Plugin crate. Other families reach the Engine only through `promptforge` and `promptforge-plugin`; `crates/promptforge-internal/` is private to those two. + - Harness crates (`harness*`) depend outside their family only on `promptforge`, `promptforge-plugin`, and `workspace-hack`. Others reach the Harness only through `harness` and `harness-gateway-client`; `harness-runner` is private to `harness`. + - Plugin crates depend only on `promptforge-plugin`, `shared-*`, and outside libraries. The only product crate allowed to depend on a Plugin is `harness-gateway-client`, which implements `plugin-web`'s `SearchProvider`. + - Gateway crates depend on no promptforge, workshop, or Harness crate. Their container crates are private to the family, and Workshop sees the gateway only through `gateway-api-types` and `gateway-api-discovery`. + - Workshop tiers flow one way: server, then features, then services, then vocabulary. Same-tier crates do not depend on each other, except `workshop-registry` on `workshop-protocol`. Each workshop crate root names its tier. The desktop app depends only on `workshop-server-api`, which depends only on `workshop-server`. + - `shared-*` crates depend on no product crate. +- Conventions summary: + - Engine, Harness, Host, and Plugin are capitalized terms with exactly one meaning each (AGENTS.md Definitions). Engine crates say "the caller" and never mention the Host. `docs-claims.mjs` enforces the wording in every AGENTS.md, `## Invariants` crate doc, and `.cursor/rules` file. + - Every member inherits strict workspace lints: clippy `all` and `pedantic` at deny, `unwrap_used` and `expect_used` denied, `#[allow]` banned in favor of `#[expect(..., reason = "...")]`, `unsafe_code` denied, `missing_docs` warned. `clippy.toml` bans installing process-global state (panic hook, logger, tracing subscriber, crypto provider) outside binary entry points. + - Crate roots open with a `//!` doc, many with a `## Invariants` section. + - Source directories are flat: a subdirectory needs three or more files, otherwise use `parent-label.rs` siblings wired with `#[path]`; convert in either direction when touching a group on the wrong side of the line. + - Dependencies are declared once in `[workspace.dependencies]`, with comments explaining each pin and disabled feature. + - Error and status messages are written for model consumption: concise, naming what is missing with required versus actual. Comments explain only non-obvious constraints, and platform workarounds cite the upstream issue URL. + - Behavior changes ship with tests in the same change. JSON that reaches a recorder must round-trip exactly (`serde_json` with `float_roundtrip`, sorted keys, finite numbers). + - Long-running gateway work reports through `gateway-progress`. Workshop UI CSS uses only `@workshop/look` tokens, and persisted UI state goes through `ui-storage` to `ui-state.json` or the workspace file. + + + + +## Execution Instructions + + + +### Step 1: Make Plugin::call the one statement of when a call is dropped [completed] + +- Component: `none` +- Tests: + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc -p promptforge --no-deps` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + - `Remove-Item Env:CARGO_BUILD_WARNINGS -ErrorAction SilentlyContinue; cargo +nightly-2026-09-05 xtask api --check` + - `git grep -n "only a cancel" -- . ':(exclude)vibe'`, which must print nothing (exit code 1) + +This step changes documentation only. It covers D1-1's contract text plus the two ride-alongs, D1-14 and D1-16. No Harness, Engine, or Plugin behavior changes: a withdrawal keeps dropping stop-surviving calls, and Stop, the cap, and what a stopped round returns stay as they are. Don't edit `../promptforge-docs`, and don't make `Reach::Withdrawn` spare stop-surviving calls. Line numbers are at `9cdef9389`; search by name if they have moved. + +- `crates/promptforge-plugin/src/plugin.rs`, `Plugin::call`'s `# Invariants`: + - Replace the cancellation-aware bullet (lines 84-86, ending "in which case only a cancel drops it") with the full drop-occasion list, using the Technical Design's suggested wording. It must name every occasion: a stop; the run's end, a cancel included; and the early end of the task that made the call (the task is cancelled, its owner ends, or an ancestor is aborted). It must say that only a stop spares a call whose tool's descriptor sets `survives_stop`, and that the run goes on after a stop or a task's early end. + - Add a D1-14 bullet to the same `# Invariants`: a call must not wait for another call of the same run to finish. The reason is that the Harness runs a bounded number of a run's tool calls at once and holds the rest, so a call that waits on one held behind it can stall the run until a stop or a cancel. Name no number, because the cap is the Harness's to set. +- `crates/promptforge-plugin/src/lib.rs`, the crate-level "[`Plugin::call`] is cancellation-aware" bullet (lines 54-56): stop restating the occasions and point at `Plugin::call`'s invariants, as the Technical Design suggests. +- `crates/plugin-user-input/src/lib.rs`: + - The overview sentence (line 33): keep only this crate's own fact, that a stop leaves the question open, and say every other occasion [`Plugin::call`](promptforge_plugin::Plugin::call) lists drops it, the early end of the task that asked included. + - The `## Invariants` bullet (lines 72-73): the ask tool's descriptor sets `survives_stop`, so a stop leaves the operator's question open; every other occasion `Plugin::call` lists drops it. +- `crates/promptforge-internal/types/src/tools/descriptor.rs`, the `survives_stop` field doc (lines 35-39): replace "a cancel still ends this one" with an accurate sentence of its own, for example "A stop cancels every other call in flight and lets the run go on; the run's end and the early end of the task that made the call still end this one." Don't link to `Plugin::call`, because `promptforge-types` doesn't depend on `promptforge-plugin`. This is an Engine crate doc, so it names neither the Host nor how the Harness works. Leave the `survives_stop` builder method's doc (lines 69-70) as it is. +- `crates/harness-internal/runner/src/harness-control.rs`, `RunControl::stop_round`'s doc, line 51: "A stop reaches only the effects in flight when the loop that drives the run sees it" becomes "A stop reaches only the effects in flight or waiting when the loop that drives the run sees it". Nothing else in that doc changes. +- Leave the internal `ToolPerformer::survives_stop` doc (`crates/harness-internal/runner/src/performers.rs`) and the `InputBroker` example sentence unchanged. Neither says "only", and neither misleads. +- Every `.rs` file stays at or under 500 lines; each crate's build script fails a file over 500. + +Acceptance: + +- No public doc says "only a cancel drops it", or lists a stop and a cancel as the only occasions. The `git grep` excludes `vibe/` because dated plan records, this plan's seed copy included, quote the old text. +- D1-1 contract check: read the new `Plugin::call` bullet against `Reach::takes` (`crates/harness-internal/runner/src/effect_loop-flight.rs` lines 60-70), the Driver's withdrawal block (`effect_loop.rs` lines 208-212), and the decided branch's `drop_in_flight(Reach::All)` (`effect_loop.rs` around line 189). Each occasion the bullet names has a code path, and no code path is left out. +- D1-14 contract check: read the new bullet against fill (`effect_loop-flight.rs` lines 161-174). The Harness holds calls beyond the cap and starts one only when a running call lands or is dropped, so a call that blocks on a held one keeps its slot. +- D1-16 contract check: read the edited sentence against `act_on_stop` and `abort(Reach::AllButSurvivors)`. A stop seen by the loop takes the waiting calls that don't survive stops as well as those in flight, and nothing that started after the loop saw it. +- Rustdoc's `broken_intra_doc_links` denies under `-D warnings`, so the first docs build checks the new `Plugin::call` links. `docs-claims.mjs` scans `## Invariants` wording and the Engine wording rule. The API check runs because `ToolDescriptor` is facade surface. Clear `CARGO_BUILD_WARNINGS` first, as written, because a leftover `deny` makes nightly cargo's `workspace-hack` manifest warnings fatal. + + + + + +### Step 2: Pin that cancelling a task drops its question to the operator + +- Component: `none` +- Tests: + - `cargo nextest run --locked -p harness-runner --all-features harness::stop_cap::` + - `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-features` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy --workspace --exclude workshop --exclude workshop-server --exclude workshop-server-api --all-targets --all-features` + - `cargo build --locked -p gateway --no-default-features` + - `node tools/stage-gateway-sidecar.mjs stage --target x86_64-pc-windows-msvc --source target/debug/promptforge-gateway.exe` + - `cargo nextest run --locked -p workshop -p workshop-server -p workshop-server-api` + - `$env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p workshop -p workshop-server -p workshop-server-api --all-targets` + - `node tools/stage-gateway-sidecar.mjs remove --target x86_64-pc-windows-msvc`, always, even when a Workshop command fails + - `cargo fmt --all --check` + - `cargo test -p build-xtask` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server --exclude workshop-server-api` + - `$env:RUSTDOCFLAGS='-D warnings'; cargo doc -p promptforge --no-deps` + - `node --test crates/workshop/ui/test/docs-claims.mjs` + - `Remove-Item Env:CARGO_BUILD_WARNINGS -ErrorAction SilentlyContinue; cargo +nightly-2026-09-05 xtask api --check` + - `git grep -n "only a cancel" -- . ':(exclude)vibe'`, which must print nothing (exit code 1) + +The first command is this step's focused test: all three `stop_cap` tests pass, the new one included. The rest are the plan's exit gates, run here because both steps are in by now. + +Add `#[tokio::test] async fn cancelling_a_task_drops_its_question_to_the_operator` to `crates/harness-internal/runner/tests/it/harness-stop-cap.rs` (177 lines). It drives the run through `Harness::run` with `run_beside`, as the Technical Design describes. The test passes at `9cdef9389` by design, because it pins the behavior Step 1 documents rather than a bug. No production code changes. In place of a failing-first run, check that it fails when `Reach::takes` (`crates/harness-internal/runner/src/effect_loop-flight.rs` lines 60-70) spares stop-surviving calls for `Reach::Withdrawn`, then restore that file exactly. + +- Host and broker: + - Write `let (operator, _answers) = Operator::new();` and keep the sender bound for the whole test. If it is dropped, `Operator::wait` fails with "the operator left" instead of being torn down, and `abandoned` stays 0. + - Build `Harness::new(recorder.clone(), Arc::new(broker), Arc::new(PendingTimer::default()), Arc::new(with_asker(hold_host(&Arc::default()))), operator.services())` with a `MemoryRecorder`. + - Follow the shape of the file's existing tests and helpers (`calls_once`, `settle`). `run_beside`, `until`, `answers_where`, and `calls` come from the parent `tests/it/harness.rs`. `Operator`, `ScriptedBroker`, `hold_host`, `with_asker`, and `reply` come from `crate::scripted`. +- Prompt: the frontmatter declares `plugins: - user-input` and a `writer` model, with `models.default('writer')` in the H1 block. + - `## Main` runs `local t = tasks.spawn('## Child')`, `models.infer('wait')`, `tasks.cancel(t)`, and `models.infer('after')`, then returns `'cancelled'`. + - `## Child` runs `return (input.ask())`. +- The `ScriptedBroker` tells Main's two rounds apart by the last message's content, and holds both back: + - It answers the `wait` round only once the operator has been asked (`operator.asked` reaches 1). + - It answers the `after` round only once the question has been torn down (`operator.abandoned` reaches 1). +- Expect: + - `report.outcome` is `RunOutcome::Completed { final_text: "cancelled" }`. + - `operator.abandoned` is 1. + - The ask's `ToolCall` answer is recorded `Dropped`: `answers_where(&records, "ToolCall", calls(ASK_TOOL))` equals `vec![json!("Dropped")]`. +- Why it pins the contract, so the hold-backs must not be weakened: + - If a withdrawal spared stop-surviving calls, the question would stay open and Main's `after` round would never be answered. The run would never end, and `run_beside`'s 10-second `PATIENCE` would fail the test. + - Holding `after` back is what makes the test meaningful. Without it, the run's end would drop the question through the decided branch's `Reach::All` and hide a regression. +- Update the file's `//!` doc (lines 1-3) and the `stop_cap` clause of the parent module doc in `tests/it/harness.rs` (lines 7-8) so they also name this case: a task's early end drops its stop-surviving call while the run goes on. +- Leave every existing test untouched. They must still pass, including `tests/it/effect_loop-cap.rs`, `tests/it/harness-stop.rs`, `crates/plugin-user-input/tests/it/ask.rs`, and the Workshop `server/tests/it/agents/turns.rs` ("a stop cancels no question"). +- Every `.rs` file stays at or under 500 lines; each crate's build script fails a file over 500. + + + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..58eb91794 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-10-11-7-tool-call-cap-debt.md From 1fff0b71be34469dc8b2683e3e2192c468403f11 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 22:42:29 -0700 Subject: [PATCH 33/34] Pin that cancelling a task drops its operator question Add a test that cancels a task while it waits on a question to the operator. It checks that the question is torn down and answered as dropped while the run goes on to complete. The ask tool survives stops, so this pins that a task's early end still drops such a call. The scripted model holds back the round after the cancel until the question is gone, so a regression that left the question open stalls the run and fails the test on timeout. The module docs now name this case beside the stop cases. - `holds_back` tells Main's rounds apart by the last message's text, answering the `wait` round only once `asked` is nonzero and every other round only once `abandoned` is nonzero. It spins on `yield_now` without bound, so a question left open fails the test through `run_beside`'s patience. - `cancelling_a_task_drops_its_question_to_the_operator` runs `CANCELS_AN_ASKER`, where Main spawns Child to ask the operator, waits on a `wait` round, cancels Child, and then waits on an `after` round. It asserts the run completes with `cancelled`, `operator.abandoned` is 1, and the ask's one `ToolCall` answer is recorded as `Dropped`. - `_answers` holds the answer sender for the whole test, but nothing is ever sent on it, so the question is never answered. Design: new shared-mutable-state @ crates/harness-internal/runner/tests/it/harness-stop-cap.rs::holds_back deps: Operator Plan: vibe/2026-10-11-7-tool-call-cap-debt.md --- .../runner/tests/it/harness-stop-cap.rs | 75 ++++++++++++++++++- .../runner/tests/it/harness.rs | 7 +- vibe/2026-10-11-7-tool-call-cap-debt.md | 2 +- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/crates/harness-internal/runner/tests/it/harness-stop-cap.rs b/crates/harness-internal/runner/tests/it/harness-stop-cap.rs index 21f9546ae..c195a069a 100644 --- a/crates/harness-internal/runner/tests/it/harness-stop-cap.rs +++ b/crates/harness-internal/runner/tests/it/harness-stop-cap.rs @@ -1,6 +1,8 @@ //! A stop under the real performer's default cap of 10: the stop drops //! the calls waiting for a slot without starting them, and a waiting call -//! that survives stops starts once the stop frees a slot. +//! that survives stops starts once the stop frees a slot. A task's early +//! end spares no call that survives stops: cancelling a task drops its +//! question to the operator while the run goes on. use std::fmt::Write as _; use std::sync::Arc; @@ -72,6 +74,41 @@ async fn settle() { } } +/// Main spawns Child, which asks the operator; Main then waits on a +/// `wait` round, cancels Child, and waits on an `after` round. +const CANCELS_AN_ASKER: &str = "---\nname: cancels\ndescription: d\npromptforge: 0\n\ + plugins:\n - user-input\nmodels:\n writer: {}\n---\n\n\ + # Cancels\n\n```lua\nmodels.default('writer')\n```\n\n\ + ## Main\n\n```lua\n\ + local t = tasks.spawn('## Child')\n\ + models.infer('wait')\n\ + tasks.cancel(t)\n\ + models.infer('after')\n\ + return 'cancelled'\n```\n\n\ + ## Child\n\n```lua\nreturn (input.ask())\n```\n"; + +/// A broker that answers the `wait` round once `operator` has been asked, +/// and every other round once the question has been torn down. It waits +/// without bound, so a question left open fails the test through +/// `run_beside`'s patience. +fn holds_back(operator: &Operator) -> ScriptedBroker { + let asked = Arc::clone(&operator.asked); + let abandoned = Arc::clone(&operator.abandoned); + ScriptedBroker::new(move |_round, messages| { + let text = messages + .last() + .map(|message| message.content().to_owned()) + .unwrap_or_default(); + let gate = Arc::clone(if text == "wait" { &asked } else { &abandoned }); + Box::pin(async move { + while gate.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + reply(format!("re: {text}")) + }) + }) +} + #[tokio::test] async fn a_stop_drops_waiting_calls_without_starting_them() { let held = Arc::new(Held::default()); @@ -175,3 +212,39 @@ async fn a_stop_starts_a_waiting_call_that_survives_stops() { "the question survived the stop and was answered" ); } + +#[tokio::test] +async fn cancelling_a_task_drops_its_question_to_the_operator() { + let (operator, _answers) = Operator::new(); + let recorder = Arc::new(MemoryRecorder::new()); + let harness = Harness::new( + recorder.clone(), + Arc::new(holds_back(&operator)), + Arc::new(PendingTimer::default()), + Arc::new(with_asker(hold_host(&Arc::default()))), + operator.services(), + ); + + let report = run_beside(harness, request(CANCELS_AN_ASKER), |_control| async {}) + .await + .expect("the run reaches an outcome"); + + assert_eq!( + report.outcome, + RunOutcome::Completed { + final_text: "cancelled".to_owned() + }, + "the run went on past the cancelled task" + ); + assert_eq!( + operator.abandoned.load(Ordering::SeqCst), + 1, + "cancelling the task tore its question down" + ); + let records = recorder.records(report.run_id.expect("the run began")); + assert_eq!( + answers_where(&records, "ToolCall", calls(ASK_TOOL)), + vec![json!("Dropped")], + "the question that survives stops was answered Dropped" + ); +} diff --git a/crates/harness-internal/runner/tests/it/harness.rs b/crates/harness-internal/runner/tests/it/harness.rs index 1a0dcd6ae..6aa860b36 100644 --- a/crates/harness-internal/runner/tests/it/harness.rs +++ b/crates/harness-internal/runner/tests/it/harness.rs @@ -5,9 +5,10 @@ //! and one run completes under a thread-parking executor with no runtime //! behind it. How a stop and a cancel reach the effects in flight sits in //! the `stop` child module, how a stop reaches the tool calls waiting for -//! a slot in the `stop_cap` child module, how what a stop reaches depends -//! on when the loop sees it in the `stop_timing` child module, and every -//! error `run` returns in the `errors` child module. +//! a slot and how a task's early end drops its stop-surviving call while +//! the run goes on in the `stop_cap` child module, how what a stop +//! reaches depends on when the loop sees it in the `stop_timing` child +//! module, and every error `run` returns in the `errors` child module. use std::future::Future; use std::pin::Pin; diff --git a/vibe/2026-10-11-7-tool-call-cap-debt.md b/vibe/2026-10-11-7-tool-call-cap-debt.md index b10c14e3d..c0b070851 100644 --- a/vibe/2026-10-11-7-tool-call-cap-debt.md +++ b/vibe/2026-10-11-7-tool-call-cap-debt.md @@ -319,7 +319,7 @@ Acceptance: -### Step 2: Pin that cancelling a task drops its question to the operator +### Step 2: Pin that cancelling a task drops its question to the operator [completed] - Component: `none` - Tests: From 0442510170c02543db22f1c05eabacc38fcebe9e Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sun, 11 Oct 2026 22:57:30 -0700 Subject: [PATCH 34/34] Close plan: tool-call-cap-debt Plan: vibe/2026-10-11-7-tool-call-cap-debt.md --- vibe/2026-10-11-7-tool-call-cap-debt.md | 6 +++--- vibe/ACTIVE | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/2026-10-11-7-tool-call-cap-debt.md b/vibe/2026-10-11-7-tool-call-cap-debt.md index c0b070851..578408ab9 100644 --- a/vibe/2026-10-11-7-tool-call-cap-debt.md +++ b/vibe/2026-10-11-7-tool-call-cap-debt.md @@ -4,13 +4,13 @@ overview: "Remove the one debt the tool-call-cap work added: the public Plugin c todos: - id: d1-1-docs content: "D1-1: make Plugin::call's # Invariants the full drop-occasion list; point the promptforge-plugin crate doc and plugin-user-input (lines 33, 72-73) at it; make ToolDescriptor::survives_stop's field doc accurate. D1-14: add the no-waiting-on-another-call bullet to the same # Invariants. D1-16: 'in flight or waiting' in RunControl::stop_round (harness-control.rs line 51)" - status: pending + status: completed - id: d1-1-pin content: "D1-1: add cancelling_a_task_drops_its_question_to_the_operator to tests/it/harness-stop-cap.rs, holding Main's after round until the question is torn down" - status: pending + status: completed - id: verify content: Focused stop_cap tests, contract read-through, git grep for 'only a cancel', docs builds, docs-claims, API check with CARGO_BUILD_WARNINGS cleared, and the full exit gates - status: pending + status: completed isProject: false --- diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 58eb91794..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-10-11-7-tool-call-cap-debt.md