diff --git a/assets/CONTEXT.md b/assets/CONTEXT.md index 72db6b52..903408b1 100644 --- a/assets/CONTEXT.md +++ b/assets/CONTEXT.md @@ -133,6 +133,9 @@ Once the reset passes or the login is restored, the item has cleared: whoever fi The PR body's `## Models` section records the PR's size and each implementing session's harness, model, and effort. A quest PR's size is its quest's; a skill that opens its own PR, such as `/quest-plan`, sizes it as a quest is sized. +List them as `Size: M` and one `Implementer: Codex, gpt-6.1-sol, high effort` line per session, which each session adds when it first pushes. +Read your model and effort rather than guess: Claude Code knows its model and exports `CLAUDE_EFFORT`; Codex records `model` and `effort` in the last `turn_context` of `~/.codex/sessions/*/*/*/rollout-*-$CODEX_THREAD_ID.jsonl`; Grok records `model_id` and `reasoning_effort` in `~/.grok/sessions/*/$GROK_SESSION_ID/chat_history.jsonl`. +Write `unknown` for any you cannot find. The PR body's `## Review` section records the latest round: the reviewer harness, model, and effort, the reviewed head SHA, the round's score out of 10 with its rationale, and each finding's disposition (fixed, or refuted with the reason), or the skip and its reason. Never block on a score: the findings and their refutations decide when the review is done. Rewrite it after each round. diff --git a/cloudflare/src/github/service.ts b/cloudflare/src/github/service.ts index d9d6c1a8..81af9a78 100644 --- a/cloudflare/src/github/service.ts +++ b/cloudflare/src/github/service.ts @@ -13,6 +13,8 @@ import { } from "quest/core"; import { checksState, latestContexts } from "../../../src/github/checks"; import { cleanAutoMergeRefusal, immediateMerge, queueHead } from "../../../src/github/merge"; +import { implementers, withModels } from "../../../src/github/models"; +import { BODY_LIMIT } from "../jobs/trail"; import { osiApproved } from "../licenses"; import { CI_PERMISSIONS, type CIPermission, ciProblems } from "./ci"; import { OBSERVED, type Observed, observedGap, REVIEW_CHECK, unreviewed } from "./review"; @@ -2204,10 +2206,13 @@ export class GitHubService { if (pull?.state !== "open") throw error; } } - if (pull.title !== input.title || pull.body !== input.body) + // A later session's body keeps the sessions that delivered before it in its `## Models` record. + const body = implementers(pull.body ?? "").reduce(withModels, input.body); + if (body.length > BODY_LIMIT) throw new GitHubError("The pull request body is too long"); + if (pull.title !== input.title || pull.body !== body) pull = await this.#api(token, `${pulls}/${number(pull.number)}`, "PATCH", { title: input.title, - body: input.body, + body, }); // Finished work leaves draft; a maintainer's own choice to mark it ready is never reverted. if (pull.draft && !input.draft) { diff --git a/cloudflare/test/seed.test.ts b/cloudflare/test/seed.test.ts index 84126828..004920cc 100644 --- a/cloudflare/test/seed.test.ts +++ b/cloudflare/test/seed.test.ts @@ -465,6 +465,26 @@ it("publishes jobs' work end to end, through the upstream branch to the pull req // Finished work marks it ready for review. await publish({ job: "seed-start-offline", head: first.head, pull: { ...pull, draft: false } }); expect(rocket.pulls.at(-1)!.draft).toBe(false); + // A later session's body keeps the sessions that delivered before it in its `## Models` record. + const models = (lines: string) => ({ + ...pull, + draft: false, + body: `Done.\n\n## Models\n\n${lines}\n\n## Review\n\nNone`, + }); + await publish({ + job: "seed-start-offline", + head: first.head, + pull: models("- Implementer: claude, first, high effort"), + }); + await publish({ + job: "seed-start-offline", + head: first.head, + pull: models("- Size: S\n- Implementer: opencode, second, high effort"), + }); + expect(rocket.pulls.at(-1)!.body).toBe( + models("- Size: S\n- Implementer: opencode, second, high effort\n- Implementer: claude, first, high effort") + .body, + ); const continued = await latest("quest/m0/import.md"); expect(continued).toMatchObject({ id: retry.id, state: "running", head: rocket.branches["quest/m0/import"] }); diff --git a/quest/m0/model-routing/README.md b/quest/m0/model-routing/README.md index 7a0f8ce5..1cb34d9b 100644 --- a/quest/m0/model-routing/README.md +++ b/quest/m0/model-routing/README.md @@ -7,14 +7,14 @@ can run from a ranked list `quest` builds in for the quest's size, the same for every repository unless its `AGENTS.md` overrides it: Grok for XS, Opus from S up, with Sol as the fallback through M, and a usage limit falling through to the next entry. Sol reviews, at an effort set by size, and Opus -reviews whatever Sol wrote, so no model reviews its own code. Every commit -and PR records the models and efforts behind it, and fixes link the PR that +reviews whatever Sol wrote, so no model reviews its own code. Every PR +records the models and efforts behind it, and fixes link the PR that caused them, so the rankings can be audited and retuned. This epic's own work is one live spawn in moq-dev/quest from Claude Code that starts an XS quest on Grok and an S or larger on Opus, each recorded and reviewed by Sol. Non-goals: routing kix.quest's runner jobs, which pick their own accounts -and models (their commits still record their models); billing agents to API +and models (their PRs still record their models); billing agents to API keys, which this epic removes. ## Plan @@ -67,8 +67,12 @@ are mostly judgment. Sol xhigh. L and XL: Opus xhigh, no fallback. Reviews: Sol low, medium, high, xhigh, xhigh for XS to XL, never the model that wrote the code; Pullfrog and CodeRabbit add free reviews on top. -- Every commit and PR records its models and efforts, and every fix links - the PR that caused it, so the rankings can be audited (user). +- Every PR records its models and efforts, and every fix links the PR that + caused it, so the rankings can be audited (user). The PR record alone, + without commit trailers (user, 2026-11-04 design checkpoint of + [moq-dev/quest#494](https://github.com/moq-dev/quest/pull/494), revising + that day's /quest-plan pick of `Agent-*` trailers): squash merges drop + branch trailers from `main`, so audits read the PR record anyway. - Every `/quest-review` scores the change out of 10 (user, while planning [the council](/quest/m2/council.md), which joins m2). - Which milestone? ✅ m0 (rejected: m1). It ranks first among m0's work @@ -78,6 +82,5 @@ are mostly judgment. - [Grok agents](/quest/m0/model-routing/agent-harnesses.md) - `quest agent` starts Grok as well as Claude Code and Codex - [Route by size](/quest/m0/model-routing/models.md) - `/quest-spawn` starts each quest on the first entry of its size's ranked list that can run -- [Commits and PRs record their models](/quest/m0/model-routing/implementer.md) - commit trailers and PR records name each harness, model, and effort, and the PR's size - [kix records its follow-up sessions](/quest/m0/model-routing/kix-sessions.md) - kix `iterate` and `merge` jobs that change an existing PR record their `Implementer:` line in a PR comment - [moq-dev pins](/quest/m0/model-routing/pins.md) - moq-dev/moq and moq-dev/moq.pro bump their `quest` pin to pick up the routing diff --git a/quest/m0/model-routing/implementer.md b/quest/m0/model-routing/implementer.md deleted file mode 100644 index 9308016d..00000000 --- a/quest/m0/model-routing/implementer.md +++ /dev/null @@ -1,39 +0,0 @@ -# [M] Commits and PRs record their models - -## Goal - -Every commit an agent makes carries its harness, model, and effort in -trailers, and every PR's body names the PR's size and each implementing -session's harness, model, and effort beside `## Review`, which already -names the reviewer's. This covers quest PRs, skills' own PRs such as -`/quest-plan`'s, the land fixer's commits, and kix runner jobs, so the -[model check](/quest/m1/model-check.md) and -[`quest usage`](/quest/m1/usage.md) can audit how each ranking performs. - -## Plan - -Decided 2026-11-04 (user, /quest-plan follow-ups of -[moq-dev/quest#487](https://github.com/moq-dev/quest/pull/487)): record the -model and effort for every commit and PR, to audit the rankings. - -- Commit trailers (agent's choice of names): `Agent-Harness`, - `Agent-Model`, and `Agent-Effort`, one each, beside the - `Co-Authored-By` line a harness already adds. `quest context` tells - agents to add them; the land fixer (`src/cli/land-fix.ts`) and the kix - runner (`src/runner/`) add them in code, since no agent writes those - commit messages. -- The PR record: `quest context`'s Reviews section (`assets/CONTEXT.md`) - already requires the reviewer's harness, model, and effort in - `## Review`, and a `## Models` section naming the PR's size and each - implementing session's harness, model, and effort, which `quest review - --size` and `--exclude-model` read ([moq-dev/quest#493](https://github.com/moq-dev/quest/pull/493)). - Settle the section's line format, and say each session adds its line - when it pushes, so a quest that `/quest-iterate` finishes on another - model lists both. -- An agent finds its own settings rather than guessing: Claude Code - exports `CLAUDE_EFFORT` and knows its model. Find where Codex and Grok - expose their model and effort to the agent, and name them in the rule. - When an agent cannot find its effort, it writes `unknown` rather than - leaving the record out. -- Fixes link the PR that caused them under `## Caused by`, per - `quest context`'s Pull requests section. diff --git a/quest/m0/model-routing/kix-sessions.md b/quest/m0/model-routing/kix-sessions.md index 439481f1..a4c45f2d 100644 --- a/quest/m0/model-routing/kix-sessions.md +++ b/quest/m0/model-routing/kix-sessions.md @@ -48,7 +48,3 @@ Implementation guidance: comment too, and `quest context`'s `## Models` rule and its `--exclude-model` rule ("each implementing model the PR body records") to name it. - -## Required - -- [Commits and PRs record their models](/quest/m0/model-routing/implementer.md) - this builds on its `## Models` record and the runner's reporting diff --git a/quest/m0/model-routing/pins.md b/quest/m0/model-routing/pins.md index 9a3955d9..b4bed631 100644 --- a/quest/m0/model-routing/pins.md +++ b/quest/m0/model-routing/pins.md @@ -20,4 +20,3 @@ repository uses the same settings, so nothing in them changes but the pin. ## Required - [Route by size](/quest/m0/model-routing/models.md) - `/quest-spawn` reads the built-in size table -- [PRs record their implementer](/quest/m0/model-routing/implementer.md) - the record that shows the routing working diff --git a/quest/m1/model-check.md b/quest/m1/model-check.md index 9bc55ad9..c208badc 100644 --- a/quest/m1/model-check.md +++ b/quest/m1/model-check.md @@ -25,9 +25,9 @@ the size tables of [model routing](/quest/m0/model-routing/README.md): ([m0's run](/quest/m0/model-check.md)), then each milestone (rejected: each milestone only) -Audit the routing from the PRs merged since the last run, with -[`quest usage`](/quest/m1/usage.md) once it lands, which record -their size, implementer, and reviewer beside `## Review`: +Audit the routing from the records of the PRs merged since the last run, +with [`quest usage`](/quest/m1/usage.md) once it lands: each body's +`## Models` size and implementers and its `## Review` reviewer: - Tally by size and by implementing harness, model, and effort: review scores, review rounds, findings fixed versus refuted, design checkpoints reached, PRs diff --git a/quest/m1/usage.md b/quest/m1/usage.md index b313496c..d62bbeee 100644 --- a/quest/m1/usage.md +++ b/quest/m1/usage.md @@ -39,8 +39,8 @@ Agent's choices, for the implementer to revisit: - A session belongs to a quest or PR by its working directory's checkout and branch: a quest branch names its quest, and a PR's head branch names the PR. Sessions on no quest or PR branch roll up as unattributed. -- Accuracy comes from the records: commit trailers and the PR body's - size, implementers, and each `## Review` round's reviewer, score, and findings +- Accuracy comes from the PR record: the body's `## Models` size and + implementers, and each `## Review` round's reviewer, score, and findings with their dispositions; follow-up fixes are later PRs that list it under `## Caused by` (`quest context`'s Pull requests section). - Prices are a pinned table beside the model lists, by model and token diff --git a/src/cli/land-fix.ts b/src/cli/land-fix.ts index f6a0543b..141bd7c9 100644 --- a/src/cli/land-fix.ts +++ b/src/cli/land-fix.ts @@ -87,7 +87,14 @@ export const signature = (failure: string) => failure.replace(/[`'"]/g, "").repl * a commit on the base that skips the test and adds its follow-up quest, which the daemon opens as a PR * of its own. */ -export type FixResult = { agent?: string; model?: string; blame?: boolean; comment?: string; failure?: string } & ( +export type FixResult = { + agent?: string; + model?: string; + effort?: string; + blame?: boolean; + comment?: string; + failure?: string; +} & ( | { number: number; action: "requeue"; reason: string; cause?: Cause; disable?: string } | { number: number; action: "fixed"; head: string; reason: string } | { number: number; action: "handback"; diagnosis: string; recommendation: string; log?: [number, number] } @@ -97,7 +104,7 @@ export type FixResult = { agent?: string; model?: string; blame?: boolean; comme * The fixer's verdict on the base alone, when no PR is supplied: a mechanical fix committed on its * `base` branch, which the daemon opens as a PR, or a handback, which halts the queue with an issue. */ -export type BaseFix = { agent?: string; model?: string } & ( +export type BaseFix = { agent?: string; model?: string; effort?: string } & ( | { action: "fixed"; head: string; reason: string } | { action: "handback"; diagnosis: string; recommendation: string } ); @@ -122,7 +129,7 @@ function validate(item: any, pr: boolean): void { : item.action === "handback" ? [...named, "action", "diagnosis", "recommendation"] : []; - const metadata = ["agent", "model"].filter((key) => key in item); + const metadata = ["agent", "model", "effort"].filter((key) => key in item); if (metadata.some((key) => typeof item[key] !== "string" || !/^[a-zA-Z0-9 ._/-]{1,100}$/.test(item[key]))) throw new Error("invalid fixer identity"); if (pr && "blame" in item) { @@ -295,6 +302,8 @@ export async function fix( ...(model ? ["--model", model] : []), "-", ]; + // Codex names its model in the header it prints first, whether or not the command line does. + let header = ""; await new Promise((done, reject) => { // Relay the agent's output through pipes of its own rather than letting it inherit ours. Once Bun writes // to a pipe it marks that pipe non-blocking, and an inheriting child shares the flag: Codex then panicked @@ -302,6 +311,9 @@ export async function fix( const child = spawn(program, argv, { cwd: root, env, stdio: ["pipe", "pipe", "pipe"] }); child.stdout.pipe(process.stdout, { end: false }); child.stderr.pipe(process.stderr, { end: false }); + child.stderr.on("data", (chunk: Buffer) => { + if (header.length < 4096) header += chunk.toString("utf8"); + }); child.once("error", reject); child.stdin.on("error", () => {}); child.stdin.end( @@ -312,8 +324,15 @@ export async function fix( const parse = (text: string) => (numbers.length ? parseFixResult(text, numbers) : parseBaseFix(text)); const results = parse(readFixResult(root)); const index = argv.findIndex((arg) => arg === "-m" || arg === "--model"); - const ranModel = index >= 0 ? argv[index + 1] : argv.find((arg) => arg.startsWith("--model="))?.slice(8); - const identity = { agent: basename(agent), model: ranModel ?? "Codex" }; + const ranModel = + (index >= 0 ? argv[index + 1] : argv.find((arg) => arg.startsWith("--model="))?.slice(8)) ?? + /^model: ([a-zA-Z0-9._/-]{1,100})$/m.exec(header)?.[1]; + const identity = { + agent: basename(agent), + model: ranModel ?? "unknown", + // Replaced arguments may run the agent at any effort. + effort: args ? "unknown" : effort, + }; const result = JSON.stringify( Array.isArray(results) ? results.map((item) => ({ ...item, ...identity })) : { ...results, ...identity }, ); diff --git a/src/cli/land-fixer.ts b/src/cli/land-fixer.ts index 3f90fd92..40131382 100644 --- a/src/cli/land-fixer.ts +++ b/src/cli/land-fixer.ts @@ -711,14 +711,15 @@ export function fixerText(text: string): string { .replace(/[\r\n\u0000-\u001f\u007f]/g, " "); return `\`${value}\``; } -export function author(result?: Pick): string { +/** The fixer's harness, model, and effort, as a PR's `## Models` record names a session. */ +export function author(result?: Pick): string { const agent = result?.agent ?? "Codex"; const model = result?.model; return ( - [agent, model && model !== agent ? model : undefined] + [agent, model && model !== agent ? model : undefined, result?.effort && `${result.effort} effort`] .filter(Boolean) - .join(" / ") - .replace(/[^a-zA-Z0-9 ._/-]/g, " ") + .join(", ") + .replace(/[^a-zA-Z0-9 ,._/-]/g, " ") .slice(0, 100) || "Codex" ); } diff --git a/src/github/models.ts b/src/github/models.ts new file mode 100644 index 00000000..c23f9dc5 --- /dev/null +++ b/src/github/models.ts @@ -0,0 +1,33 @@ +// A pull request body's `## Models` record, which names the PR's size and each session that implemented it. + +const SECTION = /^## Models[ \t]*\r?$/m; + +/** Where `body`'s `## Models` section runs, after its heading, or null without one. */ +function section(body: string): { start: number; end: number } | null { + const heading = SECTION.exec(body); + if (!heading) return null; + const start = heading.index + heading[0].length; + const next = /^## /m.exec(body.slice(start)); + return { start, end: next ? start + next.index : body.length }; +} + +/** The `Implementer:` items of `body`'s `## Models` section, each as its line without the bullet. */ +export function implementers(body: string): string[] { + const found = section(body); + if (!found) return []; + return body + .slice(found.start, found.end) + .split(/\r?\n/) + .flatMap((line) => /^- (Implementer: .+?)\s*$/.exec(line)?.[1] ?? []); +} + +/** `body` with `line` as an item of its `## Models` section, which it gains when it has none. */ +export function withModels(body: string, line: string): string { + const item = `- ${line}`; + if (body.split(/\r?\n/).some((each) => each.trim() === item)) return body; + const found = section(body); + if (!found) return `${body.trimEnd()}\n\n## Models\n\n${item}\n`; + const { start, end } = found; + const gap = body.slice(start, end).trim() ? "\n" : "\n\n"; + return `${body.slice(0, end).trimEnd()}${gap}${item}\n${end < body.length ? "\n" : ""}${body.slice(end)}`; +} diff --git a/src/runner/adapter.ts b/src/runner/adapter.ts index ceca968b..bc75f609 100644 --- a/src/runner/adapter.ts +++ b/src/runner/adapter.ts @@ -69,6 +69,8 @@ export interface Running { done: Promise; /** Ask the CLI to stop, killing it after a grace period. */ stop(): void; + /** The model and effort the session runs on, each `unknown` where the CLI does not show it. */ + ran(): Promise<{ model: string; effort: string }>; } export interface SignedIn { diff --git a/src/runner/claude.ts b/src/runner/claude.ts index e16e12b1..799175dd 100644 --- a/src/runner/claude.ts +++ b/src/runner/claude.ts @@ -9,7 +9,7 @@ // runner's settings apply: the checkout's own settings could add hooks that // run outside the sandbox, so neither they nor its CLAUDE.md are loaded. import { execFile, spawn } from "node:child_process"; -import { rmSync, writeFileSync } from "node:fs"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; import { constants } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -60,8 +60,13 @@ export function jobEnvironment(env: NodeJS.ProcessEnv, home: string, sandbox: Sa * only the checkout and read only the checkout (outside it Claude Code would * ask), and WebFetch, which would run outside the sandbox, is off. */ -export function claudeSettings(sandbox: Sandbox): Record { +export function claudeSettings(sandbox: Sandbox, effort: string): Record { + // Claude Code shows its effort only to hooks and commands, as CLAUDE_EFFORT, and only within a turn, so + // a hook on the kix tools records it before `deliver` reads it. Hooks run outside the sandbox, so + // `effort` is in the account's home, which commands cannot reach. + const record = { type: "command", command: `printf %s "$CLAUDE_EFFORT" > '${effort.replaceAll("'", "'\\''")}'` }; return { + hooks: { PreToolUse: [{ matcher: "mcp__kix__.*", hooks: [record] }] }, sandbox: { enabled: true, failIfUnavailable: true, @@ -101,11 +106,16 @@ function json(line: string): Record | undefined { export function streamReader() { let result: Record | undefined; let limited: { until: number | null } | null = null; + let model = "unknown"; return { + /** The model the session started on. */ + model: () => model, line(line: string) { const event = json(line); if (!event) return; if (event.type === "result") result = event; + if (event.type === "system" && event.subtype === "init" && typeof event.model === "string") + model = event.model; if (event.type === "rate_limit_event") { const info = event.rate_limit_info as Record | undefined; const resets = typeof info?.resetsAt === "number" ? info.resetsAt * 1000 : null; @@ -184,10 +194,19 @@ export class Claude implements Adapter { const config = join(launch.home, `kix-tools-${launch.session}.json`); writeFileSync(config, JSON.stringify(mcp), { mode: 0o600 }); const settings = join(launch.home, `kix-settings-${launch.session}.json`); - writeFileSync(settings, JSON.stringify(claudeSettings(launch.sandbox)), { mode: 0o600 }); + const effortFile = join(launch.home, `kix-effort-${launch.session}`); + writeFileSync(settings, JSON.stringify(claudeSettings(launch.sandbox, effortFile)), { mode: 0o600 }); + const readEffort = () => { + try { + return readFileSync(effortFile, "utf8").trim() || "unknown"; + } catch { + return "unknown"; + } + }; const cleanup = () => { rmSync(config, { force: true }); rmSync(settings, { force: true }); + rmSync(effortFile, { force: true }); }; const argv = [ "-p", @@ -246,6 +265,7 @@ export class Claude implements Adapter { }); return { done, + ran: async () => ({ model: reader.model(), effort: readEffort() }), stop() { signal("SIGTERM"); killer ??= setTimeout(() => signal("SIGKILL"), STOP_GRACE_MS); diff --git a/src/runner/job.ts b/src/runner/job.ts index c82ea4b4..8aea4d19 100644 --- a/src/runner/job.ts +++ b/src/runner/job.ts @@ -6,6 +6,7 @@ import { randomUUID } from "node:crypto"; import { type Round, ResponsesError } from "../cli/review"; +import { withModels } from "../github/models"; import type { Adapter, Exit, Running } from "./adapter"; import { Refused } from "./connection"; @@ -90,6 +91,9 @@ function subject(summary: string): string { return line.length <= 72 ? line : `${line.slice(0, 71)}…`; } +/** `value` when it fits in a `## Models` line, otherwise `unknown`. */ +const field = (value: string) => (/^[\w.:/@+-]{1,100}$/.test(value) ? value : "unknown"); + function string(value: unknown, name: string, max: number): string { if (typeof value !== "string" || !value.trim()) throw new Error(`${name} is required`); if (value.length > max) throw new Error(`${name} is longer than ${max} characters`); @@ -115,6 +119,8 @@ export class Attempt { /** Settles once no review runs, so the attempt never removes a checkout a review still uses. */ #reviewRun: Promise = Promise.resolve(); #agent: Running | undefined; + /** The CLI the attempt's account runs, which its pull request's `## Models` record names. */ + #harness = "unknown"; /** Aborts the devShell build, which runs before there is an agent to stop. */ #preparing = new AbortController(); #publishing: Promise = Promise.resolve(); @@ -209,6 +215,7 @@ export class Attempt { return await this.#finish("failed", `The account ${job.account} is no longer on this runner.`); // Before any checkout: an account added while the runner runs may lack its CLI or sandbox. const adapter = ctx.adapter(account); + this.#harness = adapter.name; this.#account = account; this.#workspace = await Workspace.prepare(ctx.root, job, ctx.remote(job)); if (this.#workspace.preserved) @@ -742,11 +749,14 @@ export class Attempt { const closed = this.#closed(); if (closed) return closed; const text = string(args.summary, "summary", 4000); + // kix names this session in the body's `## Models` record, from what the CLI reports. + const ran = (await this.#agent?.ran()) ?? { model: "unknown", effort: "unknown" }; + const implementer = `Implementer: ${field(this.#harness)}, ${field(ran.model)}, ${field(ran.effort)} effort`; const pull = pulled ? undefined : { title: string(args.title, "title", 256), - body: string(args.body, "body", 60_000), + body: withModels(string(args.body, "body", 59_000), implementer), draft: args.draft !== false, }; const method = args.method as Land["method"]; diff --git a/src/runner/opencode.ts b/src/runner/opencode.ts index fd66a2a1..8398ede3 100644 --- a/src/runner/opencode.ts +++ b/src/runner/opencode.ts @@ -711,6 +711,8 @@ export class Opencode implements Adapter { ); let stopping = false; + const unknown = { model: "unknown", effort: "unknown" }; + let ran: () => Promise<{ model: string; effort: string }> = async () => unknown; let abort: () => Promise = async () => {}; const events = new AbortController(); let watched: Promise = Promise.resolve(); @@ -736,6 +738,13 @@ export class Opencode implements Adapter { writeFileSync(join(sessions, launch.session), session, { mode: 0o600 }); } abort = () => client.session.interrupt({ sessionID: session }); + // The assistant message that is calling a kix tool is the session's latest. + ran = async () => { + const { data } = await client.message.list({ sessionID: session, limit: 1, order: "desc" }); + const info = data[0]; + if (info?.type !== "assistant") return unknown; + return { model: `${info.model.providerID}/${info.model.id}`, effort: info.model.variant ?? "unknown" }; + }; if (stopping) return failure("stopped"); let limited: Exit["limited"] = null; @@ -792,6 +801,7 @@ export class Opencode implements Adapter { }); return { done, + ran: () => ran().catch(() => unknown), stop() { stopping = true; // Interrupt the turn so opencode records it, then stop the server whether or not it answered. diff --git a/src/runner/prompt.ts b/src/runner/prompt.ts index e84fc8f5..f131946f 100644 --- a/src/runner/prompt.ts +++ b/src/runner/prompt.ts @@ -22,7 +22,7 @@ export function systemPrompt(domains: string[]): string { Report only through the kix tools: - \`checkpoint\`: at each meaningful step, with a short summary the maintainer reads on their phone. kix publishes the working tree's changes to the job's branch, so another runner can continue if this one goes away. - \`ask\`: one round of independent questions for the maintainer, with options and your recommendation. kix pauses the job; end your turn right after asking. You are resumed in this session with the answers. -- \`deliver\`: when the work is done, with a summary and the pull request's title, body, and draft flag. List the follow-ups you suggest as \`followups\` too, which kix offers the maintainer to plan. kix publishes the working tree and opens or updates the pull request. Unless it stays a draft, the runner then reviews the head kix committed with the other vendor's model, as \`quest review\` does, and kix records the round as the pull request's \`quest review\` check. If the review found something, fix or refute each finding and deliver again with \`responses\`; otherwise end your turn after delivering. +- \`deliver\`: when the work is done, with a summary and the pull request's title, body, and draft flag. List the follow-ups you suggest as \`followups\` too, which kix offers the maintainer to plan. kix publishes the working tree and opens or updates the pull request, adding this session's \`Implementer:\` line to the body's \`## Models\` record, so write only its \`Size:\` there. Unless it stays a draft, the runner then reviews the head kix committed with the other vendor's model, as \`quest review\` does, and kix records the round as the pull request's \`quest review\` check. If the review found something, fix or refute each finding and deliver again with \`responses\`; otherwise end your turn after delivering. - \`context\`: on a job for an issue or pull request, read it as GitHub has it now: its text and comments, and a pull request's reviews, unresolved review threads, and checks. Use it wherever the workflow reads them with \`gh\` or \`quest status\`. On a new plan, it reads the maintainer's idea and any context pasted with it. Where the workflow says to claim with \`quest start\`, or to take or release a pull request with \`quest iterate\`, \`quest merge\`, or \`quest release\`, skip it; kix holds the job's pull request. Where it says to push, open or update a pull request, mark it ready, or comment on it, call \`deliver\` instead. Where it says to run \`quest review\` or \`/quest-review\`, deliver instead and let the runner review: its check is the head's review record, so write in the body's \`## Review\` section that the \`quest review\` check records it. Where it says to ask or prompt the user, call \`ask\`; you can always prompt, so ignore instructions for agents that cannot. Never run \`git push\`, \`gh\`, or anything else that writes to GitHub. Commit locally if you like; kix publishes your working tree as it is. diff --git a/test/land.test.ts b/test/land.test.ts index 42a884ec..74e9b096 100644 --- a/test/land.test.ts +++ b/test/land.test.ts @@ -25,6 +25,7 @@ import { ACTIONS_APP, type Pull, type World, fakeGitHub } from "../cloudflare/de import { authenticatedRead, gitAuthentication } from "../src/cli/land-git"; import { ADMITTED, + author, COLLECTION_ATTEMPTS, collectFixers, validateFix, @@ -2770,7 +2771,9 @@ describe.skipIf(process.platform === "win32")("asynchronous land fixer", () => { expect(fix.branch).toBe(branch); const head = forge.ref("upstream/project", `refs/heads/${branch}`); expect(git(forge.origin, "rev-parse", `${head}^`)).toBe(fixture.base.sha); - expect(fix.body).toContain(`mechanical fix by agent / Codex: \`restore the missing file\` (head ${head})`); + expect(fix.body).toContain( + `mechanical fix by agent, unknown, unknown effort: \`restore the missing file\` (head ${head})`, + ); expect(fix.body).toContain("Merge it as an administrator"); // Collected once, and the queue stays halted on this base. expect(await fixture.collect()).toEqual([]); @@ -2848,7 +2851,7 @@ printf '{"action":"fixed","head":"%s","reason":"edit the flake"}' "$(git rev-par expect(fixture.pull.head).not.toBe(fixture.plan.queue[0].head); expect(git(fixture.root, "show", "-s", "--format=%P", fixture.pull.head!)).toContain(fixture.plan.base.sha); expect(fixture.pull.body).toContain( - `mechanical fix by agent / Codex: \`preserve both conflict sides\` (head ${fixture.pull.head})`, + `mechanical fix by agent, unknown, unknown effort: \`preserve both conflict sides\` (head ${fixture.pull.head})`, ); expect(fixture.pull.autoMerge?.commitBody).toBe(pin(fixture.pull.head!)); }); @@ -3433,7 +3436,7 @@ ${requeueScript}`); expect(fixture.pull.autoMerge?.commitBody).toBe(pin(fixture.pull.head!)); // The fix is recorded in `## Review`; a PR that stays queued gets no outcome comment. expect(fixture.pull.body).toContain( - `mechanical fix by agent / Codex: \`preserve both conflict sides\` (head ${fixture.pull.head})`, + `mechanical fix by agent, unknown, unknown effort: \`preserve both conflict sides\` (head ${fixture.pull.head})`, ); expect(fixture.forge.comment(1)).toBeUndefined(); }); @@ -3687,6 +3690,24 @@ printf '[{"number":1,"action":"handback","diagnosis":"saw %s","recommendation":" expect(result.model).toBe("actual-model"); }); + test("records the model Codex's header names and the effort it ran at", async () => { + const dir = fixRepository(); + const log = join(dir, "failure"); + writeFileSync(log, "failure"); + const agent = join(dir, "codex"); + writeFileSync( + agent, + `#!/bin/sh\ncat >/dev/null\nprintf 'model: header-model\\nreasoning effort: high\\n' >&2\n${requeueScript}\n`, + { mode: 0o755 }, + ); + vi.stubEnv("OPENAI_BASE_URL", "http://proxy.test/v1"); + vi.stubEnv("OPENAI_API_KEY", "proxy-client"); + await fix(dir, log, [1], agent, undefined, undefined, "high"); + const [result] = parseFixResult(readFileSync(join(dir, "quest-land-result.json"), "utf8"), [1]); + expect(result).toMatchObject({ agent: "codex", model: "header-model", effort: "high" }); + expect(author(result)).toBe("codex, header-model, high effort"); + }); + test("rejects mixing individually approved protected files", async () => { const forge = new Forge(); forge.advance({ "flake.nix": "old", "shell.nix": "old" }); @@ -3745,11 +3766,12 @@ printf '[{"number":1,"action":"handback","diagnosis":"saw %s","recommendation":" const results = JSON.parse(readFileSync(resultPath, "utf8")); results[0].agent = "Claude"; results[0].model = "actual-model"; + results[0].effort = "xhigh"; writeFileSync(resultPath, JSON.stringify(results)); fixture.pull.body = "Author content\r\n\r\n## Review\r\n\r\nPrevious review\r\n\r\n(written by old-model)\n\n(written by older-model)"; expect(await fixture.collect()).toEqual(["requeued 1"]); - expect(fixture.pull.body).toContain("mechanical fix by Claude / actual-model:"); + expect(fixture.pull.body).toContain("mechanical fix by Claude, actual-model, xhigh effort:"); expect(fixture.pull.body).toContain("Author content"); expect(fixture.pull.body).toContain("Previous review"); // The author's attributions stay; the fix is credited only in its own Review line. diff --git a/test/runner.test.ts b/test/runner.test.ts index 1473278e..5f3fff4a 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -1448,7 +1448,14 @@ log({ timeout: kix.timeout, }); const out = (value) => process.stdout.write(JSON.stringify(value) + "\\n"); -out({ type: "system", subtype: "init", session_id: session }); +out({ type: "system", subtype: "init", session_id: session, model: "claude-test" }); +// Claude Code runs the settings' hooks for a tool, with the turn's effort, before calling it. +const hooks = (tool) => { + for (const entry of JSON.parse(fs.readFileSync(at("--settings"), "utf8")).hooks?.PreToolUse ?? []) + if (new RegExp("^(?:" + entry.matcher + ")$").test("mcp__kix__" + tool)) + for (const hook of entry.hooks) + require("node:child_process").execSync(hook.command, { env: { ...process.env, CLAUDE_EFFORT: "high" } }); +}; let id = 0; async function rpc(method, params) { const response = await fetch(kix.url, { @@ -1471,6 +1478,7 @@ async function rpc(method, params) { await rpc("tools/call", { name: "context", arguments: {} }); log({ backgroundWaitPid: process.pid }); } + if (step.tool) hooks(step.tool); if (step.tool) log({ tool: step.tool, reply: await rpc("tools/call", { name: step.tool, arguments: step.args }) }); if (step.wait) { process.on("SIGTERM", () => { log({ terminated: true }); process.exit(143); }); @@ -1551,6 +1559,7 @@ async function turn(session, text) { if (step.shell) log({ shell: require("node:child_process").execFileSync(config.shell, ["-c", step.shell], { encoding: "utf8" }) }); if (step.emit) send({ ...step.emit, data: { ...step.emit.data, sessionID: session } }); + if (step.assistant) messages[session].unshift({ id: "msg_" + ++id, content: [], ...step.assistant }); if (step.tool) log({ tool: step.tool, reply: await rpc("tools/call", { name: step.tool, arguments: step.args }) }); if (step.wait) { await new Promise((resolve) => (interrupted = resolve)); @@ -1951,7 +1960,12 @@ describe("quest runner", () => { expect(checkpoint!.changes).toEqual([{ path: "a.txt", content: Buffer.from("first\n").toString("base64") }]); expect(delivery).toMatchObject({ head: "0".repeat(39) + "1", - pull: { title: "feat: thing", body: "Body", draft: false }, + // The runner names the session in the body's record, from the CLI's own report of its model and effort. + pull: { + title: "feat: thing", + body: "Body\n\n## Models\n\n- Implementer: claude, claude-test, high effort\n", + draft: false, + }, }); expect(delivery!.changes.map((c: Sent) => [c.path, c.content === null])).toEqual([ ["b.txt", false], @@ -2102,7 +2116,17 @@ describe("quest runner", () => { args: { summary: "Need a choice", questions: [{ question: "Which?", options: [{ label: "A" }] }] }, }, ], - resume: [{ tool: "deliver", args: { summary: "Went with A", title: "feat: a", body: "Body" } }], + resume: [ + { assistant: { type: "assistant", model: { providerID: "openai", id: "gpt-test", variant: "high" } } }, + { + tool: "deliver", + args: { + summary: "Went with A", + title: "feat: a", + body: "## Models\n\n- Size: S\n\n## Review\n\nNone", + }, + }, + ], }); r.assign(job(r.upstream.commit), "Extra rule."); const question = await r.hub.next((m) => m.type === "question"); @@ -2209,6 +2233,10 @@ describe("quest runner", () => { const result = await r.result(); await idle(r.runner); expect(result).toMatchObject({ attempt: 2, outcome: "completed", summary: "Went with A" }); + // The resumed session names itself from its latest assistant message, in the agent's own record. + expect(r.publishes.at(-1)!.pull.body).toBe( + "## Models\n\n- Size: S\n- Implementer: opencode, openai/gpt-test, high effort\n\n## Review\n\nNone", + ); const resumed = r.calls().filter((c) => c.prompted); expect(resumed).toHaveLength(2); // The kix session maps to the opencode session that asked.