diff --git a/README.md b/README.md index eff475f3..1ef9d14c 100644 --- a/README.md +++ b/README.md @@ -1225,7 +1225,7 @@ the same for every repository unless its `AGENTS.md` names others: | Size | Writer | Fallback | |---|---|---| -| XS | Claude Code `claude-haiku-5-5` high | Codex `gpt-6.1-sol` low | +| XS | Claude Code `claude-haiku-5-5` xhigh | Codex `gpt-6.1-sol` low | | S | Claude Code `claude-opus-5-5` low | Codex `gpt-6.1-sol` medium | | M | Claude Code `claude-opus-5-5` medium | Codex `gpt-6.1-sol` high | | L | Claude Code `claude-opus-5-5` high | Codex `gpt-6.1-sol` xhigh | @@ -1247,7 +1247,7 @@ XS, and `quest models` prints the entry to run: ```console $ quest models XS -claude claude-haiku-5-5 high +claude claude-haiku-5-5 xhigh codex gpt-6.1-sol low $ quest models XS --writer claude-opus-5-5 diff --git a/assets/AGENTS.md b/assets/AGENTS.md index a61db2bc..c110d281 100644 --- a/assets/AGENTS.md +++ b/assets/AGENTS.md @@ -184,7 +184,7 @@ A long-running coordinator lets compaction bound its context: it keeps what must Run each agent at an entry of `quest`'s ranked lists unless the repository's `AGENTS.md` names other models: as a native sub-agent on this session's harness, else through `quest agent`. Size each `/quest-iterate` you dispatch as a quest, by the work left on its PR, and name that size in its brief. -Start it on the PR's writer, the model of its first `Implementer:` line, at the entry `quest models --writer ` prints, waiting out its usage limit so the other model can review; a PR with no known writer takes its size's first entry that can run. +Start it on the PR's writer, the model of its first `Implementer:` line, at the entry `quest models --writer ` prints, waiting out its usage limit so the other model can review; a PR whose writer no list names, such as `unknown` or a retired model, takes its size's first entry that can run. `quest agent start --harness --model --effort --permission-mode ` runs a headless agent on that CLI's own login; start it as a background command in its own isolated worktree, so the harness reports its exit as it would a sub-agent's. `--permission-mode` is Claude Code's permission mode, or Codex's sandbox mode. diff --git a/quest/m1/model-check.md b/quest/m1/model-check.md index 50d04740..a1ce5121 100644 --- a/quest/m1/model-check.md +++ b/quest/m1/model-check.md @@ -75,8 +75,28 @@ checkpoints and outages to count by hand: - Too thin (0-4 PRs per cell) to move a size; no table changed. Watch iterates below the size's effort, and `Implementer:` lines without an effort (6 of the 20 lines said `unknown`). + - Changed after that run rather than on tallies (maintainer, 2026-11-09): + XS writes on Haiku xhigh instead of high. Tally that cell fresh. The + source was [mypriors.com/adamholter](https://mypriors.com/adamholter)'s + score against subscription cost per task, a 10-benchmark index read + off its chart (score, then $ per task), which this check compares each + cell with again: + - Haiku 5.5: high 30, $0.0034; xhigh 34.6, $0.0053. + - Sonnet 5.5: medium 30.7, $0.012; below the frontier at every effort, + so rejected for XS though Anthropic's + [model guide](https://claude.dev/blog/which-claude-model-should-you-build-with/) + favors it for scoped coding. + - Opus 5.5: low 34.7, $0.015; medium 48, $0.037; high 51.7, $0.052; + xhigh 57.3, $0.092; max 58.9, $0.15. + - gpt-6.1-sol: low 34.8, $0.0094; medium 46.3, $0.021; high 50.3, + $0.034; xhigh 52.5, $0.045. + - Fable 5.2: medium 56, $3.8; max 64.6, $26.5. Rejected for XL, which + stays on Opus xhigh, on the frontier; zero-data-retention + organizations may also be refused Fable. + - S looks lopsided: Opus low scores well below its Sol medium fallback, + which costs less. Check its tallies before moving it. - Compare both tables' models (in `src/cli/models.ts`) with each vendor's current models: the newest - general model. Bump any that are stale, and check that both CLIs + model of each tier the tables pin. Bump any that are stale, and check that both CLIs accept the new ids and the pinned efforts. `--deep` and its strongest-model tier were dropped in [moq-dev/quest#361](https://github.com/moq-dev/quest/pull/361). @@ -85,7 +105,7 @@ checkpoints and outages to count by hand: 2026-11-04): Sol low, medium, medium, high, and xhigh for XS to XL since the [token waste](/quest/m0/token-waste/README.md) audit (user, 2026-11-05; it had high and xhigh for M and L), and Opus low, low, - medium, high, and high when Sol wrote the code (agent's choice). They replaced Codex `xhigh` and Claude `high` (maintainer, + medium, high, and xhigh when Sol wrote the code (agent's choice). They replaced Codex `xhigh` and Claude `high` (maintainer, 2026-11-03), for which Claude Code 2.1.316 ran `claude-opus-5-5` with `--effort high` and `/code-review high`, and codex-cli 0.166.0 ran `gpt-6.1-sol` with `model_reasoning_effort="xhigh"`, both with `quest review`'s flags. diff --git a/src/cli/models.ts b/src/cli/models.ts index dd1af733..5aedf873 100644 --- a/src/cli/models.ts +++ b/src/cli/models.ts @@ -22,10 +22,11 @@ const haiku = (effort: string): Model => ({ harness: "claude", model: "claude-ha /** * Each size's writers, ranked: a spawn starts the quest on the first that can * run, falling through only while nothing is written, since the model that - * writes a PR runs all its iterates. + * writes a PR runs all its iterates. XS runs Haiku at xhigh, which scores about + * as Opus low does for a third of its subscription cost per task. */ export const WRITERS: Record = { - XS: [haiku("high"), sol("low")], + XS: [haiku("xhigh"), sol("low")], S: [opus("low"), sol("medium")], M: [opus("medium"), sol("high")], L: [opus("high"), sol("xhigh")], @@ -34,7 +35,7 @@ export const WRITERS: Record = { /** * Each PR size's reviewers, ranked: the first whose model is not the latest - * writer's reviews it, so Opus reviews what Sol changed last, at its own write efforts. + * writer's reviews it, so Claude reviews what Sol changed last. */ export const REVIEWERS: Record = { XS: [sol("low"), opus("low")], diff --git a/test/cli.test.ts b/test/cli.test.ts index aa4dd466..16af78d7 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -186,7 +186,7 @@ test("models ranks a size's writers, or the one entry that runs a PR's writer", const tree = Tree.baseline(); expect(tree.run("models", "XS")).toEqual({ code: 0, - stdout: "claude\tclaude-haiku-5-5\thigh\ncodex\tgpt-6.1-sol\tlow\n", + stdout: "claude\tclaude-haiku-5-5\txhigh\ncodex\tgpt-6.1-sol\tlow\n", stderr: "", }); expect(JSON.parse(tree.run("models", "XL", "--json").stdout)).toEqual([ @@ -196,7 +196,7 @@ test("models ranks a size's writers, or the one entry that runs a PR's writer", expect(writer("M", "gpt-6.1-sol")).toBe("codex\tgpt-6.1-sol\thigh\n"); // A writer its size does not list runs at the nearest size that lists it. expect(writer("XS", "claude-opus-5-5")).toBe("claude\tclaude-opus-5-5\tlow\n"); - expect(writer("L", "claude-haiku-5-5")).toBe("claude\tclaude-haiku-5-5\thigh\n"); + expect(writer("L", "claude-haiku-5-5")).toBe("claude\tclaude-haiku-5-5\txhigh\n"); expect(writer("XL", "gpt-6.1-sol")).toBe("codex\tgpt-6.1-sol\txhigh\n"); const unknown = tree.run("models", "S", "--writer", "unknown"); expect(unknown.code).toBe(2); diff --git a/test/review.test.ts b/test/review.test.ts index 1e20b9fa..f71808d5 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -304,7 +304,7 @@ test("each size's reviewer is the first whose model is not the latest writer's", expect(at("M", claude)).toBe("codex gpt-6.1-sol medium"); expect(at("L", claude)).toBe("codex gpt-6.1-sol high"); expect(at("XL", claude)).toBe("codex gpt-6.1-sol xhigh"); - // Opus reviews what Sol wrote. + // Claude reviews what Sol wrote. expect(at("XS", codex)).toBe("claude claude-opus-5-5 low"); expect(at("S", codex)).toBe("claude claude-opus-5-5 low"); expect(at("M", codex)).toBe("claude claude-opus-5-5 medium");