diff --git a/CLAUDE.md b/CLAUDE.md index c36d4cb..834bab8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,6 +73,20 @@ Pre-commit hooks may modify files (e.g. ruff format); re-`git add` if a hook rep fast option 10x quicker but similarly error-prone; full data, a parked "fast draft tier" idea, and the privacy implications of any multi-provider routing are in issue #31. Don't propose down-tiering the default model without rerunning that eval. +- **Haiku 5.5, Opus 5.5, Fable 5.1: evaluated 2026-10-10, Sonnet 5.5 stays.** Production prompt, 21 cases, + Opus 5 adaptive reviewer, assembly cases only, 3 runs per config unless noted. Sonnet 5.5: 43/48 correct (3 + were reviewer failures, not wrong answers), 2 errors, mean 5.2s, p95 9.4s, $0.011/call. + - *Haiku 5.5 with `thinking: disabled`* (accepted, unlike Sonnet 5.5): 36/48, 14 errors; mean 3.1s, p95 5.2s, + $0.0005/call (20x cheaper). The errors are failures to trace: misreading control flow (a straight-line + `.L4` called a loop), wrong register arithmetic, a hallucinated `imul eax, edi, edi`, "-O0 doesn't + constant-fold". It also missed a `je` targeting its own fall-through in about 5 of 12 samples; Sonnet, + Opus 5.5 and Fable 5.1 caught it every time. + - *Haiku 5.5 adaptive (the API default)*: 41/48, 8 errors, but no faster than Sonnet (mean 5.0s at low, + 5.8s at medium, max 17-18s). At `high` effort thinking exhausted `max_tokens=4096` and returned an empty + response on `loop_experienced_assembly` in 2 of 3 runs. + - *Opus 5.5* (thinking cannot be disabled): low 16/16, medium 30/31 reviewed, which is within noise of + Sonnet, at 2-2.5x the cost and 6.6s / 9.6s mean. *Fable 5.1*: 15/16 at $0.059/call, with a 37s max that + breaks the 30s ceiling. - **Prompt caching: evaluated 2026-07 and rejected at current traffic.** ~104 fresh Claude calls/day (CloudWatch, 14-day window), only ~35 hours/fortnight above 12 calls/hour, against a 5-minute cache TTL and a prefix fragmented by language/arch/audience/type. Generous math: ~$0.40 saved per fortnight of ~$22 spend. diff --git a/app/model_costs.py b/app/model_costs.py index e8247d5..0d96282 100644 --- a/app/model_costs.py +++ b/app/model_costs.py @@ -19,7 +19,7 @@ class ModelCost(NamedTuple): # Model family costs in USD per million tokens -# Updated: 2026-09-29 based on https://platform.claude.com/docs/en/about-claude/pricing +# Updated: 2026-10-10 based on https://platform.claude.com/docs/en/about-claude/pricing # # Notes: # - Opus 4.5+ moved to a new lower price tier ($5/$25) and now bundle the 1M @@ -32,8 +32,14 @@ class ModelCost(NamedTuple): MODEL_FAMILIES = { # Claude 5 family. Sonnet 5's launch $2/$10 became its standard price; # the planned 2026-09-01 rise to $3/$15 was cancelled. + "fable-5.1": ModelCost(10.0, 50.0), + "fable-5": ModelCost(10.0, 50.0), "sonnet-5.5": ModelCost(2.0, 10.0), "sonnet-5": ModelCost(2.0, 10.0), + # Haiku 5.5 is the first model with length-tiered pricing: these are the + # rates for prompts up to 100k tokens ($0.50/$2.50 above that). Explain + # requests are capped far below 100k, so the base tier always applies. + "haiku-5.5": ModelCost(0.10, 0.50), "opus-5.5": ModelCost(4.0, 20.0), "opus-5": ModelCost(5.0, 25.0), # Opus 4.5+: new pricing tier, 1M context bundled diff --git a/app/test_model_costs.py b/app/test_model_costs.py index 5f41be9..3f15eb2 100644 --- a/app/test_model_costs.py +++ b/app/test_model_costs.py @@ -93,6 +93,23 @@ def test_sonnet_5_5_cost(self): assert input_cost == 2.0 / 1_000_000 assert output_cost == 10.0 / 1_000_000 + def test_haiku_5_5_cost(self): + """Haiku 5.5 base tier (prompts up to 100k tokens).""" + input_cost, output_cost = get_model_cost("claude-haiku-5-5") + assert input_cost == 0.10 / 1_000_000 + assert output_cost == 0.50 / 1_000_000 + + def test_opus_5_5_cost(self): + input_cost, output_cost = get_model_cost("claude-opus-5-5") + assert input_cost == 4.0 / 1_000_000 + assert output_cost == 20.0 / 1_000_000 + + def test_fable_cost(self): + for model in ("claude-fable-5", "claude-fable-5-1"): + input_cost, output_cost = get_model_cost(model) + assert input_cost == 10.0 / 1_000_000 + assert output_cost == 50.0 / 1_000_000 + def test_opus_4_cost(self): """Test Claude 4 Opus costs (legacy $15/$75 pricing).""" input_cost, output_cost = get_model_cost("claude-opus-4-0")