From bac532988a31f61e8203a3b96b5f9a192836e085 Mon Sep 17 00:00:00 2001 From: Nick Sullivan Date: Sat, 25 Jul 2026 23:28:25 -0500 Subject: [PATCH 1/6] feat(benchmarks): auto-discover new models + public EQ-Bench as its own metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the "ADD, don't just refresh" half of the update loop, plus a public EQ-Bench v3 source, and grows the dataset from 13 to 26 models. The central finding: eqbench.com's public `rubric_0_100` is NOT the same metric as the site's `v3_score`. `v3_score` comes from Nick's own local v3 runs over 22 traits; the public leaderboard scores 17 traits and lands 5-9 points higher on every model present in both: openai/gpt-5.4 local 73.20 public 82.4 +9.20 anthropic/claude-sonnet-4.6 local 71.70 public 80.0 +8.30 anthropic/claude-opus-4.6 local 71.85 public 79.2 +7.35 google/gemini-3.1-pro-... local 68.95 public 74.3 +5.35 google/gemma-4-31b-it local 66.10 public 70.8 +4.70 The public Elo differs too (sonnet-4.6: stored 1876.8 vs public 1714.1). So public values land in their own namespaced fields (public_rubric_0_100, public_elo_norm, public_traits_17, public_source_key) and never overwrite v3_score, v3_traits, or elo. Writing them into v3_score would have silently inflated 13 new rows against the 13 existing ones. Mapping is hand-verified, never fuzzy. Substring matching actively mis-assigns: gpt-5.4-mini→gpt-5.4, glm-5-turbo→GLM-5, grok-4.20→grok-4. Unmapped models get an empty cell, which is honest. 13 models added, all with real public scores and no local v3_score invented: claude-opus-4.8 (83.3, top of the public leaderboard), claude-fable-5 (82.6), gpt-5.5 (82.4), glm-5.2, deepseek-v4-pro/flash, kimi-k2.6/k2.5, glm-5.1, glm-4.7, glm-4.7-flash, gemma-4-26b-a4b, qwen3.5-397b. Verified: 13 pre-existing models diffed field-by-field before/after — zero values changed, zero fields lost. Local v3_score count stays 13. Every public score re-checked against the upstream leaderboard row. Deliberately NOT added (confirmed absent from EQ-Bench, so no real score exists): claude-opus-5, claude-opus-5-fast, grok-4.5, kimi-k3, gpt-5.6-*, gemini-3.6-flash. New flags: --discover (add untracked models with verified EQ entries), --eq-public (fetch public leaderboard fields). Tests 11 → 19. --- model-benchmarks/data/model-data.json | 1519 ++++++++++++++++++-- model-benchmarks/llms.txt | 132 +- model-benchmarks/scripts/eqbench_public.py | 186 +++ model-benchmarks/scripts/fetch-model.py | 97 +- model-benchmarks/tests/test_fetch_model.py | 87 ++ 5 files changed, 1927 insertions(+), 94 deletions(-) create mode 100644 model-benchmarks/scripts/eqbench_public.py diff --git a/model-benchmarks/data/model-data.json b/model-benchmarks/data/model-data.json index 3e1900f..ee882dd 100644 --- a/model-benchmarks/data/model-data.json +++ b/model-benchmarks/data/model-data.json @@ -1,5 +1,5 @@ { - "generated": "2026-07-25T08:58:52Z", + "generated": "2026-07-26T04:27:13Z", "schema_version": "2.0", "sources": { "openrouter": "https://openrouter.ai/api/v1/models", @@ -15,6 +15,91 @@ "blended_cost": "3:1 input-to-output ratio: (3 * input_per_m + output_per_m) / 4" }, "models": [ + { + "id": "anthropic/claude-fable-5", + "canonical_slug": "anthropic/claude-5-fable-20260609", + "name": "Claude Fable 5", + "provider": "Anthropic", + "description": "Claude Fable 5 is a Mythos-class model from Anthropic, built for autonomous knowledge work and coding. It supports text, image, and file inputs with text output, with reasoning support and...", + "context_window": 1000000, + "max_output": 128000, + "modalities": { + "input": [ + "text", + "image", + "file" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Claude", + "pricing": { + "input_per_m": 10.0, + "output_per_m": 50.0, + "cache_read_per_m": 1.0, + "cache_write_per_m": 12.5, + "web_search_per_req": 0.01, + "blended_per_m": 20.0 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Amazon Bedrock", + "Anthropic", + "Azure", + "Google" + ], + "uptime_24h": 98.93, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "claude-fable-5", + "public_rubric_0_100": 82.6, + "public_elo_norm": 2049.7, + "public_traits_17": { + "demonstrated_empathy": 16.12, + "pragmatic_ei": 16.0, + "depth_of_insight": 18.0, + "social_dexterity": 15.15, + "emotional_reasoning": 17.16, + "message_tailoring": 14.69, + "boundary_setting": 14.54, + "safety_conscious": 13.62, + "moralising": 7.62, + "compliant": 9.73, + "challenging": 13.35, + "warmth": 14.04, + "validating": 13.35, + "analytical": 18.04, + "reactive": 9.0, + "conversational": 11.23, + "humanlike": 15.27 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1781007515, + "last_updated": "2026-07-26T04:27:11Z" + }, { "id": "anthropic/claude-haiku-4.5", "canonical_slug": "anthropic/claude-4.5-haiku-20251001", @@ -226,6 +311,28 @@ "theory_of_mind": 13.5, "validating": 13.2, "warmth": 13.6 + }, + "public_source_key": "claude-opus-4-6", + "public_rubric_0_100": 79.2, + "public_elo_norm": 1717.4, + "public_traits_17": { + "demonstrated_empathy": 15.62, + "pragmatic_ei": 15.73, + "depth_of_insight": 17.33, + "social_dexterity": 14.73, + "emotional_reasoning": 16.62, + "message_tailoring": 14.23, + "boundary_setting": 14.27, + "safety_conscious": 13.62, + "moralising": 8.81, + "compliant": 9.85, + "challenging": 13.81, + "warmth": 14.04, + "validating": 13.38, + "analytical": 17.42, + "reactive": 9.42, + "conversational": 11.58, + "humanlike": 14.69 } }, "pinchbench": { @@ -243,7 +350,8 @@ "openrouter": true, "artificial_analysis": false, "eq_bench": false, - "pinchbench": true + "pinchbench": true, + "eq_bench_public": true }, "created": 1770219050, "last_updated": "2026-07-25T08:58:51Z", @@ -253,6 +361,91 @@ "source": "artificial_analysis" } }, + { + "id": "anthropic/claude-opus-4.8", + "canonical_slug": "anthropic/claude-4.8-opus-20260528", + "name": "Claude Opus 4.8", + "provider": "Anthropic", + "description": "Claude Opus 4.8 is Anthropic's most capable generally available model in the Opus family. It supports text, image, and file inputs with text output, with reasoning support and a 1M-token...", + "context_window": 1000000, + "max_output": 128000, + "modalities": { + "input": [ + "text", + "image", + "file" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Claude", + "pricing": { + "input_per_m": 5.0, + "output_per_m": 25.0, + "cache_read_per_m": 0.5, + "cache_write_per_m": 6.25, + "web_search_per_req": 0.01, + "blended_per_m": 10.0 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Amazon Bedrock", + "Anthropic", + "Azure", + "Google" + ], + "uptime_24h": 100, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "claude-opus-4-8", + "public_rubric_0_100": 83.3, + "public_elo_norm": 2029.8, + "public_traits_17": { + "demonstrated_empathy": 16.58, + "pragmatic_ei": 16.54, + "depth_of_insight": 17.8, + "social_dexterity": 15.88, + "emotional_reasoning": 17.18, + "message_tailoring": 15.42, + "boundary_setting": 14.27, + "safety_conscious": 13.5, + "moralising": 6.88, + "compliant": 9.42, + "challenging": 13.35, + "warmth": 14.38, + "validating": 13.58, + "analytical": 17.46, + "reactive": 8.62, + "conversational": 13.15, + "humanlike": 15.73 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1779905091, + "last_updated": "2026-07-26T04:27:11Z" + }, { "id": "anthropic/claude-sonnet-4.6", "canonical_slug": "anthropic/claude-4.6-sonnet-20260217", @@ -351,6 +544,28 @@ "theory_of_mind": 13.6, "validating": 12.9, "warmth": 13.4 + }, + "public_source_key": "claude-sonnet-4-6", + "public_rubric_0_100": 80.0, + "public_elo_norm": 1714.1, + "public_traits_17": { + "demonstrated_empathy": 16.15, + "pragmatic_ei": 16.0, + "depth_of_insight": 17.29, + "social_dexterity": 15.19, + "emotional_reasoning": 16.64, + "message_tailoring": 14.65, + "boundary_setting": 13.77, + "safety_conscious": 13.35, + "moralising": 6.08, + "compliant": 9.88, + "challenging": 12.58, + "warmth": 13.77, + "validating": 13.27, + "analytical": 17.46, + "reactive": 8.88, + "conversational": 11.31, + "humanlike": 15.35 } }, "pinchbench": { @@ -368,7 +583,8 @@ "openrouter": true, "artificial_analysis": false, "eq_bench": false, - "pinchbench": true + "pinchbench": true, + "eq_bench_public": true }, "created": 1771342990, "last_updated": "2026-07-25T08:58:51Z", @@ -378,6 +594,202 @@ "source": "artificial_analysis" } }, + { + "id": "deepseek/deepseek-v4-flash", + "canonical_slug": "deepseek/deepseek-v4-flash-20260423", + "name": "DeepSeek V4 Flash", + "provider": "DeepSeek", + "description": "DeepSeek V4 Flash is an efficiency-optimized Mixture-of-Experts model from DeepSeek with 284B total parameters and 13B activated parameters, supporting a 1M-token context window. It is designed for fast inference and...", + "context_window": 1048576, + "max_output": 393216, + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "DeepSeek", + "pricing": { + "input_per_m": 0.14, + "output_per_m": 0.28, + "cache_read_per_m": 0.028, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 0.175 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "AkashML", + "Alibaba", + "Ambient", + "AtlasCloud", + "Baidu", + "Cloudflare", + "CoreWeave", + "DeepInfra", + "DeepSeek", + "DigitalOcean", + "Fireworks", + "GMICloud", + "Io Net", + "Ionstream", + "Morph", + "Novita", + "Parasail", + "SiliconFlow", + "StreamLake", + "Venice" + ], + "uptime_24h": 100.0, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "deepseek-ai/DeepSeek-V4-Flash", + "public_rubric_0_100": 72.5, + "public_elo_norm": 1491.3, + "public_traits_17": { + "demonstrated_empathy": 14.65, + "pragmatic_ei": 14.04, + "depth_of_insight": 16.07, + "social_dexterity": 13.31, + "emotional_reasoning": 15.18, + "message_tailoring": 12.73, + "boundary_setting": 12.12, + "safety_conscious": 11.88, + "moralising": 8.5, + "compliant": 11.69, + "challenging": 11.38, + "warmth": 13.88, + "validating": 13.58, + "analytical": 16.54, + "reactive": 10.73, + "conversational": 11.54, + "humanlike": 13.88 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1777000666, + "last_updated": "2026-07-26T04:27:11Z" + }, + { + "id": "deepseek/deepseek-v4-pro", + "canonical_slug": "deepseek/deepseek-v4-pro-20260423", + "name": "DeepSeek V4 Pro", + "provider": "DeepSeek", + "description": "DeepSeek V4 Pro is a large-scale Mixture-of-Experts model from DeepSeek with 1.6T total parameters and 49B activated parameters, supporting a 1M-token context window. It is designed for advanced reasoning, coding,...", + "context_window": 1048576, + "max_output": 384000, + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "DeepSeek", + "pricing": { + "input_per_m": 0.435, + "output_per_m": 0.87, + "cache_read_per_m": 0.0036, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 0.5437 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Alibaba", + "AtlasCloud", + "Baidu", + "BaseTen", + "CoreWeave", + "DeepInfra", + "DeepSeek", + "DigitalOcean", + "Fireworks", + "GMICloud", + "Ionstream", + "Novita", + "Parasail", + "SiliconFlow", + "StreamLake", + "Together", + "Venice", + "Wafer" + ], + "uptime_24h": 99.95, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "deepseek-ai/DeepSeek-V4-Pro", + "public_rubric_0_100": 77.6, + "public_elo_norm": 1570.2, + "public_traits_17": { + "demonstrated_empathy": 15.73, + "pragmatic_ei": 15.04, + "depth_of_insight": 17.29, + "social_dexterity": 14.5, + "emotional_reasoning": 16.38, + "message_tailoring": 13.92, + "boundary_setting": 12.92, + "safety_conscious": 12.58, + "moralising": 8.5, + "compliant": 11.31, + "challenging": 11.96, + "warmth": 14.38, + "validating": 13.69, + "analytical": 17.58, + "reactive": 10.23, + "conversational": 12.04, + "humanlike": 15.23 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1777000679, + "last_updated": "2026-07-26T04:27:11Z" + }, { "id": "google/gemini-3.1-pro-preview-20260219", "canonical_slug": "google/gemini-3.1-pro-preview-20260219", @@ -471,6 +883,28 @@ "theory_of_mind": 12.8, "validating": 14.4, "warmth": 13.8 + }, + "public_source_key": "gemini-3.1-pro-preview", + "public_rubric_0_100": 74.3, + "public_elo_norm": 1537.7, + "public_traits_17": { + "demonstrated_empathy": 15.19, + "pragmatic_ei": 14.23, + "depth_of_insight": 16.6, + "social_dexterity": 13.35, + "emotional_reasoning": 15.6, + "message_tailoring": 13.38, + "boundary_setting": 13.42, + "safety_conscious": 12.92, + "moralising": 9.62, + "compliant": 11.42, + "challenging": 12.31, + "warmth": 13.92, + "validating": 14.0, + "analytical": 17.38, + "reactive": 11.54, + "conversational": 11.15, + "humanlike": 14.04 } }, "pinchbench": { @@ -488,7 +922,8 @@ "openrouter": true, "artificial_analysis": false, "eq_bench": false, - "pinchbench": false + "pinchbench": false, + "eq_bench_public": true }, "created": 1771509627, "last_updated": "2026-07-25T08:58:51Z", @@ -500,11 +935,11 @@ } }, { - "id": "google/gemma-4-31b-it", - "canonical_slug": "google/gemma-4-31b-it-20260402", - "name": "Gemma 4 31B", + "id": "google/gemma-4-26b-a4b-it", + "canonical_slug": "google/gemma-4-26b-a4b-it-20260403", + "name": "Gemma 4 26B A4B ", "provider": "Google", - "description": "Gemma 4 31B Instruct is Google DeepMind's 30.7B dense multimodal model supporting text and image input with text output. Features a 256K token context window, configurable thinking/reasoning mode, native function...", + "description": "Gemma 4 26B A4B IT is an instruction-tuned Mixture-of-Experts (MoE) model from Google DeepMind. Despite 25.2B total parameters, only 3.8B activate per token during inference \u2014 delivering near-31B quality at...", "context_window": 262144, "max_output": 262144, "modalities": { @@ -519,12 +954,12 @@ }, "tokenizer": "Gemma", "pricing": { - "input_per_m": 0.14, - "output_per_m": 0.4, - "cache_read_per_m": null, + "input_per_m": 0.12, + "output_per_m": 0.35, + "cache_read_per_m": 0.05, "cache_write_per_m": null, "web_search_per_req": null, - "blended_per_m": 0.205 + "blended_per_m": 0.1775 }, "capabilities": { "tool_use": true, @@ -536,52 +971,142 @@ }, "endpoint_stats": { "providers": [ - "Cerebras", - "Chutes", - "CoreWeave", - "Crusoe", + "Cloudflare", "DeepInfra", - "Friendli", - "ModelRun", - "Morph", + "Google", + "Ionstream", + "NextBit", "Novita", - "OpenInference", "Parasail", - "Phala", - "SambaNova", "SiliconFlow", - "Together", "Venice" ], - "uptime_24h": 99.99, + "uptime_24h": 99.92, "latency_ms": null, "throughput_tps": null }, - "scores": { - "reasoning": 52.3, - "coding": 38.7, - "agentic": 75.6 - }, + "scores": {}, "benchmarks": { - "artificial_analysis": { - "intelligence_index": 39.2, - "coding_index": 38.7, - "gpqa": 0.857, - "hle": 0.227, - "scicode": 0.434, - "ifbench": 0.756, - "terminalbench_hard": 0.364 - }, + "artificial_analysis": {}, "eq_bench": { - "v3_score": 66.1, - "v3_traits": { - "analytical": 16.2, - "boundary_setting": 13.1, - "challenging": 12.2, - "compliant": 9.3, - "conversational": 10.6, - "correctness": 12.9, - "demonstrated_empathy": 14.0, + "public_source_key": "google/gemma-4-26B-A4B-it", + "public_rubric_0_100": 70.0, + "public_elo_norm": 1430.6, + "public_traits_17": { + "demonstrated_empathy": 14.85, + "pragmatic_ei": 14.54, + "depth_of_insight": 15.42, + "social_dexterity": 13.42, + "emotional_reasoning": 14.51, + "message_tailoring": 12.96, + "boundary_setting": 13.35, + "safety_conscious": 12.58, + "moralising": 9.38, + "compliant": 11.69, + "challenging": 11.54, + "warmth": 13.0, + "validating": 14.08, + "analytical": 17.42, + "reactive": 10.81, + "conversational": 9.62, + "humanlike": 12.12 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1775227989, + "last_updated": "2026-07-26T04:27:11Z" + }, + { + "id": "google/gemma-4-31b-it", + "canonical_slug": "google/gemma-4-31b-it-20260402", + "name": "Gemma 4 31B", + "provider": "Google", + "description": "Gemma 4 31B Instruct is Google DeepMind's 30.7B dense multimodal model supporting text and image input with text output. Features a 256K token context window, configurable thinking/reasoning mode, native function...", + "context_window": 262144, + "max_output": 262144, + "modalities": { + "input": [ + "image", + "text", + "video" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Gemma", + "pricing": { + "input_per_m": 0.14, + "output_per_m": 0.4, + "cache_read_per_m": null, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 0.205 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Cerebras", + "Chutes", + "CoreWeave", + "Crusoe", + "DeepInfra", + "Friendli", + "ModelRun", + "Morph", + "Novita", + "OpenInference", + "Parasail", + "Phala", + "SambaNova", + "SiliconFlow", + "Together", + "Venice" + ], + "uptime_24h": 99.99, + "latency_ms": null, + "throughput_tps": null + }, + "scores": { + "reasoning": 52.3, + "coding": 38.7, + "agentic": 75.6 + }, + "benchmarks": { + "artificial_analysis": { + "intelligence_index": 39.2, + "coding_index": 38.7, + "gpqa": 0.857, + "hle": 0.227, + "scicode": 0.434, + "ifbench": 0.756, + "terminalbench_hard": 0.364 + }, + "eq_bench": { + "v3_score": 66.1, + "v3_traits": { + "analytical": 16.2, + "boundary_setting": 13.1, + "challenging": 12.2, + "compliant": 9.3, + "conversational": 10.6, + "correctness": 12.9, + "demonstrated_empathy": 14.0, "depth_of_insight": 14.2, "emotional_reasoning": 13.6, "humanlike": 11.8, @@ -597,6 +1122,28 @@ "theory_of_mind": 11.7, "validating": 12.5, "warmth": 11.7 + }, + "public_source_key": "google/gemma-4-31B-it", + "public_rubric_0_100": 70.8, + "public_elo_norm": 1434.1, + "public_traits_17": { + "demonstrated_empathy": 14.73, + "pragmatic_ei": 15.08, + "depth_of_insight": 15.38, + "social_dexterity": 13.88, + "emotional_reasoning": 14.62, + "message_tailoring": 13.65, + "boundary_setting": 14.31, + "safety_conscious": 12.62, + "moralising": 10.19, + "compliant": 10.62, + "challenging": 12.85, + "warmth": 12.0, + "validating": 12.38, + "analytical": 17.31, + "reactive": 10.88, + "conversational": 10.46, + "humanlike": 12.77 } }, "pinchbench": {}, @@ -609,7 +1156,8 @@ "openrouter": true, "artificial_analysis": false, "eq_bench": false, - "pinchbench": false + "pinchbench": false, + "eq_bench_public": true }, "created": 1775148486, "last_updated": "2026-07-25T08:58:51Z", @@ -736,6 +1284,197 @@ "source": "artificial_analysis" } }, + { + "id": "moonshotai/kimi-k2.5", + "canonical_slug": "moonshotai/kimi-k2.5-0127", + "name": "Kimi K2.5", + "provider": "MoonshotAI", + "description": "Kimi K2.5 is Moonshot AI's native multimodal model, delivering state-of-the-art visual coding capability and a self-directed agent swarm paradigm. Built on Kimi K2 with continued pretraining over approximately 15T mixed...", + "context_window": 262144, + "max_output": 262144, + "modalities": { + "input": [ + "text", + "image" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Other", + "pricing": { + "input_per_m": 0.57, + "output_per_m": 2.85, + "cache_read_per_m": 0.095, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 1.14 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "AtlasCloud", + "Chutes", + "DeepInfra", + "DigitalOcean", + "Moonshot AI", + "Novita", + "Phala", + "SiliconFlow", + "StreamLake", + "Venice" + ], + "uptime_24h": 99.96, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "moonshotai/Kimi-K2.5", + "public_rubric_0_100": 72.5, + "public_elo_norm": 1544.7, + "public_traits_17": { + "demonstrated_empathy": 14.58, + "pragmatic_ei": 13.54, + "depth_of_insight": 16.78, + "social_dexterity": 12.65, + "emotional_reasoning": 15.44, + "message_tailoring": 12.23, + "boundary_setting": 14.38, + "safety_conscious": 13.77, + "moralising": 11.12, + "compliant": 10.04, + "challenging": 14.15, + "warmth": 12.23, + "validating": 12.19, + "analytical": 17.77, + "reactive": 11.54, + "conversational": 9.81, + "humanlike": 13.19 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1769487076, + "last_updated": "2026-07-26T04:27:11Z" + }, + { + "id": "moonshotai/kimi-k2.6", + "canonical_slug": "moonshotai/kimi-k2.6-20260420", + "name": "Kimi K2.6", + "provider": "MoonshotAI", + "description": "Kimi K2.6 is Moonshot AI's next-generation multimodal model, designed for long-horizon coding, coding-driven UI/UX generation, and multi-agent orchestration. It handles complex end-to-end coding tasks across Python, Rust, and Go, and...", + "context_window": 262144, + "max_output": 262144, + "modalities": { + "input": [ + "text", + "image" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Other", + "pricing": { + "input_per_m": 0.646, + "output_per_m": 2.72, + "cache_read_per_m": 0.1088, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 1.1645 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "AtlasCloud", + "Baidu", + "BaseTen", + "Chutes", + "Cloudflare", + "CoreWeave", + "Crusoe", + "Decart", + "DeepInfra", + "DigitalOcean", + "Fireworks", + "Inceptron", + "ModelRun", + "Moonshot AI", + "Novita", + "Parasail", + "Phala", + "SiliconFlow", + "StreamLake", + "Together", + "Venice" + ], + "uptime_24h": 99.98, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "moonshotai/Kimi-K2.6", + "public_rubric_0_100": 76.6, + "public_elo_norm": 1561.2, + "public_traits_17": { + "demonstrated_empathy": 15.08, + "pragmatic_ei": 13.46, + "depth_of_insight": 17.64, + "social_dexterity": 12.65, + "emotional_reasoning": 16.67, + "message_tailoring": 12.5, + "boundary_setting": 14.69, + "safety_conscious": 13.5, + "moralising": 12.0, + "compliant": 10.58, + "challenging": 13.88, + "warmth": 12.46, + "validating": 12.31, + "analytical": 18.04, + "reactive": 11.96, + "conversational": 9.69, + "humanlike": 14.08 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1776699402, + "last_updated": "2026-07-26T04:27:12Z" + }, { "id": "openai/gpt-5.4", "canonical_slug": "openai/gpt-5.4-20260305", @@ -827,6 +1566,28 @@ "theory_of_mind": 14.7, "validating": 13.1, "warmth": 11.8 + }, + "public_source_key": "gpt-5.4", + "public_rubric_0_100": 82.4, + "public_elo_norm": 1562.9, + "public_traits_17": { + "demonstrated_empathy": 15.77, + "pragmatic_ei": 16.62, + "depth_of_insight": 17.84, + "social_dexterity": 15.19, + "emotional_reasoning": 17.18, + "message_tailoring": 14.5, + "boundary_setting": 15.5, + "safety_conscious": 14.27, + "moralising": 7.58, + "compliant": 10.19, + "challenging": 12.88, + "warmth": 12.15, + "validating": 13.65, + "analytical": 17.96, + "reactive": 8.65, + "conversational": 10.46, + "humanlike": 14.31 } }, "pinchbench": { @@ -844,7 +1605,8 @@ "openrouter": true, "artificial_analysis": false, "eq_bench": false, - "pinchbench": false + "pinchbench": false, + "eq_bench_public": true }, "created": 1772734352, "last_updated": "2026-07-25T08:58:52Z", @@ -899,58 +1661,233 @@ "latency_ms": null, "throughput_tps": null }, - "scores": { - "reasoning": 48, - "coding": 51, - "agentic": 56 - }, + "scores": { + "reasoning": 48, + "coding": 51, + "agentic": 56 + }, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "v3_score": 68.65, + "v3_traits": { + "analytical": 18.0, + "boundary_setting": 15.5, + "challenging": 12.0, + "compliant": 8.2, + "conversational": 8.7, + "correctness": 14.5, + "demonstrated_empathy": 13.4, + "depth_of_insight": 15.4, + "emotional_reasoning": 14.9, + "humanlike": 11.4, + "intellectual_grounding": 12.6, + "message_tailoring": 10.7, + "moralising": 6.0, + "pragmatic_ei": 13.1, + "reactive": 7.7, + "safety_conscious": 15.2, + "social_dexterity": 11.2, + "subtext_identification": 16.0, + "sycophantic": 2.7, + "theory_of_mind": 14.2, + "validating": 10.9, + "warmth": 8.6 + } + }, + "pinchbench": {}, + "arena": { + "elo": 1455 + } + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false + }, + "created": 1773748178, + "last_updated": "2026-07-25T08:58:52Z", + "notes": "Agentic score from AA Agentic Index (not PinchBench-based)", + "speed": { + "output_tokens_per_sec": 186, + "source": "artificial_analysis" + } + }, + { + "id": "openai/gpt-5.5", + "canonical_slug": "openai/gpt-5.5-20260423", + "name": "GPT-5.5", + "provider": "OpenAI", + "description": "GPT-5.5 is OpenAI\u2019s frontier model designed for complex professional workloads, building on GPT-5.4 with stronger reasoning, higher reliability, and improved token efficiency on hard tasks. It features a 1M+ token...", + "context_window": 1050000, + "max_output": 128000, + "modalities": { + "input": [ + "file", + "image", + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "GPT", + "pricing": { + "input_per_m": 5.0, + "output_per_m": 30.0, + "cache_read_per_m": 0.5, + "cache_write_per_m": null, + "web_search_per_req": 0.01, + "blended_per_m": 11.25 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Azure", + "OpenAI" + ], + "uptime_24h": 99.99, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "gpt-5.5", + "public_rubric_0_100": 82.4, + "public_elo_norm": 1576.9, + "public_traits_17": { + "demonstrated_empathy": 15.88, + "pragmatic_ei": 16.5, + "depth_of_insight": 17.67, + "social_dexterity": 15.23, + "emotional_reasoning": 17.07, + "message_tailoring": 14.46, + "boundary_setting": 15.77, + "safety_conscious": 14.38, + "moralising": 8.46, + "compliant": 9.65, + "challenging": 13.38, + "warmth": 13.23, + "validating": 13.58, + "analytical": 17.69, + "reactive": 8.62, + "conversational": 10.42, + "humanlike": 13.69 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1777051893, + "last_updated": "2026-07-26T04:27:12Z" + }, + { + "id": "qwen/qwen3.5-397b-a17b", + "canonical_slug": "qwen/qwen3.5-397b-a17b-20260216", + "name": "Qwen3.5 397B A17B", + "provider": "Qwen", + "description": "The Qwen3.5 series 397B-A17B native vision-language model is built on a hybrid architecture that integrates a linear attention mechanism with a sparse mixture-of-experts model, achieving higher inference efficiency. It delivers...", + "context_window": 262144, + "max_output": 65536, + "modalities": { + "input": [ + "text", + "image", + "video" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Qwen3", + "pricing": { + "input_per_m": 0.39, + "output_per_m": 2.34, + "cache_read_per_m": null, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 0.8775 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Alibaba", + "AtlasCloud", + "Chutes", + "DeepInfra", + "DigitalOcean", + "GMICloud", + "Novita", + "Parasail", + "Phala", + "StreamLake", + "Venice" + ], + "uptime_24h": 99.95, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, "benchmarks": { "artificial_analysis": {}, "eq_bench": { - "v3_score": 68.65, - "v3_traits": { - "analytical": 18.0, - "boundary_setting": 15.5, - "challenging": 12.0, - "compliant": 8.2, - "conversational": 8.7, - "correctness": 14.5, - "demonstrated_empathy": 13.4, - "depth_of_insight": 15.4, - "emotional_reasoning": 14.9, - "humanlike": 11.4, - "intellectual_grounding": 12.6, - "message_tailoring": 10.7, - "moralising": 6.0, - "pragmatic_ei": 13.1, - "reactive": 7.7, - "safety_conscious": 15.2, - "social_dexterity": 11.2, - "subtext_identification": 16.0, - "sycophantic": 2.7, - "theory_of_mind": 14.2, - "validating": 10.9, - "warmth": 8.6 + "public_source_key": "Qwen/Qwen3.5-397B-A17B", + "public_rubric_0_100": 71.3, + "public_elo_norm": 1413.1, + "public_traits_17": { + "demonstrated_empathy": 14.35, + "pragmatic_ei": 14.0, + "depth_of_insight": 15.87, + "social_dexterity": 12.69, + "emotional_reasoning": 15.0, + "message_tailoring": 12.23, + "boundary_setting": 13.65, + "safety_conscious": 12.88, + "moralising": 9.85, + "compliant": 12.12, + "challenging": 11.23, + "warmth": 13.0, + "validating": 13.58, + "analytical": 16.81, + "reactive": 10.92, + "conversational": 9.42, + "humanlike": 12.27 } }, - "pinchbench": {}, - "arena": { - "elo": 1455 - } + "pinchbench": {} }, "sources": { "openrouter": true, "artificial_analysis": false, "eq_bench": false, - "pinchbench": false + "pinchbench": false, + "eq_bench_public": true }, - "created": 1773748178, - "last_updated": "2026-07-25T08:58:52Z", - "notes": "Agentic score from AA Agentic Index (not PinchBench-based)", - "speed": { - "output_tokens_per_sec": 186, - "source": "artificial_analysis" - } + "created": 1771223018, + "last_updated": "2026-07-26T04:27:12Z" }, { "id": "qwen/qwen3.6-plus:free", @@ -1247,6 +2184,177 @@ "notes": "Delisted from OpenRouter. Pricing and availability are frozen as of the last successful fetch; successor is xiaomi/mimo-v2.5-pro, which has not been benchmarked here.", "delisted": true }, + { + "id": "z-ai/glm-4.7", + "canonical_slug": "z-ai/glm-4.7-20251222", + "name": "GLM 4.7", + "provider": "Z.ai", + "description": "GLM-4.7 is Z.ai\u2019s latest flagship model, featuring upgrades in two key areas: enhanced programming capabilities and more stable multi-step reasoning/execution. It demonstrates significant improvements in executing complex agent tasks while...", + "context_window": 204800, + "max_output": 131072, + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Other", + "pricing": { + "input_per_m": 0.4, + "output_per_m": 1.75, + "cache_read_per_m": 0.08, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 0.7375 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "AtlasCloud", + "Cerebras", + "DeepInfra", + "Google", + "Novita", + "Phala", + "StreamLake", + "Venice", + "Z.AI" + ], + "uptime_24h": 99.98, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "zai-org/GLM-4.7", + "public_rubric_0_100": 68.9, + "public_elo_norm": 1428.3, + "public_traits_17": { + "demonstrated_empathy": 13.96, + "pragmatic_ei": 13.85, + "depth_of_insight": 15.49, + "social_dexterity": 12.54, + "emotional_reasoning": 14.47, + "message_tailoring": 12.23, + "boundary_setting": 13.15, + "safety_conscious": 12.58, + "moralising": 9.08, + "compliant": 11.46, + "challenging": 11.38, + "warmth": 11.88, + "validating": 13.15, + "analytical": 17.08, + "reactive": 11.42, + "conversational": 10.81, + "humanlike": 13.27 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1766378014, + "last_updated": "2026-07-26T04:27:12Z" + }, + { + "id": "z-ai/glm-4.7-flash", + "canonical_slug": "z-ai/glm-4.7-flash-20260119", + "name": "GLM 4.7 Flash", + "provider": "Z.ai", + "description": "As a 30B-class SOTA model, GLM-4.7-Flash offers a new option that balances performance and efficiency. It is further optimized for agentic coding use cases, strengthening coding capabilities, long-horizon task planning,...", + "context_window": 202752, + "max_output": 16384, + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Other", + "pricing": { + "input_per_m": 0.06, + "output_per_m": 0.4, + "cache_read_per_m": 0.01, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 0.145 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Cloudflare", + "DeepInfra", + "Novita", + "Venice" + ], + "uptime_24h": 99.39, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "zai-org/GLM-4.7-Flash", + "public_rubric_0_100": 52.5, + "public_elo_norm": 1083.0, + "public_traits_17": { + "demonstrated_empathy": 10.72, + "pragmatic_ei": 9.96, + "depth_of_insight": 12.55, + "social_dexterity": 8.72, + "emotional_reasoning": 11.5, + "message_tailoring": 8.84, + "boundary_setting": 12.52, + "safety_conscious": 10.12, + "moralising": 11.44, + "compliant": 11.2, + "challenging": 12.04, + "warmth": 9.36, + "validating": 11.2, + "analytical": 15.88, + "reactive": 13.0, + "conversational": 8.0, + "humanlike": 9.8 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1768833913, + "last_updated": "2026-07-26T04:27:12Z" + }, { "id": "z-ai/glm-5-turbo", "canonical_slug": "z-ai/glm-5-turbo-20260315", @@ -1367,6 +2475,212 @@ "source": "openrouter" } }, + { + "id": "z-ai/glm-5.1", + "canonical_slug": "z-ai/glm-5.1-20260406", + "name": "GLM 5.1", + "provider": "Z.ai", + "description": "GLM-5.1 delivers a major leap in coding capability, with particularly significant gains in handling long-horizon tasks. Unlike previous models built around minute-level interactions, GLM-5.1 can work independently and continuously on...", + "context_window": 204800, + "max_output": 128000, + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Other", + "pricing": { + "input_per_m": 0.966, + "output_per_m": 3.036, + "cache_read_per_m": 0.1794, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 1.4835 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "AtlasCloud", + "Baidu", + "Chutes", + "CoreWeave", + "Crusoe", + "DeepInfra", + "DigitalOcean", + "Fireworks", + "Friendli", + "GMICloud", + "Nebius", + "Novita", + "Parasail", + "Phala", + "SiliconFlow", + "StreamLake", + "Venice", + "Wafer", + "Z.AI" + ], + "uptime_24h": 99.98, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "zai-org/GLM-5.1", + "public_rubric_0_100": 75.2, + "public_elo_norm": 1566.5, + "public_traits_17": { + "demonstrated_empathy": 14.42, + "pragmatic_ei": 14.0, + "depth_of_insight": 17.04, + "social_dexterity": 13.23, + "emotional_reasoning": 16.09, + "message_tailoring": 12.5, + "boundary_setting": 13.27, + "safety_conscious": 12.88, + "moralising": 9.92, + "compliant": 9.54, + "challenging": 13.58, + "warmth": 12.62, + "validating": 12.35, + "analytical": 17.04, + "reactive": 10.46, + "conversational": 11.38, + "humanlike": 14.69 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1775578025, + "last_updated": "2026-07-26T04:27:12Z" + }, + { + "id": "z-ai/glm-5.2", + "canonical_slug": "z-ai/glm-5.2-20260616", + "name": "GLM 5.2", + "provider": "Z.ai", + "description": "GLM 5.2 is a large-scale reasoning model from Z.ai. It supports text input and output with a 1M-token context window, and is suited for long-horizon agent workflows, project-level software engineering,...", + "context_window": 1048576, + "max_output": 131072, + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Other", + "pricing": { + "input_per_m": 0.714, + "output_per_m": 2.244, + "cache_read_per_m": 0.1326, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 1.0965 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "AkashML", + "Alibaba", + "Ambient", + "AtlasCloud", + "Baidu", + "BaseTen", + "Chutes", + "Cloudflare", + "CoreWeave", + "Decart", + "DeepInfra", + "DigitalOcean", + "Fireworks", + "Friendli", + "GMICloud", + "Inceptron", + "Io Net", + "Ionstream", + "Morph", + "Novita", + "Parasail", + "Phala", + "Sail Research", + "SiliconFlow", + "StreamLake", + "Together", + "Venice", + "Wafer", + "Z.AI" + ], + "uptime_24h": 100, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "zai-org/GLM-5.2", + "public_rubric_0_100": 78.7, + "public_elo_norm": 1575.1, + "public_traits_17": { + "demonstrated_empathy": 15.73, + "pragmatic_ei": 15.54, + "depth_of_insight": 17.29, + "social_dexterity": 14.73, + "emotional_reasoning": 16.36, + "message_tailoring": 14.31, + "boundary_setting": 14.04, + "safety_conscious": 12.85, + "moralising": 8.54, + "compliant": 10.15, + "challenging": 13.69, + "warmth": 13.23, + "validating": 13.12, + "analytical": 17.54, + "reactive": 10.31, + "conversational": 12.12, + "humanlike": 15.42 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1781631930, + "last_updated": "2026-07-26T04:27:12Z" + }, { "id": "x-ai/grok-4.20-20260309", "canonical_slug": "x-ai/grok-4.20-20260309", @@ -1446,6 +2760,28 @@ "theory_of_mind": 14.2, "validating": 13.5, "warmth": 13.5 + }, + "public_source_key": "grok-4.20-beta", + "public_rubric_0_100": 55.8, + "public_elo_norm": 851.5, + "public_traits_17": { + "demonstrated_empathy": 7.38, + "pragmatic_ei": 7.54, + "depth_of_insight": 12.49, + "social_dexterity": 6.85, + "emotional_reasoning": 11.69, + "message_tailoring": 6.31, + "boundary_setting": 15.31, + "safety_conscious": 14.38, + "moralising": 12.54, + "compliant": 6.96, + "challenging": 11.85, + "warmth": 6.77, + "validating": 7.73, + "analytical": 10.88, + "reactive": 13.19, + "conversational": 5.77, + "humanlike": 7.65 } }, "pinchbench": { @@ -1464,7 +2800,8 @@ "openrouter": true, "artificial_analysis": false, "eq_bench": false, - "pinchbench": false + "pinchbench": false, + "eq_bench_public": true }, "created": 1774979019, "last_updated": "2026-07-25T08:58:52Z", diff --git a/model-benchmarks/llms.txt b/model-benchmarks/llms.txt index b873df7..7734ca7 100644 --- a/model-benchmarks/llms.txt +++ b/model-benchmarks/llms.txt @@ -1,11 +1,21 @@ # HeartCentered AI — LLM Model Benchmarks -# Generated: 2026-07-25T08:58:52Z +# Generated: 2026-07-26T04:27:13Z # Full data: https://heartcentered.ai/model-benchmarks/data/model-data.json # Web UI: https://heartcentered.ai/model-benchmarks/ # Current-generation LLM models ranked by community usage on OpenRouter. # Pricing is per million tokens. Capabilities derived from API metadata. +## Claude Fable 5 +Provider: Anthropic +ID: anthropic/claude-fable-5 +Context: 1,000,000 tokens +Max output: 128,000 tokens +Pricing: $10.00 input / $50.00 output per 1M tokens (blended: $20.00) +Capabilities: tool use, reasoning, vision, web search, structured output +Available via: Amazon Bedrock, Anthropic, Azure, Google +Description: Claude Fable 5 is a Mythos-class model from Anthropic, built for autonomous knowledge work and coding. It supports text, image, and file inputs with text output, with reasoning support and... + ## Claude Haiku 4.5 Provider: Anthropic ID: anthropic/claude-haiku-4.5 @@ -37,6 +47,16 @@ EQ-Bench Elo: 1857.8 PinchBench: 93.3% best, 83.1% avg (19 runs) Description: Opus 4.6 is Anthropic’s strongest model for coding and long-running professional tasks. It is built for agents that operate across entire workflows rather than single prompts, making it especially eff +## Claude Opus 4.8 +Provider: Anthropic +ID: anthropic/claude-opus-4.8 +Context: 1,000,000 tokens +Max output: 128,000 tokens +Pricing: $5.00 input / $25.00 output per 1M tokens (blended: $10.00) +Capabilities: tool use, reasoning, vision, web search, structured output +Available via: Amazon Bedrock, Anthropic, Azure, Google +Description: Claude Opus 4.8 is Anthropic's most capable generally available model in the Opus family. It supports text, image, and file inputs with text output, with reasoning support and a 1M-token... + ## Claude Sonnet 4.6 Provider: Anthropic ID: anthropic/claude-sonnet-4.6 @@ -53,6 +73,26 @@ EQ-Bench Elo: 1876.8 PinchBench: 88.0% best, 81.1% avg (19 runs) Description: Sonnet 4.6 is Anthropic's most capable Sonnet-class model yet, with frontier performance across coding, agents, and professional work. It excels at iterative development, complex codebase navigation, +## DeepSeek V4 Flash +Provider: DeepSeek +ID: deepseek/deepseek-v4-flash +Context: 1,048,576 tokens +Max output: 393,216 tokens +Pricing: $0.14 input / $0.28 output per 1M tokens (blended: $0.17) +Capabilities: tool use, reasoning, structured output +Available via: AkashML, Alibaba, Ambient, AtlasCloud, Baidu, Cloudflare, CoreWeave, DeepInfra, DeepSeek, DigitalOcean, Fireworks, GMICloud, Io Net, Ionstream, Morph, Novita, Parasail, SiliconFlow, StreamLake, Venice +Description: DeepSeek V4 Flash is an efficiency-optimized Mixture-of-Experts model from DeepSeek with 284B total parameters and 13B activated parameters, supporting a 1M-token context window. It is designed for fa + +## DeepSeek V4 Pro +Provider: DeepSeek +ID: deepseek/deepseek-v4-pro +Context: 1,048,576 tokens +Max output: 384,000 tokens +Pricing: $0.43 input / $0.87 output per 1M tokens (blended: $0.54) +Capabilities: tool use, reasoning, structured output +Available via: Alibaba, AtlasCloud, Baidu, BaseTen, CoreWeave, DeepInfra, DeepSeek, DigitalOcean, Fireworks, GMICloud, Ionstream, Novita, Parasail, SiliconFlow, StreamLake, Together, Venice, Wafer +Description: DeepSeek V4 Pro is a large-scale Mixture-of-Experts model from DeepSeek with 1.6T total parameters and 49B activated parameters, supporting a 1M-token context window. It is designed for advanced reaso + ## Gemini 3.1 Pro Preview Provider: Google ID: google/gemini-3.1-pro-preview-20260219 @@ -69,6 +109,16 @@ EQ-Bench Elo: 1548.7 PinchBench: 86.7% best, 77.0% avg (15 runs) Description: Gemini 3.1 Pro Preview is Google’s frontier reasoning model, delivering enhanced software engineering performance, improved agentic reliability, and more efficient token usage across complex workflows +## Gemma 4 26B A4B +Provider: Google +ID: google/gemma-4-26b-a4b-it +Context: 262,144 tokens +Max output: 262,144 tokens +Pricing: $0.12 input / $0.35 output per 1M tokens (blended: $0.18) +Capabilities: tool use, reasoning, vision, structured output +Available via: Cloudflare, DeepInfra, Google, Ionstream, NextBit, Novita, Parasail, SiliconFlow, Venice +Description: Gemma 4 26B A4B IT is an instruction-tuned Mixture-of-Experts (MoE) model from Google DeepMind. Despite 25.2B total parameters, only 3.8B activate per token during inference — delivering near-31B qual + ## Gemma 4 31B Provider: Google ID: google/gemma-4-31b-it @@ -98,6 +148,26 @@ EQ-Bench v3 Score: 68.75/100 PinchBench: 89.8% best, 83.2% avg (11 runs) Description: MiniMax-M2.7 is a next-generation large language model designed for autonomous, real-world productivity and continuous improvement. Built to actively participate in its own evolution, M2.7 integrates +## Kimi K2.5 +Provider: MoonshotAI +ID: moonshotai/kimi-k2.5 +Context: 262,144 tokens +Max output: 262,144 tokens +Pricing: $0.57 input / $2.85 output per 1M tokens (blended: $1.14) +Capabilities: tool use, reasoning, vision, structured output +Available via: AtlasCloud, Chutes, DeepInfra, DigitalOcean, Moonshot AI, Novita, Phala, SiliconFlow, StreamLake, Venice +Description: Kimi K2.5 is Moonshot AI's native multimodal model, delivering state-of-the-art visual coding capability and a self-directed agent swarm paradigm. Built on Kimi K2 with continued pretraining over appr + +## Kimi K2.6 +Provider: MoonshotAI +ID: moonshotai/kimi-k2.6 +Context: 262,144 tokens +Max output: 262,144 tokens +Pricing: $0.65 input / $2.72 output per 1M tokens (blended: $1.16) +Capabilities: tool use, reasoning, vision, structured output +Available via: AtlasCloud, Baidu, BaseTen, Chutes, Cloudflare, CoreWeave, Crusoe, Decart, DeepInfra, DigitalOcean, Fireworks, Inceptron, ModelRun, Moonshot AI, Novita, Parasail, Phala, SiliconFlow, StreamLake, Together, Venice +Description: Kimi K2.6 is Moonshot AI's next-generation multimodal model, designed for long-horizon coding, coding-driven UI/UX generation, and multi-agent orchestration. It handles complex end-to-end coding tasks + ## GPT-5.4 Provider: OpenAI ID: openai/gpt-5.4 @@ -127,6 +197,26 @@ Available via: Azure, OpenAI EQ-Bench v3 Score: 68.65/100 Description: GPT-5.4 mini brings the core capabilities of GPT-5.4 to a faster, more efficient model optimized for high-throughput workloads. It supports text and image inputs with strong performance across reasoni +## GPT-5.5 +Provider: OpenAI +ID: openai/gpt-5.5 +Context: 1,050,000 tokens +Max output: 128,000 tokens +Pricing: $5.00 input / $30.00 output per 1M tokens (blended: $11.25) +Capabilities: tool use, reasoning, vision, web search, structured output +Available via: Azure, OpenAI +Description: GPT-5.5 is OpenAI’s frontier model designed for complex professional workloads, building on GPT-5.4 with stronger reasoning, higher reliability, and improved token efficiency on hard tasks. It feature + +## Qwen3.5 397B A17B +Provider: Qwen +ID: qwen/qwen3.5-397b-a17b +Context: 262,144 tokens +Max output: 65,536 tokens +Pricing: $0.39 input / $2.34 output per 1M tokens (blended: $0.88) +Capabilities: tool use, reasoning, vision, structured output +Available via: Alibaba, AtlasCloud, Chutes, DeepInfra, DigitalOcean, GMICloud, Novita, Parasail, Phala, StreamLake, Venice +Description: The Qwen3.5 series 397B-A17B native vision-language model is built on a hybrid architecture that integrates a linear attention mechanism with a sparse mixture-of-experts model, achieving higher infere + ## Qwen3.6 Plus Provider: Qwen ID: qwen/qwen3.6-plus:free @@ -173,6 +263,26 @@ EQ-Bench v3 Score: 70.55/100 PinchBench: 83.95% best, 80.7% avg (15 runs) Description: Xiaomi's flagship foundation model with 1T+ parameters and 1M context length, deeply optimized for agentic scenarios. +## GLM 4.7 +Provider: Z.ai +ID: z-ai/glm-4.7 +Context: 204,800 tokens +Max output: 131,072 tokens +Pricing: $0.40 input / $1.75 output per 1M tokens (blended: $0.74) +Capabilities: tool use, reasoning, structured output +Available via: AtlasCloud, Cerebras, DeepInfra, Google, Novita, Phala, StreamLake, Venice, Z.AI +Description: GLM-4.7 is Z.ai’s latest flagship model, featuring upgrades in two key areas: enhanced programming capabilities and more stable multi-step reasoning/execution. It demonstrates significant improvements + +## GLM 4.7 Flash +Provider: Z.ai +ID: z-ai/glm-4.7-flash +Context: 202,752 tokens +Max output: 16,384 tokens +Pricing: $0.06 input / $0.40 output per 1M tokens (blended: $0.14) +Capabilities: tool use, reasoning, structured output +Available via: Cloudflare, DeepInfra, Novita, Venice +Description: As a 30B-class SOTA model, GLM-4.7-Flash offers a new option that balances performance and efficiency. It is further optimized for agentic coding use cases, strengthening coding capabilities, long-hor + ## GLM 5 Turbo Provider: Z.ai ID: z-ai/glm-5-turbo @@ -189,6 +299,26 @@ EQ-Bench Elo: 1631.9 PinchBench: 86.5% best, 81.6% avg (11 runs) Description: GLM-5 Turbo is a new model from Z.ai designed for fast inference and strong performance in agent-driven environments such as OpenClaw scenarios. It is deeply optimized for real-world agent workflows.. +## GLM 5.1 +Provider: Z.ai +ID: z-ai/glm-5.1 +Context: 204,800 tokens +Max output: 128,000 tokens +Pricing: $0.97 input / $3.04 output per 1M tokens (blended: $1.48) +Capabilities: tool use, reasoning, structured output +Available via: AtlasCloud, Baidu, Chutes, CoreWeave, Crusoe, DeepInfra, DigitalOcean, Fireworks, Friendli, GMICloud, Nebius, Novita, Parasail, Phala, SiliconFlow, StreamLake, Venice, Wafer, Z.AI +Description: GLM-5.1 delivers a major leap in coding capability, with particularly significant gains in handling long-horizon tasks. Unlike previous models built around minute-level interactions, GLM-5.1 can work + +## GLM 5.2 +Provider: Z.ai +ID: z-ai/glm-5.2 +Context: 1,048,576 tokens +Max output: 131,072 tokens +Pricing: $0.71 input / $2.24 output per 1M tokens (blended: $1.10) +Capabilities: tool use, reasoning, structured output +Available via: AkashML, Alibaba, Ambient, AtlasCloud, Baidu, BaseTen, Chutes, Cloudflare, CoreWeave, Decart, DeepInfra, DigitalOcean, Fireworks, Friendli, GMICloud, Inceptron, Io Net, Ionstream, Morph, Novita, Parasail, Phala, Sail Research, SiliconFlow, StreamLake, Together, Venice, Wafer, Z.AI +Description: GLM 5.2 is a large-scale reasoning model from Z.ai. It supports text input and output with a 1M-token context window, and is suited for long-horizon agent workflows, project-level software engineering + ## Grok 4.20 Provider: xAI ID: x-ai/grok-4.20-20260309 diff --git a/model-benchmarks/scripts/eqbench_public.py b/model-benchmarks/scripts/eqbench_public.py new file mode 100644 index 0000000..2cdfa2e --- /dev/null +++ b/model-benchmarks/scripts/eqbench_public.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +EQ-Bench v3 PUBLIC leaderboard client. + +WHY THIS IS A SEPARATE METRIC (read before touching anything) +------------------------------------------------------------- +`benchmarks.eq_bench.v3_score` in model-data.json comes from Nick's OWN local +EQ-Bench v3 runs (~$6/model). Those runs emit a **22-trait** breakdown, five of +which do not exist on the public site at all: correctness, +intellectual_grounding, subtext_identification, sycophantic, theory_of_mind. + +eqbench.com publishes a **17-trait** rubric under the name `rubric_0_100`. +For the five models present in both, the public number is consistently HIGHER: + + model local v3_score public rubric_0_100 delta + openai/gpt-5.4 73.20 82.4 +9.20 + anthropic/claude-sonnet-4.6 71.70 80.0 +8.30 + anthropic/claude-opus-4.6 71.85 79.2 +7.35 + google/gemini-3.1-pro-... 68.95 74.3 +5.35 + google/gemma-4-31b-it 66.10 70.8 +4.70 + +The gap is not noise and not a constant offset — it is a different rubric over a +different trait set. Writing a public `rubric_0_100` into `v3_score` would +silently inflate every new model by 5-9 points against Nick's locally-run rows +and make the leaderboard's ordering meaningless. So the public number is stored +in its own field, `public_rubric_0_100`, and NEVER into `v3_score`. + +The public Elo (`elo_norm`) is likewise a different quantity from the stored +`elo` (site claude-sonnet-4.6 elo=1876.8, public elo_norm=1714.1). It is stored +as `public_elo_norm`, never into `elo`. + +MAPPING IS EXPLICIT AND NEVER FUZZY +----------------------------------- +EQ-Bench keys are not derivable from OpenRouter ids. Substring matching against +the live leaderboard actively mis-assigns: + + openai/gpt-5.4-mini --fuzzy--> gpt-5.4 (WRONG: different model) + z-ai/glm-5-turbo --fuzzy--> zai-org/GLM-5 (WRONG: different model) + x-ai/grok-4.20-... --fuzzy--> grok-4 (WRONG: different model) + +So EQBENCH_PUBLIC_MAP below is hand-verified, and a model absent from it simply +gets no public score. An empty cell is honest; a fuzzy match is a lie that +looks like data. +""" + +from __future__ import annotations + +import json +import re +import urllib.request + +EQBENCH_LEADERBOARD_URL = "https://eqbench.com/eqbench3.js?v=1.0.4" +EQBENCH_CHARTDATA_URL = "https://eqbench.com/eqbench3_chartdata.js?v=1.0.4" + +# Browser-ish UA: eqbench.com and OpenRouter both reject the default urllib UA. +_UA = "curl/8.7.1" + +# OpenRouter model id -> EQ-Bench leaderboard key. HAND-VERIFIED ONLY. +# Never add an entry you have not confirmed refers to the same model. +# A missing entry means "no public score", which is a correct outcome. +EQBENCH_PUBLIC_MAP: dict[str, str] = { + # --- already tracked on the site --- + "anthropic/claude-opus-4.6": "claude-opus-4-6", + "anthropic/claude-sonnet-4.6": "claude-sonnet-4-6", + "openai/gpt-5.4": "gpt-5.4", + "google/gemini-3.1-pro-preview-20260219": "gemini-3.1-pro-preview", + "google/gemma-4-31b-it": "google/gemma-4-31B-it", + "x-ai/grok-4.20-20260309": "grok-4.20-beta", + # --- untracked frontier models that DO have public EQ-Bench results --- + "anthropic/claude-opus-4.8": "claude-opus-4-8", + "anthropic/claude-fable-5": "claude-fable-5", + "openai/gpt-5.5": "gpt-5.5", + "z-ai/glm-5.2": "zai-org/GLM-5.2", + "z-ai/glm-5.1": "zai-org/GLM-5.1", + "z-ai/glm-4.7": "zai-org/GLM-4.7", + "z-ai/glm-4.7-flash": "zai-org/GLM-4.7-Flash", + "deepseek/deepseek-v4-pro": "deepseek-ai/DeepSeek-V4-Pro", + "deepseek/deepseek-v4-flash": "deepseek-ai/DeepSeek-V4-Flash", + "moonshotai/kimi-k2.6": "moonshotai/Kimi-K2.6", + "moonshotai/kimi-k2.5": "moonshotai/Kimi-K2.5", + "google/gemma-4-26b-a4b-it": "google/gemma-4-26B-A4B-it", + "qwen/qwen3.5-397b-a17b": "Qwen/Qwen3.5-397B-A17B", + # NOTE deliberately absent, verified as NOT on the public leaderboard: + # anthropic/claude-opus-5, anthropic/claude-opus-5-fast, x-ai/grok-4.5, + # moonshotai/kimi-k3, openai/gpt-5.6-*, google/gemini-3.6-flash + # NOTE deliberately absent, because fuzzy matching would mis-assign them: + # openai/gpt-5.4-mini, z-ai/glm-5-turbo, minimax/minimax-m2.7, + # stepfun/step-3.5-flash, xiaomi/mimo-v2-pro, qwen/qwen3.6-plus:free, + # anthropic/claude-haiku-4.5 +} + +_LEADERBOARD_RE = re.compile(r"leaderboardDataEQBench3\s*=\s*`(.*?)`", re.DOTALL) + + +def _get(url: str) -> str: + req = urllib.request.Request(url, headers={"User-Agent": _UA}) + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read().decode("utf-8") + + +def fetch_public_leaderboard() -> dict[str, dict]: + """Return {eqbench_key: {column: float}} from the public v3 leaderboard. + + Raises on failure — a silent empty result would let a caller conclude + "no models have scores" and quietly write nothing, which is exactly the + green-checkmark-over-zero-work failure mode. + """ + body = _get(EQBENCH_LEADERBOARD_URL) + m = _LEADERBOARD_RE.search(body) + if not m: + raise RuntimeError( + f"leaderboardDataEQBench3 block not found at {EQBENCH_LEADERBOARD_URL} " + "— upstream format changed, refusing to guess" + ) + rows = [r for r in m.group(1).strip().split("\n") if r.strip()] + header = rows[0].split(",") + out: dict[str, dict] = {} + for row in rows[1:]: + parts = row.split(",") + if len(parts) != len(header): + print(f" WARNING: skipping malformed EQ-Bench row: {row[:60]!r}") + continue + # A leading '*' marks a recently-added entry on eqbench.com; not part + # of the model name. + name = parts[0].lstrip("*") + vals = {} + for col, raw in zip(header[1:], parts[1:]): + try: + vals[col] = float(raw) + except ValueError: + print(f" WARNING: non-numeric {col}={raw!r} for {name}, skipping cell") + out[name] = vals + if not out: + raise RuntimeError("EQ-Bench leaderboard parsed to zero rows — refusing") + return out + + +def fetch_public_traits() -> dict[str, dict[str, float]]: + """Return {eqbench_key: {trait: value}} — the public 17-trait radar.""" + body = _get(EQBENCH_CHARTDATA_URL) + start = body.index("{") + payload = json.loads(body[start:].strip().rstrip(";")) + out: dict[str, dict[str, float]] = {} + for key, entry in payload.items(): + radar = (entry or {}).get("absoluteRadar") or {} + labels, values = radar.get("labels"), radar.get("values") + if not labels or not values or len(labels) != len(values): + print(f" WARNING: no usable absoluteRadar for {key}, skipping traits") + continue + out[key] = dict(zip(labels, values)) + if not out: + raise RuntimeError("EQ-Bench chartdata parsed to zero rows — refusing") + return out + + +def public_eq_block(model_id: str, leaderboard: dict, traits: dict) -> dict | None: + """Build the public-EQ fields for one model, or None if it has no entry. + + Returns ONLY `public_*` keys. It must never emit `v3_score`, `v3_traits`, + or `elo` — those belong to Nick's local runs and are a different rubric. + """ + key = EQBENCH_PUBLIC_MAP.get(model_id) + if key is None: + return None + row = leaderboard.get(key) + if row is None: + print(f" WARNING: {model_id} mapped to '{key}' which is not on the leaderboard") + return None + block: dict = {"public_source_key": key} + if (rubric := row.get("rubric_0_100")) is not None: + block["public_rubric_0_100"] = rubric + if (elo := row.get("elo_norm")) is not None: + block["public_elo_norm"] = elo + if key in traits: + block["public_traits_17"] = traits[key] + return block + + +# Fields this module owns. Anything else in eq_bench is local-run data and must +# survive untouched. +PUBLIC_EQ_FIELDS = ( + "public_source_key", + "public_rubric_0_100", + "public_elo_norm", + "public_traits_17", +) diff --git a/model-benchmarks/scripts/fetch-model.py b/model-benchmarks/scripts/fetch-model.py index 844aea1..2cf4caa 100644 --- a/model-benchmarks/scripts/fetch-model.py +++ b/model-benchmarks/scripts/fetch-model.py @@ -30,6 +30,9 @@ from pathlib import Path from typing import Any +sys.path.insert(0, str(Path(__file__).parent)) +import eqbench_public # noqa: E402 + SCRIPT_DIR = Path(__file__).parent DATA_DIR = SCRIPT_DIR.parent / "data" MODEL_DATA_PATH = DATA_DIR / "model-data.json" @@ -427,6 +430,65 @@ def load_model_data() -> dict: } +def apply_public_eq(data: dict, only_ids: list[str] | None = None) -> tuple[int, int]: + """Attach public EQ-Bench v3 leaderboard fields to models already in `data`. + + Writes ONLY the `public_*` keys owned by eqbench_public. Nick's locally-run + `v3_score` / `v3_traits` / `elo` are a DIFFERENT rubric (see the module + docstring) and are never read or written here. + + Returns (models_updated, models_with_no_public_entry). + """ + leaderboard = eqbench_public.fetch_public_leaderboard() + traits = eqbench_public.fetch_public_traits() + print(f" EQ-Bench public leaderboard: {len(leaderboard)} models") + + updated = missing = 0 + for m in data["models"]: + if only_ids is not None and m["id"] not in only_ids: + continue + block = eqbench_public.public_eq_block(m["id"], leaderboard, traits) + if block is None: + missing += 1 + continue + eq = m.setdefault("benchmarks", {}).setdefault("eq_bench", {}) + before = {k: eq.get(k) for k in eqbench_public.PUBLIC_EQ_FIELDS} + eq.update(block) + if {k: eq.get(k) for k in eqbench_public.PUBLIC_EQ_FIELDS} != before: + updated += 1 + print( + f" + {m['id']} public rubric=" + f"{block.get('public_rubric_0_100')} elo={block.get('public_elo_norm')}" + ) + m.setdefault("sources", {})["eq_bench_public"] = True + return updated, missing + + +def discover_new_models(data: dict, openrouter_models: list[dict]) -> list[str]: + """Return OpenRouter ids that have a hand-verified EQ-Bench entry but are + not yet in the dataset. + + This is the "ADD, don't just refresh" half. It is deliberately gated on + EQBENCH_PUBLIC_MAP rather than on OpenRouter's full catalogue: the site's + whole thesis is emotional intelligence, so a model with no EQ evidence has + nothing to say on the page. Gating on a hand-verified map also means + discovery can never invent a mapping. + """ + known = {m["id"] for m in data["models"]} + # Aliased rows keep a stable dataset id; treat their live id as known too. + known |= {OPENROUTER_ID_ALIASES[k] for k in OPENROUTER_ID_ALIASES if k in known} + live = {m["id"] for m in openrouter_models} + found = [] + for model_id in eqbench_public.EQBENCH_PUBLIC_MAP: + if model_id in known: + continue + if model_id not in live: + print(f" skip {model_id}: has EQ-Bench data but not live on OpenRouter") + continue + found.append(model_id) + return sorted(found) + + def merge_model(existing_data: dict, new_model: dict) -> dict: """Merge a new/updated model into the dataset. Preserves manually-added data.""" models = existing_data["models"] @@ -610,6 +672,17 @@ def main(): parser.add_argument( "--no-aa", action="store_true", help="Skip Artificial Analysis enrichment" ) + parser.add_argument( + "--discover", + action="store_true", + help="Add models that have hand-verified EQ-Bench entries but aren't tracked yet", + ) + parser.add_argument( + "--eq-public", + action="store_true", + help="Fetch public EQ-Bench v3 leaderboard fields (public_rubric_0_100, " + "public_elo_norm, public_traits_17). Never touches locally-run v3_score.", + ) args = parser.parse_args() if args.curated: @@ -624,6 +697,8 @@ def main(): sys.exit(1) elif args.model_ids: target_ids = args.model_ids + elif args.discover or args.eq_public: + target_ids = [] else: parser.print_help() sys.exit(1) @@ -633,6 +708,18 @@ def main(): all_models = fetch_openrouter_models() models_by_id = {m["id"]: m for m in all_models} + data = load_model_data() + + if args.discover: + discovered = discover_new_models(data, all_models) + if discovered: + print(f"Discovered {len(discovered)} new model(s) with EQ-Bench data:") + for d in discovered: + print(f" {d}") + target_ids = list(target_ids) + discovered + else: + print("Discovered 0 new models with EQ-Bench data.") + # Fetch Artificial Analysis data if available aa_models = {} if not args.no_aa: @@ -647,9 +734,7 @@ def main(): else: print(" No AA API key found (set AA_API_KEY or add to env.local)") - data = load_model_data() fetched = 0 - for model_id in target_ids: raw = models_by_id.get(model_id) alias = None @@ -694,6 +779,14 @@ def main(): data = merge_model(data, transformed) fetched += 1 + if args.eq_public: + print("Fetching public EQ-Bench v3 leaderboard...") + updated, missing = apply_public_eq(data) + print( + f" Public EQ applied to {updated} model(s); " + f"{missing} have no hand-verified public entry (left empty)." + ) + if not args.dry_run: save_model_data(data) if not args.no_llms_txt: diff --git a/model-benchmarks/tests/test_fetch_model.py b/model-benchmarks/tests/test_fetch_model.py index 59eafb2..7aa6778 100644 --- a/model-benchmarks/tests/test_fetch_model.py +++ b/model-benchmarks/tests/test_fetch_model.py @@ -19,6 +19,8 @@ sys.modules["fetch_model"] = fm _spec.loader.exec_module(fm) +import eqbench_public as eqp # noqa: E402 (path set up by fetch-model.py import) + REAL = {"intelligence_index": 46.5, "coding_index": 47.6, "gpqa": 0.84} NULLED = dict.fromkeys(REAL) @@ -124,5 +126,90 @@ def test_noted_models_are_documented(self): self.assertTrue(fm.MODEL_NOTES[model_id].strip()) +class PublicEqIsolation(unittest.TestCase): + """The public EQ-Bench rubric is a DIFFERENT metric from Nick's local runs. + + Public `rubric_0_100` runs 5-9 points higher than local `v3_score` over 17 + traits instead of 22. Leaking it into `v3_score` would silently inflate new + models against existing rows and corrupt the leaderboard ordering. + """ + + def setUp(self): + self.leaderboard = { + "claude-opus-4-6": {"rubric_0_100": 79.2, "elo_norm": 1717.4}, + } + self.traits = {"claude-opus-4-6": {"warmth": 14.04, "analytical": 17.42}} + + def test_public_block_never_emits_local_fields(self): + block = eqp.public_eq_block( + "anthropic/claude-opus-4.6", self.leaderboard, self.traits + ) + for forbidden in ("v3_score", "v3_traits", "elo"): + self.assertNotIn(forbidden, block) + + def test_public_block_emits_namespaced_fields(self): + block = eqp.public_eq_block( + "anthropic/claude-opus-4.6", self.leaderboard, self.traits + ) + self.assertEqual(block["public_rubric_0_100"], 79.2) + self.assertEqual(block["public_elo_norm"], 1717.4) + self.assertEqual(block["public_source_key"], "claude-opus-4-6") + + def test_unmapped_model_gets_nothing(self): + """An unmapped model must yield None, never a fuzzy-matched score.""" + self.assertIsNone( + eqp.public_eq_block("acme/never-heard-of-it", self.leaderboard, self.traits) + ) + + def test_local_v3_score_survives_public_fetch(self): + data = { + "models": [ + { + "id": "anthropic/claude-opus-4.6", + "benchmarks": { + "eq_bench": {"v3_score": 71.85, "v3_traits": {"warmth": 13.6}} + }, + } + ] + } + eq = data["models"][0]["benchmarks"]["eq_bench"] + eq.update( + eqp.public_eq_block( + "anthropic/claude-opus-4.6", self.leaderboard, self.traits + ) + ) + self.assertEqual(eq["v3_score"], 71.85, "local v3_score was clobbered") + self.assertEqual(eq["v3_traits"], {"warmth": 13.6}) + self.assertEqual(eq["public_rubric_0_100"], 79.2) + + def test_map_has_no_fuzzy_lookalikes(self): + """Models whose names invite a wrong substring match stay unmapped.""" + for risky in ( + "openai/gpt-5.4-mini", + "z-ai/glm-5-turbo", + "anthropic/claude-haiku-4.5", + ): + self.assertNotIn(risky, eqp.EQBENCH_PUBLIC_MAP) + + +class DiscoverNewModels(unittest.TestCase): + def test_untracked_and_live_is_discovered(self): + data = {"models": [{"id": "anthropic/claude-opus-4.6"}]} + live = [{"id": "anthropic/claude-opus-4.8"}, {"id": "anthropic/claude-opus-4.6"}] + self.assertIn("anthropic/claude-opus-4.8", fm.discover_new_models(data, live)) + + def test_already_tracked_is_not_rediscovered(self): + data = {"models": [{"id": "anthropic/claude-opus-4.8"}]} + live = [{"id": "anthropic/claude-opus-4.8"}] + self.assertNotIn( + "anthropic/claude-opus-4.8", fm.discover_new_models(data, live) + ) + + def test_not_live_on_openrouter_is_skipped(self): + """EQ-Bench lists models OpenRouter doesn't serve; never invent a row.""" + data = {"models": []} + self.assertEqual(fm.discover_new_models(data, []), []) + + if __name__ == "__main__": unittest.main() From 0239f5c3e064776082ecfe00e7b976a416d017dd Mon Sep 17 00:00:00 2001 From: Nick Sullivan Date: Sat, 25 Jul 2026 23:32:35 -0500 Subject: [PATCH 2/6] fix(benchmarks): remove qwen3.6-plus's inherited EQ score from its predecessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Charter non-negotiable: "never present a successor model's score as a predecessor's." qwen/qwen3.6-plus:free carried EQ-Bench data (elo 1417.4, v3_score 60.45, and a full 22-trait breakdown) copied verbatim from Qwen3.5-397B-A17B. It was footnoted, not fixed — a note explaining inherited data does not make it that model's score, and the number still sorted and rendered as if it were real. The previous commit added Qwen3.5-397B-A17B as its own tracked row, so the borrowed copy is now both a violation and redundant. Removed elo, v3_score, v3_traits, and the inline note. Qwen3.6 Plus now shows an empty EQ cell, which is the honest state: it has not been benchmarked on EQ-Bench. PinchBench (88.6/84.0) is retained — per the original note it was run on Qwen3.6 Plus directly, so it is genuinely this model's own data. Verified: field-by-field diff shows exactly 5 changes, all on qwen3.6-plus, 0 changes to the other 25 models. Four regression tests added (19 -> 23) reading the real data file: no inherited EQ fields on qwen3.6, predecessor tracked separately, PinchBench survives, and no two rows may share a public_source_key (two rows citing one EQ-Bench entry means one of them inherited it). Found by the pass's independent review panel, which correctly flagged that the steward had reported this as a "pre-existing integrity issue" instead of enforcing the charter on it. --- model-benchmarks/data/model-data.json | 34 ++---------------- model-benchmarks/llms.txt | 2 -- model-benchmarks/scripts/fetch-model.py | 8 +++-- model-benchmarks/tests/test_fetch_model.py | 42 ++++++++++++++++++++++ 4 files changed, 50 insertions(+), 36 deletions(-) diff --git a/model-benchmarks/data/model-data.json b/model-benchmarks/data/model-data.json index ee882dd..aa2d8ef 100644 --- a/model-benchmarks/data/model-data.json +++ b/model-benchmarks/data/model-data.json @@ -1948,35 +1948,7 @@ "terminalbench_hard": 0.409, "note": "Data from Qwen3.5-397B (predecessor)" }, - "eq_bench": { - "elo": 1417.4, - "v3_score": 60.45, - "v3_traits": { - "analytical": 14.5, - "boundary_setting": 10.5, - "challenging": 8.1, - "compliant": 12.0, - "conversational": 9.8, - "correctness": 11.7, - "demonstrated_empathy": 13.5, - "depth_of_insight": 12.6, - "emotional_reasoning": 12.7, - "humanlike": 11.2, - "intellectual_grounding": 10.8, - "message_tailoring": 11.4, - "moralising": 10.1, - "pragmatic_ei": 12.6, - "reactive": 10.0, - "safety_conscious": 11.2, - "social_dexterity": 12.0, - "subtext_identification": 10.6, - "sycophantic": 6.2, - "theory_of_mind": 10.8, - "validating": 13.4, - "warmth": 13.4 - }, - "note": "Benchmark run on Qwen3.5-397B (predecessor)" - }, + "eq_bench": {}, "pinchbench": { "best_score": 88.6, "avg_score": 84.0, @@ -1997,7 +1969,7 @@ }, "created": 1775133557, "last_updated": "2026-07-25T08:58:52Z", - "notes": "Benchmark scores from Qwen3.5-397B (predecessor). PinchBench is from Qwen3.6 Plus directly. The :free OpenRouter tier has been retired; pricing shown is the paid qwen/qwen3.6-plus endpoint.", + "notes": "EQ-Bench scores removed 2026-07-26: they were inherited from predecessor Qwen3.5-397B-A17B, which is now tracked as its own row (qwen/qwen3.5-397b-a17b). Qwen3.6 Plus has not been benchmarked on EQ-Bench. PinchBench is from Qwen3.6 Plus directly. The :free OpenRouter tier has been retired; pricing shown is the paid qwen/qwen3.6-plus endpoint.", "speed": { "output_tokens_per_sec": 44.0, "ttft_seconds": 1.59, @@ -2813,4 +2785,4 @@ } } ] -} \ No newline at end of file +} diff --git a/model-benchmarks/llms.txt b/model-benchmarks/llms.txt index 7734ca7..ba37af9 100644 --- a/model-benchmarks/llms.txt +++ b/model-benchmarks/llms.txt @@ -228,8 +228,6 @@ Scores (0-100): Reasoning: 57.0, Coding: 41.3, Agentic: 87.1 Speed: 44.0 tokens/sec, TTFT 1.59s Available via: Alibaba Benchmarks (Artificial Analysis): intelligence_index: 45.0, coding_index: 41.3, gpqa: 0.893, hle: 0.273, scicode: 0.42, ifbench: 0.788, terminalbench_hard: 0.409, note: Data from Qwen3.5-397B (predecessor) -EQ-Bench v3 Score: 60.45/100 -EQ-Bench Elo: 1417.4 PinchBench: 88.6% best, 84.0% avg (5 runs) Description: Qwen 3.6 Plus builds on a hybrid architecture that combines efficient linear attention with sparse mixture-of-experts routing, enabling strong scalability and high-performance inference. Compared to t diff --git a/model-benchmarks/scripts/fetch-model.py b/model-benchmarks/scripts/fetch-model.py index 2cf4caa..76a63ad 100644 --- a/model-benchmarks/scripts/fetch-model.py +++ b/model-benchmarks/scripts/fetch-model.py @@ -84,9 +84,11 @@ # incoming model has none, and OpenRouter never supplies these). MODEL_NOTES = { "qwen/qwen3.6-plus:free": ( - "Benchmark scores from Qwen3.5-397B (predecessor). PinchBench is from " - "Qwen3.6 Plus directly. The :free OpenRouter tier has been retired; " - "pricing shown is the paid qwen/qwen3.6-plus endpoint." + "EQ-Bench scores removed 2026-07-26: they were inherited from predecessor " + "Qwen3.5-397B-A17B, which is now tracked as its own row " + "(qwen/qwen3.5-397b-a17b). Qwen3.6 Plus has not been benchmarked on " + "EQ-Bench. PinchBench is from Qwen3.6 Plus directly. The :free OpenRouter " + "tier has been retired; pricing shown is the paid qwen/qwen3.6-plus endpoint." ), "xiaomi/mimo-v2-pro": ( "Delisted from OpenRouter. Pricing and availability are frozen as of " diff --git a/model-benchmarks/tests/test_fetch_model.py b/model-benchmarks/tests/test_fetch_model.py index 7aa6778..39e085a 100644 --- a/model-benchmarks/tests/test_fetch_model.py +++ b/model-benchmarks/tests/test_fetch_model.py @@ -9,6 +9,7 @@ """ import importlib.util +import json import sys import unittest from pathlib import Path @@ -211,5 +212,46 @@ def test_not_live_on_openrouter_is_skipped(self): self.assertEqual(fm.discover_new_models(data, []), []) +class NoInheritedScores(unittest.TestCase): + """Charter non-negotiable: never present a predecessor's score as a model's own. + + qwen/qwen3.6-plus:free carried EQ-Bench data copied from Qwen3.5-397B-A17B. + That predecessor is now tracked as its own row, so the inherited copy was + removed 2026-07-26 rather than merely footnoted. + """ + + DATA = Path(__file__).resolve().parent.parent / "data" / "model-data.json" + + def setUp(self): + with open(self.DATA) as f: + self.models = {m["id"]: m for m in json.load(f)["models"]} + + def test_qwen36_has_no_inherited_eq(self): + eq = self.models["qwen/qwen3.6-plus:free"]["benchmarks"].get("eq_bench", {}) + for field in ("v3_score", "v3_traits", "elo"): + self.assertIsNone( + eq.get(field), + f"qwen3.6-plus has {field} again — it belongs to Qwen3.5-397B", + ) + + def test_predecessor_is_tracked_separately(self): + self.assertIn("qwen/qwen3.5-397b-a17b", self.models) + + def test_qwen36_keeps_its_own_pinchbench(self): + """PinchBench was run on Qwen3.6 directly, so it must survive.""" + pb = self.models["qwen/qwen3.6-plus:free"]["benchmarks"]["pinchbench"] + self.assertEqual(pb["best_score"], 88.6) + + def test_no_two_models_share_a_public_source_key(self): + """Two rows citing one EQ-Bench entry means one of them inherited it.""" + seen = {} + for mid, m in self.models.items(): + key = m["benchmarks"].get("eq_bench", {}).get("public_source_key") + if key is None: + continue + self.assertNotIn(key, seen, f"{mid} and {seen.get(key)} share EQ key {key}") + seen[key] = mid + + if __name__ == "__main__": unittest.main() From f3795f11e178962c877d985f8fa700f9490247b3 Mon Sep 17 00:00:00 2001 From: Nick Sullivan Date: Sat, 25 Jul 2026 23:35:50 -0500 Subject: [PATCH 3/6] style(benchmarks): apply ruff-format to new EQ-Bench module and tests --- model-benchmarks/scripts/eqbench_public.py | 4 +++- model-benchmarks/tests/test_fetch_model.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/model-benchmarks/scripts/eqbench_public.py b/model-benchmarks/scripts/eqbench_public.py index 2cdfa2e..6cf6fa9 100644 --- a/model-benchmarks/scripts/eqbench_public.py +++ b/model-benchmarks/scripts/eqbench_public.py @@ -164,7 +164,9 @@ def public_eq_block(model_id: str, leaderboard: dict, traits: dict) -> dict | No return None row = leaderboard.get(key) if row is None: - print(f" WARNING: {model_id} mapped to '{key}' which is not on the leaderboard") + print( + f" WARNING: {model_id} mapped to '{key}' which is not on the leaderboard" + ) return None block: dict = {"public_source_key": key} if (rubric := row.get("rubric_0_100")) is not None: diff --git a/model-benchmarks/tests/test_fetch_model.py b/model-benchmarks/tests/test_fetch_model.py index 39e085a..4a775a0 100644 --- a/model-benchmarks/tests/test_fetch_model.py +++ b/model-benchmarks/tests/test_fetch_model.py @@ -196,7 +196,10 @@ def test_map_has_no_fuzzy_lookalikes(self): class DiscoverNewModels(unittest.TestCase): def test_untracked_and_live_is_discovered(self): data = {"models": [{"id": "anthropic/claude-opus-4.6"}]} - live = [{"id": "anthropic/claude-opus-4.8"}, {"id": "anthropic/claude-opus-4.6"}] + live = [ + {"id": "anthropic/claude-opus-4.8"}, + {"id": "anthropic/claude-opus-4.6"}, + ] self.assertIn("anthropic/claude-opus-4.8", fm.discover_new_models(data, live)) def test_already_tracked_is_not_rediscovered(self): From 120aec3076aa302c0bb2c32f123a28813961a408 Mon Sep 17 00:00:00 2001 From: Nick Sullivan Date: Sun, 26 Jul 2026 08:06:40 -0500 Subject: [PATCH 4/6] feat(benchmarks): add 9 frontier models with real public EQ-Bench scores Extends EQBENCH_PUBLIC_MAP with 9 hand-verified mappings, each confirmed both present on eqbench.com and live on OpenRouter (matched by name + canonical_slug, never by substring). Dataset 26 -> 35 models. Added: claude-opus-4.7 (82.6), gpt-5.2 (80.4), gpt-5.1 (78.1), claude-opus-4.5 (76.8), glm-5 (75.0), kimi-k2 (75.0), sonnet-4.5 (72.2), gpt-5.3-chat (70.7), hermes-4-405b (65.2). None carries a v3_score: those are Nick's local runs and remain empty rather than invented. Verified: field-by-field diff of the pre-existing 26 shows 0 changes and 0 fields lost; all 56 public values re-read from upstream by an independent parser match exactly; no public rubric leaked into v3_score. Resolves the glm-5-turbo provenance question: it is NOT a copy of GLM-5. Only 4 of 11 rescaled traits agree within rounding and the stored elo (1631.9) differs from GLM-5's public elo_norm (1526.0). GLM-5 is a distinct OpenRouter model and now has its own row. Tests 23 -> 26: public map is injective (no two models can claim one upstream row), and glm-5-turbo can never be mapped to GLM-5. --- model-benchmarks/data/model-data.json | 783 ++++++++++++++++++++- model-benchmarks/llms.txt | 91 ++- model-benchmarks/scripts/eqbench_public.py | 25 + model-benchmarks/tests/test_fetch_model.py | 37 + 4 files changed, 922 insertions(+), 14 deletions(-) diff --git a/model-benchmarks/data/model-data.json b/model-benchmarks/data/model-data.json index aa2d8ef..e5d7006 100644 --- a/model-benchmarks/data/model-data.json +++ b/model-benchmarks/data/model-data.json @@ -1,5 +1,5 @@ { - "generated": "2026-07-26T04:27:13Z", + "generated": "2026-07-26T13:04:59Z", "schema_version": "2.0", "sources": { "openrouter": "https://openrouter.ai/api/v1/models", @@ -213,6 +213,91 @@ "source": "artificial_analysis" } }, + { + "id": "anthropic/claude-opus-4.5", + "canonical_slug": "anthropic/claude-4.5-opus-20251124", + "name": "Claude Opus 4.5", + "provider": "Anthropic", + "description": "Claude Opus 4.5 is Anthropic\u2019s frontier reasoning model optimized for complex software engineering, agentic workflows, and long-horizon computer use. It offers strong multimodal capabilities, competitive performance across real-world coding and...", + "context_window": 200000, + "max_output": 64000, + "modalities": { + "input": [ + "file", + "image", + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Claude", + "pricing": { + "input_per_m": 5.0, + "output_per_m": 25.0, + "cache_read_per_m": 0.5, + "cache_write_per_m": 6.25, + "web_search_per_req": 0.01, + "blended_per_m": 10.0 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Amazon Bedrock", + "Anthropic", + "Azure", + "Google" + ], + "uptime_24h": 99.98, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "claude-opus-4-5-20251101", + "public_rubric_0_100": 76.8, + "public_elo_norm": 1545.7, + "public_traits_17": { + "demonstrated_empathy": 15.35, + "pragmatic_ei": 15.04, + "depth_of_insight": 16.76, + "social_dexterity": 14.27, + "emotional_reasoning": 16.09, + "message_tailoring": 13.62, + "boundary_setting": 13.15, + "safety_conscious": 12.88, + "moralising": 7.77, + "compliant": 10.38, + "challenging": 12.54, + "warmth": 13.62, + "validating": 13.12, + "analytical": 16.73, + "reactive": 9.42, + "conversational": 11.85, + "humanlike": 14.65 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1764010580, + "last_updated": "2026-07-26T13:04:58Z" + }, { "id": "anthropic/claude-opus-4.6", "canonical_slug": "anthropic/claude-4.6-opus-20260205", @@ -361,6 +446,91 @@ "source": "artificial_analysis" } }, + { + "id": "anthropic/claude-opus-4.7", + "canonical_slug": "anthropic/claude-4.7-opus-20260416", + "name": "Claude Opus 4.7", + "provider": "Anthropic", + "description": "Opus 4.7 is the next generation of Anthropic's Opus family, built for long-running, asynchronous agents. Building on the coding and agentic strengths of Opus 4.6, it delivers stronger performance on...", + "context_window": 1000000, + "max_output": 128000, + "modalities": { + "input": [ + "text", + "image", + "file" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Claude", + "pricing": { + "input_per_m": 5.0, + "output_per_m": 25.0, + "cache_read_per_m": 0.5, + "cache_write_per_m": 6.25, + "web_search_per_req": 0.01, + "blended_per_m": 10.0 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Amazon Bedrock", + "Anthropic", + "Azure", + "Google" + ], + "uptime_24h": 100.0, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "claude-opus-4-7", + "public_rubric_0_100": 82.6, + "public_elo_norm": 1884.3, + "public_traits_17": { + "demonstrated_empathy": 16.42, + "pragmatic_ei": 16.85, + "depth_of_insight": 17.76, + "social_dexterity": 15.81, + "emotional_reasoning": 17.04, + "message_tailoring": 15.35, + "boundary_setting": 14.62, + "safety_conscious": 13.15, + "moralising": 6.88, + "compliant": 9.46, + "challenging": 13.23, + "warmth": 13.46, + "validating": 13.46, + "analytical": 17.81, + "reactive": 8.46, + "conversational": 12.77, + "humanlike": 15.92 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1776351100, + "last_updated": "2026-07-26T13:04:58Z" + }, { "id": "anthropic/claude-opus-4.8", "canonical_slug": "anthropic/claude-4.8-opus-20260528", @@ -446,6 +616,91 @@ "created": 1779905091, "last_updated": "2026-07-26T04:27:11Z" }, + { + "id": "anthropic/claude-sonnet-4.5", + "canonical_slug": "anthropic/claude-4.5-sonnet-20250929", + "name": "Claude Sonnet 4.5", + "provider": "Anthropic", + "description": "Claude Sonnet 4.5 is Anthropic\u2019s most advanced Sonnet model to date, optimized for real-world agents and coding workflows. It delivers state-of-the-art performance on coding benchmarks such as SWE-bench Verified, with...", + "context_window": 1000000, + "max_output": 64000, + "modalities": { + "input": [ + "text", + "image", + "file" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Claude", + "pricing": { + "input_per_m": 3.0, + "output_per_m": 15.0, + "cache_read_per_m": 0.3, + "cache_write_per_m": 3.75, + "web_search_per_req": 0.01, + "blended_per_m": 6.0 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Amazon Bedrock", + "Anthropic", + "Azure", + "Google" + ], + "uptime_24h": 100, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "claude-sonnet-4.5", + "public_rubric_0_100": 72.2, + "public_elo_norm": 1511.0, + "public_traits_17": { + "demonstrated_empathy": 13.92, + "pragmatic_ei": 13.31, + "depth_of_insight": 16.67, + "social_dexterity": 11.92, + "emotional_reasoning": 15.42, + "message_tailoring": 11.69, + "boundary_setting": 14.19, + "safety_conscious": 12.85, + "moralising": 10.65, + "compliant": 8.92, + "challenging": 14.15, + "warmth": 11.77, + "validating": 11.35, + "analytical": 17.15, + "reactive": 11.08, + "conversational": 10.42, + "humanlike": 13.73 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1759161676, + "last_updated": "2026-07-26T13:04:58Z" + }, { "id": "anthropic/claude-sonnet-4.6", "canonical_slug": "anthropic/claude-4.6-sonnet-20260217", @@ -1284,6 +1539,86 @@ "source": "artificial_analysis" } }, + { + "id": "moonshotai/kimi-k2", + "canonical_slug": "moonshotai/kimi-k2", + "name": "Kimi K2 0711", + "provider": "MoonshotAI", + "description": "Kimi K2 Instruct is a large-scale Mixture-of-Experts (MoE) language model developed by Moonshot AI, featuring 1 trillion total parameters with 32 billion active per forward pass. It is optimized for...", + "context_window": 131072, + "max_output": 100352, + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Other", + "pricing": { + "input_per_m": 0.57, + "output_per_m": 2.3, + "cache_read_per_m": null, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 1.0025 + }, + "capabilities": { + "tool_use": true, + "reasoning": false, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": false + }, + "endpoint_stats": { + "providers": [ + "Novita" + ], + "uptime_24h": 99.98, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "moonshotai/Kimi-K2-Instruct", + "public_rubric_0_100": 75.0, + "public_elo_norm": 1562.2, + "public_traits_17": { + "demonstrated_empathy": 15.12, + "pragmatic_ei": 14.69, + "depth_of_insight": 16.96, + "social_dexterity": 13.73, + "emotional_reasoning": 15.89, + "message_tailoring": 13.15, + "boundary_setting": 13.65, + "safety_conscious": 13.46, + "moralising": 8.42, + "compliant": 11.31, + "challenging": 11.58, + "warmth": 13.81, + "validating": 13.58, + "analytical": 17.46, + "reactive": 10.73, + "conversational": 10.54, + "humanlike": 14.04 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1752263252, + "last_updated": "2026-07-26T13:04:58Z" + }, { "id": "moonshotai/kimi-k2.5", "canonical_slug": "moonshotai/kimi-k2.5-0127", @@ -1476,24 +1811,353 @@ "last_updated": "2026-07-26T04:27:12Z" }, { - "id": "openai/gpt-5.4", - "canonical_slug": "openai/gpt-5.4-20260305", - "name": "GPT-5.4", - "provider": "OpenAI", - "description": "GPT-5.4 is OpenAI\u2019s latest frontier model, unifying the Codex and GPT lines into a single system. It features a 1M+ token context window (922K input, 128K output) with support for...", - "context_window": 1050000, - "max_output": 128000, + "id": "nousresearch/hermes-4-405b", + "canonical_slug": "nousresearch/hermes-4-405b", + "name": "Hermes 4 405B", + "provider": "Nous", + "description": "Hermes 4 is a large-scale reasoning model built on Meta-Llama-3.1-405B and released by Nous Research. It introduces a hybrid reasoning mode, where the model can choose to deliberate internally with...", + "context_window": 131072, + "max_output": null, "modalities": { "input": [ - "text", - "image", - "file" + "text" ], "output": [ "text" ] }, - "tokenizer": "GPT", + "tokenizer": "Other", + "pricing": { + "input_per_m": 1.0, + "output_per_m": 3.0, + "cache_read_per_m": null, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 1.5 + }, + "capabilities": { + "tool_use": false, + "reasoning": true, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Nebius" + ], + "uptime_24h": 100, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "NousResearch/Hermes-4-405B", + "public_rubric_0_100": 65.2, + "public_elo_norm": 1361.4, + "public_traits_17": { + "demonstrated_empathy": 13.88, + "pragmatic_ei": 13.54, + "depth_of_insight": 14.16, + "social_dexterity": 12.5, + "emotional_reasoning": 13.69, + "message_tailoring": 12.31, + "boundary_setting": 12.38, + "safety_conscious": 11.46, + "moralising": 9.0, + "compliant": 12.0, + "challenging": 10.73, + "warmth": 13.0, + "validating": 13.85, + "analytical": 16.42, + "reactive": 10.88, + "conversational": 10.23, + "humanlike": 11.96 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1756235463, + "last_updated": "2026-07-26T13:04:58Z" + }, + { + "id": "openai/gpt-5.1", + "canonical_slug": "openai/gpt-5.1-20251113", + "name": "GPT-5.1", + "provider": "OpenAI", + "description": "GPT-5.1 is the latest frontier-grade model in the GPT-5 series, offering stronger general-purpose reasoning, improved instruction adherence, and a more natural conversational style compared to GPT-5. It uses adaptive reasoning...", + "context_window": 400000, + "max_output": 128000, + "modalities": { + "input": [ + "image", + "text", + "file" + ], + "output": [ + "text" + ] + }, + "tokenizer": "GPT", + "pricing": { + "input_per_m": 1.25, + "output_per_m": 10.0, + "cache_read_per_m": 0.125, + "cache_write_per_m": null, + "web_search_per_req": 0.01, + "blended_per_m": 3.4375 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Azure", + "OpenAI" + ], + "uptime_24h": 99.91, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "gpt-5.1-2025-11-13", + "public_rubric_0_100": 78.1, + "public_elo_norm": 1551.0, + "public_traits_17": { + "demonstrated_empathy": 15.35, + "pragmatic_ei": 15.85, + "depth_of_insight": 16.91, + "social_dexterity": 14.69, + "emotional_reasoning": 16.29, + "message_tailoring": 13.92, + "boundary_setting": 14.42, + "safety_conscious": 13.19, + "moralising": 7.15, + "compliant": 11.58, + "challenging": 11.69, + "warmth": 13.0, + "validating": 14.31, + "analytical": 17.46, + "reactive": 9.15, + "conversational": 10.5, + "humanlike": 13.92 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1763060305, + "last_updated": "2026-07-26T13:04:58Z" + }, + { + "id": "openai/gpt-5.2", + "canonical_slug": "openai/gpt-5.2-20251211", + "name": "GPT-5.2", + "provider": "OpenAI", + "description": "GPT-5.2 is the latest frontier-grade model in the GPT-5 series, offering stronger agentic and long context perfomance compared to GPT-5.1. It uses adaptive reasoning to allocate computation dynamically, responding quickly...", + "context_window": 400000, + "max_output": 128000, + "modalities": { + "input": [ + "file", + "image", + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "GPT", + "pricing": { + "input_per_m": 1.75, + "output_per_m": 14.0, + "cache_read_per_m": 0.175, + "cache_write_per_m": null, + "web_search_per_req": 0.01, + "blended_per_m": 4.8125 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Azure", + "OpenAI" + ], + "uptime_24h": 99.99, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "gpt-5.2", + "public_rubric_0_100": 80.4, + "public_elo_norm": 1558.8, + "public_traits_17": { + "demonstrated_empathy": 15.35, + "pragmatic_ei": 16.15, + "depth_of_insight": 17.4, + "social_dexterity": 14.81, + "emotional_reasoning": 16.64, + "message_tailoring": 14.04, + "boundary_setting": 15.54, + "safety_conscious": 14.5, + "moralising": 7.31, + "compliant": 10.54, + "challenging": 13.04, + "warmth": 12.31, + "validating": 13.65, + "analytical": 18.0, + "reactive": 8.88, + "conversational": 10.19, + "humanlike": 13.54 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1765389775, + "last_updated": "2026-07-26T13:04:59Z" + }, + { + "id": "openai/gpt-5.3-chat", + "canonical_slug": "openai/gpt-5.3-chat-20260303", + "name": "GPT-5.3 Chat", + "provider": "OpenAI", + "description": "GPT-5.3 Chat is an update to ChatGPT's most-used model that makes everyday conversations smoother, more useful, and more directly helpful. It delivers more accurate answers with better contextualization and significantly...", + "context_window": 128000, + "max_output": 16384, + "modalities": { + "input": [ + "text", + "image", + "file" + ], + "output": [ + "text" + ] + }, + "tokenizer": "GPT", + "pricing": { + "input_per_m": 1.75, + "output_per_m": 14.0, + "cache_read_per_m": 0.175, + "cache_write_per_m": null, + "web_search_per_req": 0.01, + "blended_per_m": 4.8125 + }, + "capabilities": { + "tool_use": true, + "reasoning": false, + "vision": true, + "audio_input": false, + "web_search": true, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Azure", + "OpenAI" + ], + "uptime_24h": 100, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "gpt-5.3-chat", + "public_rubric_0_100": 70.7, + "public_elo_norm": 1393.0, + "public_traits_17": { + "demonstrated_empathy": 12.88, + "pragmatic_ei": 13.35, + "depth_of_insight": 15.62, + "social_dexterity": 12.42, + "emotional_reasoning": 14.69, + "message_tailoring": 11.65, + "boundary_setting": 11.77, + "safety_conscious": 11.85, + "moralising": 5.65, + "compliant": 11.04, + "challenging": 8.85, + "warmth": 11.58, + "validating": 12.96, + "analytical": 16.5, + "reactive": 7.73, + "conversational": 10.19, + "humanlike": 11.35 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1772564061, + "last_updated": "2026-07-26T13:04:59Z" + }, + { + "id": "openai/gpt-5.4", + "canonical_slug": "openai/gpt-5.4-20260305", + "name": "GPT-5.4", + "provider": "OpenAI", + "description": "GPT-5.4 is OpenAI\u2019s latest frontier model, unifying the Codex and GPT lines into a single system. It features a 1M+ token context window (922K input, 128K output) with support for...", + "context_window": 1050000, + "max_output": 128000, + "modalities": { + "input": [ + "text", + "image", + "file" + ], + "output": [ + "text" + ] + }, + "tokenizer": "GPT", "pricing": { "input_per_m": 2.5, "output_per_m": 15.0, @@ -2327,6 +2991,99 @@ "created": 1768833913, "last_updated": "2026-07-26T04:27:12Z" }, + { + "id": "z-ai/glm-5", + "canonical_slug": "z-ai/glm-5-20260211", + "name": "GLM 5", + "provider": "Z.ai", + "description": "GLM-5 is Z.ai\u2019s flagship open-source foundation model engineered for complex systems design and long-horizon agent workflows. Built for expert developers, it delivers production-grade performance on large-scale programming tasks, rivaling leading...", + "context_window": 204800, + "max_output": 131072, + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "tokenizer": "Other", + "pricing": { + "input_per_m": 0.95, + "output_per_m": 2.55, + "cache_read_per_m": 0.2, + "cache_write_per_m": null, + "web_search_per_req": null, + "blended_per_m": 1.35 + }, + "capabilities": { + "tool_use": true, + "reasoning": true, + "vision": false, + "audio_input": false, + "web_search": false, + "structured_output": true + }, + "endpoint_stats": { + "providers": [ + "Amazon Bedrock", + "AtlasCloud", + "Baidu", + "Chutes", + "DeepInfra", + "DigitalOcean", + "GMICloud", + "Novita", + "Parasail", + "Phala", + "SiliconFlow", + "StreamLake", + "Venice", + "Z.AI" + ], + "uptime_24h": 99.99, + "latency_ms": null, + "throughput_tps": null + }, + "scores": {}, + "benchmarks": { + "artificial_analysis": {}, + "eq_bench": { + "public_source_key": "zai-org/GLM-5", + "public_rubric_0_100": 75.0, + "public_elo_norm": 1526.0, + "public_traits_17": { + "demonstrated_empathy": 15.15, + "pragmatic_ei": 14.62, + "depth_of_insight": 16.71, + "social_dexterity": 14.04, + "emotional_reasoning": 15.69, + "message_tailoring": 13.73, + "boundary_setting": 12.77, + "safety_conscious": 12.15, + "moralising": 8.12, + "compliant": 10.69, + "challenging": 12.54, + "warmth": 13.85, + "validating": 13.54, + "analytical": 16.92, + "reactive": 9.73, + "conversational": 12.27, + "humanlike": 14.77 + } + }, + "pinchbench": {} + }, + "sources": { + "openrouter": true, + "artificial_analysis": false, + "eq_bench": false, + "pinchbench": false, + "eq_bench_public": true + }, + "created": 1770829182, + "last_updated": "2026-07-26T13:04:59Z" + }, { "id": "z-ai/glm-5-turbo", "canonical_slug": "z-ai/glm-5-turbo-20260315", @@ -2785,4 +3542,4 @@ } } ] -} +} \ No newline at end of file diff --git a/model-benchmarks/llms.txt b/model-benchmarks/llms.txt index ba37af9..87f30bf 100644 --- a/model-benchmarks/llms.txt +++ b/model-benchmarks/llms.txt @@ -1,5 +1,5 @@ # HeartCentered AI — LLM Model Benchmarks -# Generated: 2026-07-26T04:27:13Z +# Generated: 2026-07-26T13:04:59Z # Full data: https://heartcentered.ai/model-benchmarks/data/model-data.json # Web UI: https://heartcentered.ai/model-benchmarks/ @@ -31,6 +31,16 @@ EQ-Bench v3 Score: 68.65/100 PinchBench: 89.5% best, 78.1% avg (? runs) Description: Claude Haiku 4.5 is Anthropic’s fastest and most efficient model, delivering near-frontier intelligence at a fraction of the cost and latency of larger Claude models. Matching Claude Sonnet 4’s perfor +## Claude Opus 4.5 +Provider: Anthropic +ID: anthropic/claude-opus-4.5 +Context: 200,000 tokens +Max output: 64,000 tokens +Pricing: $5.00 input / $25.00 output per 1M tokens (blended: $10.00) +Capabilities: tool use, reasoning, vision, web search, structured output +Available via: Amazon Bedrock, Anthropic, Azure, Google +Description: Claude Opus 4.5 is Anthropic’s frontier reasoning model optimized for complex software engineering, agentic workflows, and long-horizon computer use. It offers strong multimodal capabilities, competit + ## Claude Opus 4.6 Provider: Anthropic ID: anthropic/claude-opus-4.6 @@ -47,6 +57,16 @@ EQ-Bench Elo: 1857.8 PinchBench: 93.3% best, 83.1% avg (19 runs) Description: Opus 4.6 is Anthropic’s strongest model for coding and long-running professional tasks. It is built for agents that operate across entire workflows rather than single prompts, making it especially eff +## Claude Opus 4.7 +Provider: Anthropic +ID: anthropic/claude-opus-4.7 +Context: 1,000,000 tokens +Max output: 128,000 tokens +Pricing: $5.00 input / $25.00 output per 1M tokens (blended: $10.00) +Capabilities: tool use, reasoning, vision, web search, structured output +Available via: Amazon Bedrock, Anthropic, Azure, Google +Description: Opus 4.7 is the next generation of Anthropic's Opus family, built for long-running, asynchronous agents. Building on the coding and agentic strengths of Opus 4.6, it delivers stronger performance on.. + ## Claude Opus 4.8 Provider: Anthropic ID: anthropic/claude-opus-4.8 @@ -57,6 +77,16 @@ Capabilities: tool use, reasoning, vision, web search, structured output Available via: Amazon Bedrock, Anthropic, Azure, Google Description: Claude Opus 4.8 is Anthropic's most capable generally available model in the Opus family. It supports text, image, and file inputs with text output, with reasoning support and a 1M-token... +## Claude Sonnet 4.5 +Provider: Anthropic +ID: anthropic/claude-sonnet-4.5 +Context: 1,000,000 tokens +Max output: 64,000 tokens +Pricing: $3.00 input / $15.00 output per 1M tokens (blended: $6.00) +Capabilities: tool use, reasoning, vision, web search, structured output +Available via: Amazon Bedrock, Anthropic, Azure, Google +Description: Claude Sonnet 4.5 is Anthropic’s most advanced Sonnet model to date, optimized for real-world agents and coding workflows. It delivers state-of-the-art performance on coding benchmarks such as SWE-ben + ## Claude Sonnet 4.6 Provider: Anthropic ID: anthropic/claude-sonnet-4.6 @@ -148,6 +178,16 @@ EQ-Bench v3 Score: 68.75/100 PinchBench: 89.8% best, 83.2% avg (11 runs) Description: MiniMax-M2.7 is a next-generation large language model designed for autonomous, real-world productivity and continuous improvement. Built to actively participate in its own evolution, M2.7 integrates +## Kimi K2 0711 +Provider: MoonshotAI +ID: moonshotai/kimi-k2 +Context: 131,072 tokens +Max output: 100,352 tokens +Pricing: $0.57 input / $2.30 output per 1M tokens (blended: $1.00) +Capabilities: tool use +Available via: Novita +Description: Kimi K2 Instruct is a large-scale Mixture-of-Experts (MoE) language model developed by Moonshot AI, featuring 1 trillion total parameters with 32 billion active per forward pass. It is optimized for.. + ## Kimi K2.5 Provider: MoonshotAI ID: moonshotai/kimi-k2.5 @@ -168,6 +208,45 @@ Capabilities: tool use, reasoning, vision, structured output Available via: AtlasCloud, Baidu, BaseTen, Chutes, Cloudflare, CoreWeave, Crusoe, Decart, DeepInfra, DigitalOcean, Fireworks, Inceptron, ModelRun, Moonshot AI, Novita, Parasail, Phala, SiliconFlow, StreamLake, Together, Venice Description: Kimi K2.6 is Moonshot AI's next-generation multimodal model, designed for long-horizon coding, coding-driven UI/UX generation, and multi-agent orchestration. It handles complex end-to-end coding tasks +## Hermes 4 405B +Provider: Nous +ID: nousresearch/hermes-4-405b +Context: 131,072 tokens +Pricing: $1.00 input / $3.00 output per 1M tokens (blended: $1.50) +Capabilities: reasoning, structured output +Available via: Nebius +Description: Hermes 4 is a large-scale reasoning model built on Meta-Llama-3.1-405B and released by Nous Research. It introduces a hybrid reasoning mode, where the model can choose to deliberate internally with... + +## GPT-5.1 +Provider: OpenAI +ID: openai/gpt-5.1 +Context: 400,000 tokens +Max output: 128,000 tokens +Pricing: $1.25 input / $10.00 output per 1M tokens (blended: $3.44) +Capabilities: tool use, reasoning, vision, web search, structured output +Available via: Azure, OpenAI +Description: GPT-5.1 is the latest frontier-grade model in the GPT-5 series, offering stronger general-purpose reasoning, improved instruction adherence, and a more natural conversational style compared to GPT-5. + +## GPT-5.2 +Provider: OpenAI +ID: openai/gpt-5.2 +Context: 400,000 tokens +Max output: 128,000 tokens +Pricing: $1.75 input / $14.00 output per 1M tokens (blended: $4.81) +Capabilities: tool use, reasoning, vision, web search, structured output +Available via: Azure, OpenAI +Description: GPT-5.2 is the latest frontier-grade model in the GPT-5 series, offering stronger agentic and long context perfomance compared to GPT-5.1. It uses adaptive reasoning to allocate computation dynamicall + +## GPT-5.3 Chat +Provider: OpenAI +ID: openai/gpt-5.3-chat +Context: 128,000 tokens +Max output: 16,384 tokens +Pricing: $1.75 input / $14.00 output per 1M tokens (blended: $4.81) +Capabilities: tool use, vision, web search, structured output +Available via: Azure, OpenAI +Description: GPT-5.3 Chat is an update to ChatGPT's most-used model that makes everyday conversations smoother, more useful, and more directly helpful. It delivers more accurate answers with better contextualizati + ## GPT-5.4 Provider: OpenAI ID: openai/gpt-5.4 @@ -281,6 +360,16 @@ Capabilities: tool use, reasoning, structured output Available via: Cloudflare, DeepInfra, Novita, Venice Description: As a 30B-class SOTA model, GLM-4.7-Flash offers a new option that balances performance and efficiency. It is further optimized for agentic coding use cases, strengthening coding capabilities, long-hor +## GLM 5 +Provider: Z.ai +ID: z-ai/glm-5 +Context: 204,800 tokens +Max output: 131,072 tokens +Pricing: $0.95 input / $2.55 output per 1M tokens (blended: $1.35) +Capabilities: tool use, reasoning, structured output +Available via: Amazon Bedrock, AtlasCloud, Baidu, Chutes, DeepInfra, DigitalOcean, GMICloud, Novita, Parasail, Phala, SiliconFlow, StreamLake, Venice, Z.AI +Description: GLM-5 is Z.ai’s flagship open-source foundation model engineered for complex systems design and long-horizon agent workflows. Built for expert developers, it delivers production-grade performance on l + ## GLM 5 Turbo Provider: Z.ai ID: z-ai/glm-5-turbo diff --git a/model-benchmarks/scripts/eqbench_public.py b/model-benchmarks/scripts/eqbench_public.py index 6cf6fa9..0341f3e 100644 --- a/model-benchmarks/scripts/eqbench_public.py +++ b/model-benchmarks/scripts/eqbench_public.py @@ -80,6 +80,31 @@ "moonshotai/kimi-k2.5": "moonshotai/Kimi-K2.5", "google/gemma-4-26b-a4b-it": "google/gemma-4-26B-A4B-it", "qwen/qwen3.5-397b-a17b": "Qwen/Qwen3.5-397B-A17B", + # --- added 2026-07-26: each confirmed live on OpenRouter AND present + # upstream. Verified pairwise by name + canonical_slug, never by substring. + # eqbench key -> openrouter canonical_slug + # claude-opus-4-7 -> anthropic/claude-4.7-opus-20260416 + # gpt-5.2 -> openai/gpt-5.2-20251211 + # claude-opus-4-5-20251101 -> anthropic/claude-4.5-opus-20251124 + # zai-org/GLM-5 -> z-ai/glm-5-20260211 + # claude-sonnet-4.5 -> anthropic/claude-4.5-sonnet-20250929 + # gpt-5.3-chat -> openai/gpt-5.3-chat-20260303 + # gpt-5.1-2025-11-13 -> openai/gpt-5.1-20251113 + # moonshotai/Kimi-K2-Instruct-> moonshotai/kimi-k2 ("Kimi K2 0711") + # NousResearch/Hermes-4-405B -> nousresearch/hermes-4-405b + "anthropic/claude-opus-4.7": "claude-opus-4-7", + "openai/gpt-5.2": "gpt-5.2", + "anthropic/claude-opus-4.5": "claude-opus-4-5-20251101", + "z-ai/glm-5": "zai-org/GLM-5", + "anthropic/claude-sonnet-4.5": "claude-sonnet-4.5", + "openai/gpt-5.3-chat": "gpt-5.3-chat", + "openai/gpt-5.1": "gpt-5.1-2025-11-13", + "moonshotai/kimi-k2": "moonshotai/Kimi-K2-Instruct", + "nousresearch/hermes-4-405b": "NousResearch/Hermes-4-405B", + # NOTE deliberately absent, verified NOT live on OpenRouter (2026-07-26), + # so there is no row to attach a real score to: + # grok-4, grok-4.1-fast, google/gemma-4-12B-it, + # HiveLabsAI/hivemind-32b-preview, openrouter/horizon-alpha # NOTE deliberately absent, verified as NOT on the public leaderboard: # anthropic/claude-opus-5, anthropic/claude-opus-5-fast, x-ai/grok-4.5, # moonshotai/kimi-k3, openai/gpt-5.6-*, google/gemini-3.6-flash diff --git a/model-benchmarks/tests/test_fetch_model.py b/model-benchmarks/tests/test_fetch_model.py index 4a775a0..9c6c3c8 100644 --- a/model-benchmarks/tests/test_fetch_model.py +++ b/model-benchmarks/tests/test_fetch_model.py @@ -193,6 +193,43 @@ def test_map_has_no_fuzzy_lookalikes(self): self.assertNotIn(risky, eqp.EQBENCH_PUBLIC_MAP) +class PublicMapIsInjective(unittest.TestCase): + """Two OpenRouter ids must never claim the same upstream EQ-Bench row. + + That is exactly the shape of the inherited-score bug (D-001): one real run + presented as two models' own results. Cheap to assert, so assert it. + """ + + def test_no_two_models_share_an_eqbench_key(self): + seen: dict[str, str] = {} + for model_id, key in eqp.EQBENCH_PUBLIC_MAP.items(): + self.assertNotIn( + key, + seen, + f"{model_id} and {seen.get(key)} both map to upstream {key!r} " + "— one of them would be showing the other's score", + ) + seen[key] = model_id + + +class Glm5TurboProvenance(unittest.TestCase): + """z-ai/glm-5-turbo and z-ai/glm-5 are DIFFERENT models. + + glm-5-turbo carries a legacy 11-trait EQ block of unknown provenance. It was + suspected of being GLM-5's data. Checked against upstream 2026-07-26: on the + shared 0-10 rescaling only 4 of 11 traits agree within rounding (humanlike + 7.20 vs 6.76, social_iq 7.20 vs 6.94), and the stored elo 1631.9 differs from + GLM-5's public elo_norm 1526.0. So it is NOT a copy of GLM-5 and stays put — + but glm-5-turbo must never be mapped to the GLM-5 row. + """ + + def test_turbo_is_not_mapped_to_glm5(self): + self.assertNotIn("z-ai/glm-5-turbo", eqp.EQBENCH_PUBLIC_MAP) + + def test_glm5_maps_to_glm5(self): + self.assertEqual(eqp.EQBENCH_PUBLIC_MAP.get("z-ai/glm-5"), "zai-org/GLM-5") + + class DiscoverNewModels(unittest.TestCase): def test_untracked_and_live_is_discovered(self): data = {"models": [{"id": "anthropic/claude-opus-4.6"}]} From 4aca944defaf624928caccca7b30d05a76ab8185 Mon Sep 17 00:00:00 2001 From: Nick Sullivan Date: Sun, 26 Jul 2026 12:26:07 -0500 Subject: [PATCH 5/6] Pin model-benchmarks to Python 3.13 via uv venv Apple's python3 is 3.9 and chokes on str | None in fetch-model.py. Pin 3.13 in .python-version, document the uv venv workflow, and switch all AGENTS.md commands to uv run --no-project python. Ignore .venv/ and __pycache__/. --- .gitignore | 4 ++++ .python-version | 1 + model-benchmarks/AGENTS.md | 17 ++++++++++++----- 3 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 .python-version diff --git a/.gitignore b/.gitignore index 12d235f..ca189a3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,9 @@ # Local secrets (not committed) env.local +# Python virtualenv (uv) +.venv/ +__pycache__/ + # Playwright MCP artifacts .playwright-mcp/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/model-benchmarks/AGENTS.md b/model-benchmarks/AGENTS.md index b814779..357e289 100644 --- a/model-benchmarks/AGENTS.md +++ b/model-benchmarks/AGENTS.md @@ -19,9 +19,16 @@ Static HTML/CSS/JS page — no build step, no framework. ## Data Pipeline +### Python environment + +Python 3.13 via uv. The repo pins it in `.python-version`; create the venv once with +`uv venv --python 3.13` from the repo root, then prefix every command with +`uv run --no-project python`. Do NOT use bare `python3` — on macOS that resolves to +Apple's 3.9 and the scripts fail on `str | None` syntax. Stdlib only, no deps to install. + ### Adding a new model -1. **Fetch from OpenRouter** — `python scripts/fetch-model.py ` +1. **Fetch from OpenRouter** — `uv run --no-project python scripts/fetch-model.py ` - Gets pricing, context window, capabilities, endpoint stats - Optionally enriches with Artificial Analysis (needs AA_API_KEY in env.local) @@ -40,7 +47,7 @@ Static HTML/CSS/JS page — no build step, no framework. - Elo and traits show in tooltip on hover 5. **Regenerate llms.txt** — - `python -c "import sys; sys.path.insert(0, 'model-benchmarks/scripts'); fm = __import__('importlib').import_module('fetch-model'); data = fm.load_model_data(); fm.generate_llms_txt(data)"` + `uv run --no-project python -c "import sys; sys.path.insert(0, 'model-benchmarks/scripts'); fm = __import__('importlib').import_module('fetch-model'); data = fm.load_model_data(); fm.generate_llms_txt(data)"` 6. **Update Model Personalities** — the "Model Personalities" section in `index.html` has hardcoded editorial insight cards (9 curated profiles with personality reads and @@ -50,8 +57,8 @@ Static HTML/CSS/JS page — no build step, no framework. ### Refreshing existing models ```bash -python scripts/fetch-model.py --refresh # all models -python scripts/fetch-model.py --refresh --no-aa # skip AA (rate limited at 1000/day) +uv run --no-project python scripts/fetch-model.py --refresh # all models +uv run --no-project python scripts/fetch-model.py --refresh --no-aa # skip AA (rate limited at 1000/day) ``` `merge_model()` preserves manually-entered data: eq_bench, arena, pinchbench, scores, @@ -70,7 +77,7 @@ models keep their last-known data and are flagged `"delisted": true` with a note ### Tests ```bash -python -m unittest discover -s model-benchmarks/tests +uv run --no-project python -m unittest discover -s model-benchmarks/tests ``` Stdlib unittest, no dependencies. Covers the merge guarantees above. From 68bc4e5d5797be5c6a0ffe699d9bd918aca17987 Mon Sep 17 00:00:00 2001 From: Nick Sullivan Date: Sun, 26 Jul 2026 12:26:23 -0500 Subject: [PATCH 6/6] ci: match local Python 3.13 pin in GitHub Actions --- .github/workflows/pre-commit.yml | 2 +- .github/workflows/refresh-model-benchmarks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 8a633a3..6a15248 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -12,5 +12,5 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/refresh-model-benchmarks.yml b/.github/workflows/refresh-model-benchmarks.yml index d85b8f1..784b2c1 100644 --- a/.github/workflows/refresh-model-benchmarks.yml +++ b/.github/workflows/refresh-model-benchmarks.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - name: Refresh OpenRouter metadata run: python model-benchmarks/scripts/fetch-model.py --refresh --no-aa