From 9953e88391fbc14ec12cfbf08d0bf82ef6fe2772 Mon Sep 17 00:00:00 2001 From: yl231 <111810457+yl231@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:16:02 +0000 Subject: [PATCH] Sync leaderboard data from RouterArena@e95ba6764eb4a02d89a32c0cc7e858ec19b3ee6f --- src/data/flip_labels/flip_labels_vllm-sr.json | 62 +++++++++--------- src/data/routerMetrics/leaderboard.json | 64 +++++++++---------- 2 files changed, 63 insertions(+), 63 deletions(-) diff --git a/src/data/flip_labels/flip_labels_vllm-sr.json b/src/data/flip_labels/flip_labels_vllm-sr.json index e584bb1..17ec37a 100644 --- a/src/data/flip_labels/flip_labels_vllm-sr.json +++ b/src/data/flip_labels/flip_labels_vllm-sr.json @@ -221,11 +221,11 @@ }, { "global index": "FinQA_56", - "flip": 0 + "flip": 1 }, { "global index": "FinQA_60", - "flip": 0 + "flip": 1 }, { "global index": "GSM8K_43", @@ -297,31 +297,31 @@ }, { "global index": "GeoGraphyData_100k_42", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_105", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_114", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_118", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_131", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_136", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_181", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_237", @@ -329,7 +329,7 @@ }, { "global index": "LiveCodeBench_271", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_350", @@ -345,7 +345,7 @@ }, { "global index": "LiveCodeBench_43", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_431", @@ -357,23 +357,23 @@ }, { "global index": "LiveCodeBench_476", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_485", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_49", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_491", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_499", - "flip": 0 + "flip": 1 }, { "global index": "MATH_108", @@ -437,7 +437,7 @@ }, { "global index": "MMLUPro_business_507", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_6", @@ -645,7 +645,7 @@ }, { "global index": "MMLUPro_health_5144", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5214", @@ -841,7 +841,7 @@ }, { "global index": "MMLUPro_physics_8888", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_9017", @@ -961,7 +961,7 @@ }, { "global index": "MedMCQA_1005", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_1054", @@ -1061,7 +1061,7 @@ }, { "global index": "NarrativeQA_131", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_1683", @@ -1357,7 +1357,7 @@ }, { "global index": "QANTA_Fine Arts_1212", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Fine Arts_1702", @@ -1369,7 +1369,7 @@ }, { "global index": "QANTA_Fine Arts_865", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Geography_1023", @@ -1385,7 +1385,7 @@ }, { "global index": "QANTA_History_1084", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_1154", @@ -1393,7 +1393,7 @@ }, { "global index": "QANTA_History_433", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_473", @@ -1421,7 +1421,7 @@ }, { "global index": "QANTA_Literature_1727", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_1843", @@ -1433,11 +1433,11 @@ }, { "global index": "QANTA_Literature_408", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_475", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_833", @@ -1461,7 +1461,7 @@ }, { "global index": "QANTA_Science_1473", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Science_308", @@ -1477,11 +1477,11 @@ }, { "global index": "QANTA_Social Science_2", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Social Science_77", - "flip": 0 + "flip": 1 }, { "global index": "SocialiQA_13810", diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index 3993e9e..9497aa9 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -1,15 +1,4 @@ [ - { - "Router Name": "vllm", - "Arena Score": 75.38, - "Optimal Selection Score": 20.12, - "Optimal Cost Score": 24.52, - "Optimal Acc. Score": 89.87, - "Robustness Score": 73.1, - "Latency Score": null, - "Accuracy": 75.97, - "Cost per 1k": 0.11 - }, { "Router Name": "Sqwish Router", "Arena Score": 75.27, @@ -32,17 +21,6 @@ "Accuracy": 74.72, "Cost per 1k": 0.13 }, - { - "Router Name": "Nadir Router", - "Arena Score": 73.33, - "Optimal Selection Score": null, - "Optimal Cost Score": null, - "Optimal Acc. Score": null, - "Robustness Score": 25.48, - "Latency Score": null, - "Accuracy": 74.87, - "Cost per 1k": 0.29 - }, { "Router Name": "Weave Router", "Arena Score": 72.82, @@ -55,26 +33,37 @@ "Cost per 1k": 0.94 }, { - "Router Name": "OrcaRouter-Adaptive", - "Arena Score": 72.08, + "Router Name": "Nadir Router", + "Arena Score": 72.29, "Optimal Selection Score": null, "Optimal Cost Score": null, "Optimal Acc. Score": null, - "Robustness Score": 22.62, + "Robustness Score": 25.48, "Latency Score": null, - "Accuracy": 75.54, - "Cost per 1k": 1.0 + "Accuracy": 75.01, + "Cost per 1k": 0.68 }, { - "Router Name": "azure", - "Arena Score": 71.87, + "Router Name": "vllm", + "Arena Score": 72.15, + "Optimal Selection Score": 20.77, + "Optimal Cost Score": 17.19, + "Optimal Acc. Score": 90.0, + "Robustness Score": 73.1, + "Latency Score": null, + "Accuracy": 73.19, + "Cost per 1k": 0.23 + }, + { + "Router Name": "OrcaRouter-Adaptive", + "Arena Score": 72.08, "Optimal Selection Score": null, "Optimal Cost Score": null, "Optimal Acc. Score": null, - "Robustness Score": 71.43, + "Robustness Score": 22.62, "Latency Score": null, - "Accuracy": 72.82, - "Cost per 1k": 0.22 + "Accuracy": 75.54, + "Cost per 1k": 1.0 }, { "Router Name": "R2-Router", @@ -87,6 +76,17 @@ "Accuracy": 71.23, "Cost per 1k": 0.06 }, + { + "Router Name": "azure", + "Arena Score": 70.42, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 71.43, + "Latency Score": null, + "Accuracy": 72.94, + "Cost per 1k": 0.73 + }, { "Router Name": "Auto Router", "Arena Score": 70.05,