diff --git a/src/data/flip_labels/flip_labels_barouter.json b/src/data/flip_labels/flip_labels_barouter.json new file mode 100644 index 0000000..d7844c0 --- /dev/null +++ b/src/data/flip_labels/flip_labels_barouter.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 1 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 1 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 1 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 1 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 1 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 1 + }, + { + "global index": "ArcMMLU_646", + "flip": 1 + }, + { + "global index": "ArcMMLU_659", + "flip": 1 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 1 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 1 + }, + { + "global index": "AsDiv_1165", + "flip": 1 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 1 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 1 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 1 + }, + { + "global index": "Ethics_deontology_32", + "flip": 1 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 1 + }, + { + "global index": "Ethics_justice_45", + "flip": 1 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 1 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 1 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 1 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 1 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 1 + }, + { + "global index": "GeoBench_1102", + "flip": 1 + }, + { + "global index": "GeoBench_1113", + "flip": 1 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 1 + }, + { + "global index": "GeoBench_502", + "flip": 1 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 1 + }, + { + "global index": "GeoBench_87", + "flip": 1 + }, + { + "global index": "GeoBench_915", + "flip": 1 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 1 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 1 + }, + { + "global index": "LiveCodeBench_237", + "flip": 1 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 1 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 1 + }, + { + "global index": "LiveCodeBench_431", + "flip": 1 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 1 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 1 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 1 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 1 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 1 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 1 + }, + { + "global index": "MedMCQA_1054", + "flip": 1 + }, + { + "global index": "MedMCQA_1298", + "flip": 1 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 1 + }, + { + "global index": "MedMCQA_145", + "flip": 1 + }, + { + "global index": "MedMCQA_2010", + "flip": 1 + }, + { + "global index": "MedMCQA_2323", + "flip": 1 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 1 + }, + { + "global index": "MedMCQA_511", + "flip": 1 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 1 + }, + { + "global index": "MedMCQA_853", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 1 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 1 + }, + { + "global index": "NarrativeQA_3282", + "flip": 1 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 1 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 1 + }, + { + "global index": "NarrativeQA_7678", + "flip": 1 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 1 + }, + { + "global index": "NarrativeQA_8598", + "flip": 1 + }, + { + "global index": "NarrativeQA_927", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 1 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 1 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 1 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 1 + }, + { + "global index": "PubMedQA_0", + "flip": 1 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 1 + }, + { + "global index": "PubMedQA_238", + "flip": 1 + }, + { + "global index": "PubMedQA_250", + "flip": 1 + }, + { + "global index": "PubMedQA_337", + "flip": 1 + }, + { + "global index": "PubMedQA_362", + "flip": 1 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 1 + }, + { + "global index": "PubMedQA_575", + "flip": 1 + }, + { + "global index": "PubMedQA_582", + "flip": 1 + }, + { + "global index": "PubMedQA_588", + "flip": 1 + }, + { + "global index": "PubMedQA_610", + "flip": 1 + }, + { + "global index": "PubMedQA_63", + "flip": 1 + }, + { + "global index": "PubMedQA_643", + "flip": 1 + }, + { + "global index": "PubMedQA_687", + "flip": 1 + }, + { + "global index": "PubMedQA_722", + "flip": 1 + }, + { + "global index": "PubMedQA_73", + "flip": 1 + }, + { + "global index": "PubMedQA_755", + "flip": 1 + }, + { + "global index": "PubMedQA_8", + "flip": 1 + }, + { + "global index": "PubMedQA_81", + "flip": 1 + }, + { + "global index": "PubMedQA_854", + "flip": 1 + }, + { + "global index": "PubMedQA_905", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 1 + }, + { + "global index": "QANTA_History_1084", + "flip": 1 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 1 + }, + { + "global index": "QANTA_Literature_386", + "flip": 1 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 1 + }, + { + "global index": "QANTA_Literature_833", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 1 + }, + { + "global index": "QANTA_Science_1360", + "flip": 1 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 1 + }, + { + "global index": "SocialiQA_13810", + "flip": 1 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 1 + }, + { + "global index": "SocialiQA_7839", + "flip": 1 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 1 + }, + { + "global index": "WMT19-de-en_715", + "flip": 1 + }, + { + "global index": "WMT19-de-en_883", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 1 + } +] diff --git a/src/data/flip_labels/flip_labels_chuzom-solo-v32.json b/src/data/flip_labels/flip_labels_chuzom-solo-v32.json new file mode 100644 index 0000000..9450917 --- /dev/null +++ b/src/data/flip_labels/flip_labels_chuzom-solo-v32.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 0 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 0 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 0 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 0 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 0 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 0 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] diff --git a/src/data/flip_labels/flip_labels_cross-router.json b/src/data/flip_labels/flip_labels_cross-router.json index ad559c6..d9c1a42 100644 --- a/src/data/flip_labels/flip_labels_cross-router.json +++ b/src/data/flip_labels/flip_labels_cross-router.json @@ -13,7 +13,7 @@ }, { "global index": "ArcMMLU_123", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_16", @@ -25,7 +25,7 @@ }, { "global index": "ArcMMLU_230", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_293", @@ -41,7 +41,7 @@ }, { "global index": "ArcMMLU_443", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_496", @@ -49,7 +49,7 @@ }, { "global index": "ArcMMLU_631", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_646", @@ -65,7 +65,7 @@ }, { "global index": "ArcMMLU_685", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_689", @@ -85,15 +85,15 @@ }, { "global index": "AsDiv_1165", - "flip": 0 + "flip": 1 }, { "global index": "AsDiv_1347", - "flip": 0 + "flip": 1 }, { "global index": "AsDiv_472", - "flip": 0 + "flip": 1 }, { "global index": "AsDiv_733", @@ -153,15 +153,15 @@ }, { "global index": "Ethics_commonsense_90", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_deontology_0", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_2", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_31", @@ -169,7 +169,7 @@ }, { "global index": "Ethics_deontology_32", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_56", @@ -181,7 +181,7 @@ }, { "global index": "Ethics_justice_45", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_justice_76", @@ -193,7 +193,7 @@ }, { "global index": "Ethics_virtue_14", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_virtue_26", @@ -209,19 +209,19 @@ }, { "global index": "Ethics_virtue_51", - "flip": 1 + "flip": 0 }, { "global index": "FinQA_149", - "flip": 1 + "flip": 0 }, { "global index": "FinQA_208", - "flip": 1 + "flip": 0 }, { "global index": "FinQA_56", - "flip": 1 + "flip": 0 }, { "global index": "FinQA_60", @@ -229,7 +229,7 @@ }, { "global index": "GSM8K_43", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_1002", @@ -269,7 +269,7 @@ }, { "global index": "GeoBench_591", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_766", @@ -281,11 +281,11 @@ }, { "global index": "GeoBench_915", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_944", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_968", @@ -293,7 +293,7 @@ }, { "global index": "GeoGraphyData_100k_27", - "flip": 1 + "flip": 0 }, { "global index": "GeoGraphyData_100k_42", @@ -301,19 +301,19 @@ }, { "global index": "LiveCodeBench_105", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_114", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_118", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_131", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_136", @@ -321,7 +321,7 @@ }, { "global index": "LiveCodeBench_181", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_237", @@ -329,7 +329,7 @@ }, { "global index": "LiveCodeBench_271", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_350", @@ -341,7 +341,7 @@ }, { "global index": "LiveCodeBench_405", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_43", @@ -349,7 +349,7 @@ }, { "global index": "LiveCodeBench_431", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_437", @@ -369,7 +369,7 @@ }, { "global index": "LiveCodeBench_491", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_499", @@ -377,11 +377,11 @@ }, { "global index": "MATH_108", - "flip": 0 + "flip": 1 }, { "global index": "MATH_442", - "flip": 0 + "flip": 1 }, { "global index": "MATH_53", @@ -405,7 +405,7 @@ }, { "global index": "MMLUPro_biology_3188", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_3215", @@ -413,27 +413,27 @@ }, { "global index": "MMLUPro_biology_3225", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_226", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_294", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_378", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_430", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_503", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_507", @@ -441,11 +441,11 @@ }, { "global index": "MMLUPro_business_6", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_784", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_chemistry_3796", @@ -465,7 +465,7 @@ }, { "global index": "MMLUPro_chemistry_4407", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9086", @@ -477,15 +477,15 @@ }, { "global index": "MMLUPro_computer science_9136", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9138", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9149", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9200", @@ -497,19 +497,19 @@ }, { "global index": "MMLUPro_computer science_9239", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9264", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9285", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9289", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9414", @@ -521,7 +521,7 @@ }, { "global index": "MMLUPro_computer science_9430", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9452", @@ -529,11 +529,11 @@ }, { "global index": "MMLUPro_computer science_9471", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9475", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5769", @@ -545,7 +545,7 @@ }, { "global index": "MMLUPro_economics_5931", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5965", @@ -557,7 +557,7 @@ }, { "global index": "MMLUPro_economics_6122", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_6135", @@ -565,7 +565,7 @@ }, { "global index": "MMLUPro_economics_6325", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_6353", @@ -585,7 +585,7 @@ }, { "global index": "MMLUPro_engineering_10195", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10199", @@ -597,15 +597,15 @@ }, { "global index": "MMLUPro_engineering_10342", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10395", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10428", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10432", @@ -617,7 +617,7 @@ }, { "global index": "MMLUPro_engineering_10537", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10701", @@ -625,11 +625,11 @@ }, { "global index": "MMLUPro_engineering_10823", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10864", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_4885", @@ -645,7 +645,7 @@ }, { "global index": "MMLUPro_health_5144", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5214", @@ -653,15 +653,15 @@ }, { "global index": "MMLUPro_health_5215", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_5261", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_5473", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5514", @@ -673,11 +673,11 @@ }, { "global index": "MMLUPro_history_4490", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4497", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4509", @@ -685,11 +685,11 @@ }, { "global index": "MMLUPro_history_4517", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4523", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4605", @@ -709,7 +709,7 @@ }, { "global index": "MMLUPro_history_4749", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4752", @@ -717,15 +717,15 @@ }, { "global index": "MMLUPro_history_4774", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4810", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4833", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4836", @@ -737,19 +737,19 @@ }, { "global index": "MMLUPro_law_1007", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1031", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1386", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1462", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1484", @@ -757,7 +757,7 @@ }, { "global index": "MMLUPro_law_1518", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1818", @@ -765,27 +765,27 @@ }, { "global index": "MMLUPro_law_806", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_899", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_6429", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6526", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6623", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6848", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_7101", @@ -797,7 +797,7 @@ }, { "global index": "MMLUPro_math_7284", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7451", @@ -813,7 +813,7 @@ }, { "global index": "MMLUPro_philosophy_9536", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_philosophy_9663", @@ -833,19 +833,19 @@ }, { "global index": "MMLUPro_physics_7887", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_7893", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_physics_8888", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_physics_9017", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2005", @@ -857,23 +857,23 @@ }, { "global index": "MMLUPro_psychology_2329", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2406", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2420", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2450", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2457", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2524", @@ -897,11 +897,11 @@ }, { "global index": "MMLU_formal_logic_7", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_70", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_85", @@ -909,31 +909,31 @@ }, { "global index": "MMLU_management_4", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_management_41", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_management_77", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_management_91", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_management_93", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_158", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_1742", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_202", @@ -945,7 +945,7 @@ }, { "global index": "MathQA_2102", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_2851", @@ -953,7 +953,7 @@ }, { "global index": "MathQA_827", - "flip": 0 + "flip": 1 }, { "global index": "MathQA_84", @@ -965,7 +965,7 @@ }, { "global index": "MedMCQA_1054", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_1298", @@ -973,15 +973,15 @@ }, { "global index": "MedMCQA_1309", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_1362", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_145", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2010", @@ -989,11 +989,11 @@ }, { "global index": "MedMCQA_2323", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2366", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2581", @@ -1009,11 +1009,11 @@ }, { "global index": "MedMCQA_643", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_853", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_126", @@ -1025,19 +1025,19 @@ }, { "global index": "MusicTheoryBench_147", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_152", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_188", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_189", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_240", @@ -1053,15 +1053,15 @@ }, { "global index": "MusicTheoryBench_340", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_70", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_131", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_1683", @@ -1081,7 +1081,7 @@ }, { "global index": "NarrativeQA_4102", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_4128", @@ -1097,7 +1097,7 @@ }, { "global index": "NarrativeQA_5022", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_5259", @@ -1109,7 +1109,7 @@ }, { "global index": "NarrativeQA_5894", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_6829", @@ -1117,7 +1117,7 @@ }, { "global index": "NarrativeQA_7678", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_7964", @@ -1137,7 +1137,7 @@ }, { "global index": "OpenTDB_Animals_2545", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Animals_262", @@ -1149,7 +1149,7 @@ }, { "global index": "OpenTDB_Celebrities_1078", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Celebrities_1456", @@ -1157,7 +1157,7 @@ }, { "global index": "OpenTDB_Celebrities_3577", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Celebrities_3968", @@ -1177,7 +1177,7 @@ }, { "global index": "OpenTDB_General Knowledge_2181", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_3404", @@ -1185,15 +1185,15 @@ }, { "global index": "OpenTDB_General Knowledge_3727", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_4019", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_2099", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_2346", @@ -1205,7 +1205,7 @@ }, { "global index": "OpenTDB_Geography_792", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_History_1162", @@ -1213,15 +1213,15 @@ }, { "global index": "OpenTDB_History_2026", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_History_3712", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_3902", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Science & Nature_1175", @@ -1229,11 +1229,11 @@ }, { "global index": "OpenTDB_Science & Nature_1560", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Science & Nature_3716", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Science & Nature_476", @@ -1261,7 +1261,7 @@ }, { "global index": "PubMedQA_0", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_154", @@ -1273,11 +1273,11 @@ }, { "global index": "PubMedQA_238", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_250", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_337", @@ -1289,7 +1289,7 @@ }, { "global index": "PubMedQA_437", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_510", @@ -1301,7 +1301,7 @@ }, { "global index": "PubMedQA_575", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_582", @@ -1309,7 +1309,7 @@ }, { "global index": "PubMedQA_588", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_610", @@ -1317,7 +1317,7 @@ }, { "global index": "PubMedQA_63", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_643", @@ -1349,7 +1349,7 @@ }, { "global index": "PubMedQA_854", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_905", @@ -1357,7 +1357,7 @@ }, { "global index": "QANTA_Fine Arts_1212", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Fine Arts_1702", @@ -1365,11 +1365,11 @@ }, { "global index": "QANTA_Fine Arts_828", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Fine Arts_865", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Geography_1023", @@ -1377,7 +1377,7 @@ }, { "global index": "QANTA_Geography_1555", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Geography_304", @@ -1385,15 +1385,15 @@ }, { "global index": "QANTA_History_1084", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_1154", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_433", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_473", @@ -1401,7 +1401,7 @@ }, { "global index": "QANTA_History_926", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1045", @@ -1421,7 +1421,7 @@ }, { "global index": "QANTA_Literature_1727", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1843", @@ -1449,7 +1449,7 @@ }, { "global index": "QANTA_Philosophy_499", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Philosophy_91", @@ -1457,11 +1457,11 @@ }, { "global index": "QANTA_Science_1360", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_1473", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_308", @@ -1469,15 +1469,15 @@ }, { "global index": "QANTA_Science_619", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_1847", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_2", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_77", @@ -1493,7 +1493,7 @@ }, { "global index": "SocialiQA_26846", - "flip": 0 + "flip": 1 }, { "global index": "SocialiQA_7839", @@ -1509,7 +1509,7 @@ }, { "global index": "SuperGLUE-ClozeTest_17965", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-ClozeTest_18766", @@ -1517,7 +1517,7 @@ }, { "global index": "SuperGLUE-Entailment_19410", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Entailment_19567", @@ -1529,15 +1529,15 @@ }, { "global index": "SuperGLUE-Entailment_767", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_1408", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_3137", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_4046", @@ -1565,11 +1565,11 @@ }, { "global index": "SuperGLUE-Wic_19695", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_19738", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_20079", @@ -1593,7 +1593,7 @@ }, { "global index": "WMT19-cs-en_156", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-cs-en_246", @@ -1601,7 +1601,7 @@ }, { "global index": "WMT19-cs-en_568", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-de-en_46", @@ -1625,7 +1625,7 @@ }, { "global index": "WMT19-gu-en_116", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-gu-en_123", @@ -1637,11 +1637,11 @@ }, { "global index": "WMT19-gu-en_491", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-gu-en_968", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-kk-en_528", @@ -1669,7 +1669,7 @@ }, { "global index": "WMT19-zh-en_218", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-zh-en_252", diff --git a/src/data/flip_labels/flip_labels_nadir-tumbler.json b/src/data/flip_labels/flip_labels_nadir-tumbler.json new file mode 100644 index 0000000..676f52f --- /dev/null +++ b/src/data/flip_labels/flip_labels_nadir-tumbler.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 1 + }, + { + "global index": "ArcMMLU_123", + "flip": 1 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 1 + }, + { + "global index": "ArcMMLU_349", + "flip": 1 + }, + { + "global index": "ArcMMLU_378", + "flip": 1 + }, + { + "global index": "ArcMMLU_443", + "flip": 1 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 1 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 1 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 1 + }, + { + "global index": "AsDiv_1347", + "flip": 1 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 1 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 1 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 1 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 1 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 1 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 1 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 1 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 1 + }, + { + "global index": "GeoBench_1094", + "flip": 1 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 1 + }, + { + "global index": "GeoBench_124", + "flip": 1 + }, + { + "global index": "GeoBench_1243", + "flip": 1 + }, + { + "global index": "GeoBench_30", + "flip": 1 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 1 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 1 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 1 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 0 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 1 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 1 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 1 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 1 + }, + { + "global index": "MMLUPro_law_806", + "flip": 1 + }, + { + "global index": "MMLUPro_law_899", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 1 + }, + { + "global index": "MMLU_management_4", + "flip": 1 + }, + { + "global index": "MMLU_management_41", + "flip": 1 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 1 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 1 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 1 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 1 + }, + { + "global index": "MedMCQA_1362", + "flip": 1 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 1 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 1 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 1 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 1 + }, + { + "global index": "NarrativeQA_5259", + "flip": 1 + }, + { + "global index": "NarrativeQA_533", + "flip": 1 + }, + { + "global index": "NarrativeQA_5894", + "flip": 1 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 1 + }, + { + "global index": "NarrativeQA_8215", + "flip": 1 + }, + { + "global index": "NarrativeQA_8598", + "flip": 1 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 1 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 1 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 0 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 1 + }, + { + "global index": "PubMedQA_250", + "flip": 1 + }, + { + "global index": "PubMedQA_337", + "flip": 1 + }, + { + "global index": "PubMedQA_362", + "flip": 1 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 1 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 1 + }, + { + "global index": "PubMedQA_687", + "flip": 1 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 1 + }, + { + "global index": "PubMedQA_854", + "flip": 1 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 1 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 1 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 1 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 1 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 1 + }, + { + "global index": "WMT19-de-en_46", + "flip": 1 + }, + { + "global index": "WMT19-de-en_715", + "flip": 1 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] diff --git a/src/data/routerMetrics/category_scores.json b/src/data/routerMetrics/category_scores.json index 62db215..b34fd9f 100644 --- a/src/data/routerMetrics/category_scores.json +++ b/src/data/routerMetrics/category_scores.json @@ -18594,48 +18594,48 @@ "cross-router": { "metrics": { "easy": { - "accuracy": 96.1, + "accuracy": 96.5, "cost": 0.0001, "robustness": 59.1 }, "medium": { - "accuracy": 73.4, + "accuracy": 78.3, "cost": 0.0003, - "robustness": 52.2 + "robustness": 69.9 }, "hard": { - "accuracy": 35.0, - "cost": 0.0003, - "robustness": 66.1 + "accuracy": 41.0, + "cost": 0.0006, + "robustness": 78.9 }, "all": { - "accuracy": 75.2, - "cost": 0.0002, - "robustness": 59.0 + "accuracy": 78.2, + "cost": 0.0003, + "robustness": 67.1 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 93.0, - "cost": 0.0001, - "robustness": 37.5 + "accuracy": 95.3, + "cost": 0.0002, + "robustness": 50.0 }, "medium": { - "accuracy": 62.0, - "cost": 0.0004, - "robustness": 63.6 + "accuracy": 67.5, + "cost": 0.0006, + "robustness": 72.7 }, "hard": { - "accuracy": 17.0, - "cost": 0.0004, - "robustness": 60.0 + "accuracy": 23.8, + "cost": 0.0014, + "robustness": 80.0 }, "all": { - "accuracy": 65.7, - "cost": 0.0003, - "robustness": 54.2 + "accuracy": 70.1, + "cost": 0.0006, + "robustness": 66.7 } }, "subcategories": { @@ -18647,41 +18647,41 @@ "robustness": 100.0 }, "medium": { - "accuracy": 74.4, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 79.5, + "cost": 0.0003, + "robustness": 100.0 }, "hard": { - "accuracy": 19.6, - "cost": 0.0002, - "robustness": 50.0 + "accuracy": 28.6, + "cost": 0.0021, + "robustness": 100.0 }, "all": { - "accuracy": 63.6, - "cost": 0.0001, - "robustness": 40.0 + "accuracy": 68.2, + "cost": 0.0009, + "robustness": 100.0 } } }, "Music": { "metrics": { "easy": { - "accuracy": 96.0, + "accuracy": 92.9, "cost": 0.0002, - "robustness": 20.0 + "robustness": 40.0 }, "medium": { - "accuracy": 51.7, - "cost": 0.0007, - "robustness": 60.0 + "accuracy": 61.2, + "cost": 0.0008, + "robustness": 40.0 }, "hard": { "accuracy": 12.0, - "cost": 0.0012, + "cost": 0.0013, "robustness": 0.0 }, "all": { - "accuracy": 65.8, + "accuracy": 69.2, "cost": 0.0006, "robustness": 36.4 } @@ -18690,23 +18690,23 @@ "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 88.6, - "cost": 0.0001, + "accuracy": 95.7, + "cost": 0.0002, "robustness": 50.0 }, "medium": { - "accuracy": 69.0, - "cost": 0.0002, + "accuracy": 70.0, + "cost": 0.0005, "robustness": 100.0 }, "hard": { - "accuracy": 16.7, - "cost": 0.0003, + "accuracy": 24.2, + "cost": 0.0008, "robustness": 100.0 }, "all": { - "accuracy": 66.7, - "cost": 0.0002, + "accuracy": 71.9, + "cost": 0.0004, "robustness": 87.5 } } @@ -18716,62 +18716,62 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 96.5, - "cost": 0.0002, - "robustness": 73.1 + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 53.8 }, "medium": { - "accuracy": 77.7, - "cost": 0.0004, - "robustness": 47.6 + "accuracy": 84.4, + "cost": 0.0003, + "robustness": 52.4 }, "hard": { - "accuracy": 38.8, + "accuracy": 54.3, "cost": 0.0007, - "robustness": 80.0 + "robustness": 60.0 }, "all": { - "accuracy": 80.2, + "accuracy": 85.5, "cost": 0.0003, - "robustness": 66.1 + "robustness": 54.8 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 97.1, - "cost": 0.0002, - "robustness": 69.2 + "accuracy": 97.8, + "cost": 0.0001, + "robustness": 30.8 }, "medium": { - "accuracy": 81.2, - "cost": 0.0005, - "robustness": 41.2 + "accuracy": 89.9, + "cost": 0.0004, + "robustness": 64.7 }, "hard": { - "accuracy": 44.0, + "accuracy": 62.2, "cost": 0.0008, - "robustness": 76.9 + "robustness": 53.8 }, "all": { - "accuracy": 80.1, + "accuracy": 87.5, "cost": 0.0004, - "robustness": 60.5 + "robustness": 51.2 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 95.6, + "accuracy": 96.4, "cost": 0.0001, "robustness": 76.9 }, "medium": { - "accuracy": 62.4, + "accuracy": 60.0, "cost": 0.0001, - "robustness": 75.0 + "robustness": 0.0 }, "hard": { "accuracy": 8.3, @@ -18781,7 +18781,7 @@ "all": { "accuracy": 80.6, "cost": 0.0001, - "robustness": 78.9 + "robustness": 63.2 } } } @@ -18792,22 +18792,22 @@ "easy": { "accuracy": 97.6, "cost": 0.0001, - "robustness": 52.2 + "robustness": 39.1 }, "medium": { - "accuracy": 80.8, + "accuracy": 76.3, "cost": 0.0002, - "robustness": 16.7 + "robustness": 100.0 }, "hard": { - "accuracy": 14.8, - "cost": 0.0002, - "robustness": 40.0 + "accuracy": 18.1, + "cost": 0.0006, + "robustness": 70.0 }, "all": { - "accuracy": 75.7, + "accuracy": 75.3, "cost": 0.0002, - "robustness": 43.6 + "robustness": 56.4 } }, "subcategories": { @@ -18815,45 +18815,45 @@ "metrics": { "easy": { "accuracy": 97.6, - "cost": 0.0, - "robustness": 0.0 + "cost": 0.0001, + "robustness": 50.0 }, "medium": { - "accuracy": 77.8, + "accuracy": 88.9, "cost": 0.0001, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0, + "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 92.3, + "accuracy": 94.2, "cost": 0.0001, - "robustness": 0.0 + "robustness": 50.0 } } }, "Geography": { "metrics": { "easy": { - "accuracy": 97.2, - "cost": 0.0, + "accuracy": 98.2, + "cost": 0.0001, "robustness": 62.5 }, "medium": { - "accuracy": 69.0, - "cost": 0.0001, + "accuracy": 65.5, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 33.3, - "cost": 0.0001, + "accuracy": 41.7, + "cost": 0.0006, "robustness": 0 }, "all": { - "accuracy": 86.7, + "accuracy": 87.3, "cost": 0.0001, "robustness": 66.7 } @@ -18862,24 +18862,24 @@ "History": { "metrics": { "easy": { - "accuracy": 97.8, - "cost": 0.0002, - "robustness": 63.6 + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 18.2 }, "medium": { - "accuracy": 83.5, - "cost": 0.0003, - "robustness": 0.0 + "accuracy": 77.7, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 13.2, - "cost": 0.0002, - "robustness": 40.0 + "accuracy": 16.2, + "cost": 0.0006, + "robustness": 70.0 }, "all": { - "accuracy": 70.7, - "cost": 0.0002, - "robustness": 42.3 + "accuracy": 69.7, + "cost": 0.0003, + "robustness": 53.8 } } } @@ -18888,48 +18888,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 91.9, + "accuracy": 95.7, "cost": 0.0001, "robustness": 72.7 }, "medium": { - "accuracy": 50.9, - "cost": 0.0002, - "robustness": 77.8 + "accuracy": 77.2, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 65.0, + "accuracy": 68.0, "cost": 0.0002, - "robustness": 72.0 + "robustness": 92.0 }, "all": { - "accuracy": 69.6, + "accuracy": 78.5, "cost": 0.0001, - "robustness": 73.3 + "robustness": 88.9 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 91.9, + "accuracy": 95.7, "cost": 0.0001, "robustness": 72.7 }, "medium": { - "accuracy": 50.9, - "cost": 0.0002, - "robustness": 77.8 + "accuracy": 77.2, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 65.0, + "accuracy": 68.0, "cost": 0.0002, - "robustness": 72.0 + "robustness": 92.0 }, "all": { - "accuracy": 69.6, + "accuracy": 78.5, "cost": 0.0001, - "robustness": 73.3 + "robustness": 88.9 } } } @@ -18943,19 +18943,19 @@ "robustness": 100.0 }, "medium": { - "accuracy": 76.5, - "cost": 0.0002, + "accuracy": 88.7, + "cost": 0.0004, "robustness": 100.0 }, "hard": { - "accuracy": 35.7, - "cost": 0.0002, - "robustness": 75.0 + "accuracy": 40.5, + "cost": 0.0008, + "robustness": 91.7 }, "all": { - "accuracy": 50.1, - "cost": 0.0002, - "robustness": 79.3 + "accuracy": 54.8, + "cost": 0.0006, + "robustness": 93.1 } }, "subcategories": { @@ -18967,19 +18967,19 @@ "robustness": 100.0 }, "medium": { - "accuracy": 76.5, - "cost": 0.0002, + "accuracy": 88.7, + "cost": 0.0004, "robustness": 100.0 }, "hard": { - "accuracy": 35.7, - "cost": 0.0002, - "robustness": 75.0 + "accuracy": 40.5, + "cost": 0.0008, + "robustness": 91.7 }, "all": { - "accuracy": 50.1, - "cost": 0.0002, - "robustness": 79.3 + "accuracy": 54.8, + "cost": 0.0006, + "robustness": 93.1 } } } @@ -18988,22 +18988,22 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 95.6, + "accuracy": 94.4, "cost": 0.0001, - "robustness": 66.7 + "robustness": 63.0 }, "medium": { - "accuracy": 70.1, - "cost": 0.0002, - "robustness": 90.0 + "accuracy": 78.7, + "cost": 0.0001, + "robustness": 70.0 }, "hard": { - "accuracy": 33.3, - "cost": 0.0002, - "robustness": 57.1 + "accuracy": 34.6, + "cost": 0.0004, + "robustness": 100.0 }, "all": { - "accuracy": 80.7, + "accuracy": 82.7, "cost": 0.0002, "robustness": 70.5 } @@ -19012,33 +19012,33 @@ "Ethics": { "metrics": { "easy": { - "accuracy": 93.5, + "accuracy": 92.4, "cost": 0.0001, "robustness": 71.4 }, "medium": { - "accuracy": 45.9, + "accuracy": 72.9, "cost": 0.0001, - "robustness": 100.0 + "robustness": 60.0 }, "hard": { - "accuracy": 30.8, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 61.5, + "cost": 0.0, + "robustness": 100.0 }, "all": { - "accuracy": 76.3, + "accuracy": 85.2, "cost": 0.0001, - "robustness": 76.2 + "robustness": 71.4 } } }, "Philosophical logic": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 98.5, "cost": 0.0002, - "robustness": 50.0 + "robustness": 100.0 }, "medium": { "accuracy": 86.8, @@ -19046,38 +19046,38 @@ "robustness": 100.0 }, "hard": { - "accuracy": 80.0, + "accuracy": 60.0, "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 94.4, + "accuracy": 92.6, "cost": 0.0002, - "robustness": 71.4 + "robustness": 100.0 } } }, "Philosophy": { "metrics": { "easy": { - "accuracy": 94.6, + "accuracy": 89.2, "cost": 0.0001, - "robustness": 33.3 + "robustness": 66.7 }, "medium": { - "accuracy": 80.0, - "cost": 0.0002, + "accuracy": 77.5, + "cost": 0.0003, "robustness": 50.0 }, "hard": { - "accuracy": 35.6, - "cost": 0.0002, - "robustness": 66.7 + "accuracy": 33.3, + "cost": 0.0006, + "robustness": 100.0 }, "all": { - "accuracy": 68.0, - "cost": 0.0002, - "robustness": 50.0 + "accuracy": 64.8, + "cost": 0.0004, + "robustness": 75.0 } } }, @@ -19085,23 +19085,23 @@ "metrics": { "easy": { "accuracy": 96.7, - "cost": 0.0002, - "robustness": 83.3 + "cost": 0.0001, + "robustness": 16.7 }, "medium": { - "accuracy": 91.7, - "cost": 0.0002, + "accuracy": 83.3, + "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 16.7, + "accuracy": 11.1, "cost": 0.0002, - "robustness": 50.0 + "robustness": 100.0 }, "all": { - "accuracy": 87.7, - "cost": 0.0002, - "robustness": 75.0 + "accuracy": 85.0, + "cost": 0.0001, + "robustness": 37.5 } } } @@ -19110,24 +19110,24 @@ "Science": { "metrics": { "easy": { - "accuracy": 98.6, - "cost": 0.0002, - "robustness": 51.3 + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 59.0 }, "medium": { - "accuracy": 81.9, + "accuracy": 83.3, "cost": 0.0003, - "robustness": 42.9 + "robustness": 57.1 }, "hard": { - "accuracy": 28.9, + "accuracy": 32.5, "cost": 0.0005, - "robustness": 66.7 + "robustness": 77.8 }, "all": { - "accuracy": 83.8, - "cost": 0.0003, - "robustness": 50.7 + "accuracy": 84.6, + "cost": 0.0002, + "robustness": 60.9 } }, "subcategories": { @@ -19136,7 +19136,7 @@ "easy": { "accuracy": 100.0, "cost": 0.0001, - "robustness": 100.0 + "robustness": 50.0 }, "medium": { "accuracy": 60.0, @@ -19151,127 +19151,127 @@ "all": { "accuracy": 92.0, "cost": 0.0001, - "robustness": 100.0 + "robustness": 50.0 } } }, "Biology": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 99.1, "cost": 0.0002, - "robustness": 0.0 + "robustness": 28.6 }, "medium": { "accuracy": 76.5, - "cost": 0.0003, + "cost": 0.0002, "robustness": 0 }, "hard": { "accuracy": 33.3, - "cost": 0.0004, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 93.0, + "accuracy": 92.3, "cost": 0.0002, - "robustness": 0.0 + "robustness": 28.6 } } }, "Chemistry": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 66.7 }, "medium": { - "accuracy": 94.1, - "cost": 0.0004, + "accuracy": 97.1, + "cost": 0.0003, "robustness": 50.0 }, "hard": { - "accuracy": 23.1, - "cost": 0.0004, + "accuracy": 15.4, + "cost": 0.0003, "robustness": 0 }, "all": { - "accuracy": 86.7, - "cost": 0.0003, - "robustness": 40.0 + "accuracy": 85.6, + "cost": 0.0002, + "robustness": 60.0 } } }, "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 97.3, + "accuracy": 96.9, "cost": 0.0001, - "robustness": 62.5 + "robustness": 75.0 }, "medium": { - "accuracy": 66.1, - "cost": 0.0002, + "accuracy": 74.6, + "cost": 0.0001, "robustness": 33.3 }, "hard": { - "accuracy": 23.1, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 0.0 }, "all": { - "accuracy": 88.8, + "accuracy": 89.1, "cost": 0.0001, - "robustness": 46.7 + "robustness": 53.3 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 98.5, - "cost": 0.0002, + "accuracy": 99.2, + "cost": 0.0001, "robustness": 63.6 }, "medium": { - "accuracy": 84.0, + "accuracy": 85.2, "cost": 0.0003, - "robustness": 45.5 + "robustness": 63.6 }, "hard": { - "accuracy": 33.7, - "cost": 0.0009, - "robustness": 100.0 + "accuracy": 48.3, + "cost": 0.001, + "robustness": 80.0 }, "all": { - "accuracy": 82.9, + "accuracy": 85.9, "cost": 0.0003, - "robustness": 63.0 + "robustness": 66.7 } } }, "Physics": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 0.0 }, "medium": { - "accuracy": 90.5, - "cost": 0.0003, - "robustness": 50.0 + "accuracy": 85.7, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 45.5, + "accuracy": 36.4, "cost": 0.0003, "robustness": 0 }, "all": { - "accuracy": 89.7, - "cost": 0.0003, - "robustness": 80.0 + "accuracy": 85.6, + "cost": 0.0002, + "robustness": 40.0 } } }, @@ -19280,22 +19280,22 @@ "easy": { "accuracy": 100.0, "cost": 0.0001, - "robustness": 40.0 + "robustness": 100.0 }, "medium": { - "accuracy": 76.0, + "accuracy": 72.0, "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 20.3, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 18.6, + "cost": 0.0, + "robustness": 100.0 }, "all": { - "accuracy": 53.5, + "accuracy": 51.8, "cost": 0.0001, - "robustness": 37.5 + "robustness": 100.0 } } } @@ -19306,22 +19306,22 @@ "easy": { "accuracy": 96.3, "cost": 0.0002, - "robustness": 55.6 + "robustness": 50.0 }, "medium": { - "accuracy": 74.7, - "cost": 0.0005, - "robustness": 33.3 + "accuracy": 76.3, + "cost": 0.0004, + "robustness": 66.7 }, "hard": { - "accuracy": 16.1, - "cost": 0.0004, - "robustness": 42.9 + "accuracy": 30.9, + "cost": 0.0007, + "robustness": 71.4 }, "all": { - "accuracy": 71.4, - "cost": 0.0003, - "robustness": 45.9 + "accuracy": 75.1, + "cost": 0.0004, + "robustness": 59.5 } }, "subcategories": { @@ -19329,56 +19329,56 @@ "metrics": { "easy": { "accuracy": 97.8, - "cost": 0.0002, - "robustness": 33.3 + "cost": 0.0001, + "robustness": 58.3 }, "medium": { - "accuracy": 79.8, + "accuracy": 82.2, "cost": 0.0004, - "robustness": 33.3 + "robustness": 66.7 }, "hard": { - "accuracy": 16.2, - "cost": 0.0004, - "robustness": 0.0 + "accuracy": 40.0, + "cost": 0.0008, + "robustness": 66.7 }, "all": { - "accuracy": 75.6, - "cost": 0.0003, - "robustness": 28.6 + "accuracy": 81.0, + "cost": 0.0004, + "robustness": 61.9 } } }, "Law": { "metrics": { "easy": { - "accuracy": 90.8, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 92.3, + "cost": 0.0003, + "robustness": 0.0 }, "medium": { "accuracy": 68.3, - "cost": 0.0006, - "robustness": 25.0 + "cost": 0.0004, + "robustness": 50.0 }, "hard": { - "accuracy": 20.0, - "cost": 0.0006, - "robustness": 100.0 + "accuracy": 26.7, + "cost": 0.0004, + "robustness": 50.0 }, "all": { - "accuracy": 68.4, - "cost": 0.0005, - "robustness": 66.7 + "accuracy": 70.3, + "cost": 0.0003, + "robustness": 33.3 } } }, "Social problems": { "metrics": { "easy": { - "accuracy": 97.9, + "accuracy": 95.7, "cost": 0.0001, - "robustness": 100.0 + "robustness": 66.7 }, "medium": { "accuracy": 22.2, @@ -19391,9 +19391,9 @@ "robustness": 100.0 }, "all": { - "accuracy": 78.7, + "accuracy": 77.0, "cost": 0.0001, - "robustness": 100.0 + "robustness": 75.0 } } }, @@ -19401,23 +19401,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, + "cost": 0.0002, "robustness": 0 }, "medium": { "accuracy": 77.8, - "cost": 0.0002, - "robustness": 50.0 + "cost": 0.0006, + "robustness": 100.0 }, "hard": { - "accuracy": 14.7, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 17.6, + "cost": 0.001, + "robustness": 100.0 }, "all": { - "accuracy": 40.0, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 41.8, + "cost": 0.0008, + "robustness": 100.0 } } } @@ -19426,62 +19426,62 @@ "Technology": { "metrics": { "easy": { - "accuracy": 94.5, - "cost": 0.0002, - "robustness": 55.8 + "accuracy": 94.9, + "cost": 0.0001, + "robustness": 69.8 }, "medium": { - "accuracy": 72.9, + "accuracy": 73.6, "cost": 0.0003, - "robustness": 47.6 + "robustness": 76.2 }, "hard": { "accuracy": 27.4, "cost": 0.0004, - "robustness": 57.1 + "robustness": 28.6 }, "all": { - "accuracy": 80.5, + "accuracy": 80.9, "cost": 0.0002, - "robustness": 53.5 + "robustness": 67.6 } }, "subcategories": { "Engineering": { "metrics": { "easy": { - "accuracy": 97.6, + "accuracy": 96.8, "cost": 0.0002, "robustness": 80.0 }, "medium": { - "accuracy": 86.8, - "cost": 0.0005, - "robustness": 61.5 + "accuracy": 87.4, + "cost": 0.0006, + "robustness": 76.9 }, "hard": { - "accuracy": 59.5, - "cost": 0.0007, - "robustness": 100.0 + "accuracy": 57.1, + "cost": 0.001, + "robustness": 0.0 }, "all": { - "accuracy": 87.5, - "cost": 0.0004, - "robustness": 68.4 + "accuracy": 87.2, + "cost": 0.0005, + "robustness": 73.7 } } }, "Management and public relations": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 96.7, "cost": 0.0001, - "robustness": 25.0 + "robustness": 75.0 }, "medium": { "accuracy": 63.6, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "hard": { "accuracy": 0.0, @@ -19489,33 +19489,33 @@ "robustness": 0 }, "all": { - "accuracy": 91.8, + "accuracy": 89.0, "cost": 0.0001, - "robustness": 20.0 + "robustness": 80.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 93.4, + "accuracy": 94.3, "cost": 0.0001, - "robustness": 55.9 + "robustness": 67.6 }, "medium": { - "accuracy": 63.6, - "cost": 0.0002, - "robustness": 28.6 + "accuracy": 64.5, + "cost": 0.0001, + "robustness": 71.4 }, "hard": { - "accuracy": 15.9, - "cost": 0.0003, - "robustness": 50.0 + "accuracy": 16.8, + "cost": 0.0002, + "robustness": 33.3 }, "all": { - "accuracy": 77.3, - "cost": 0.0002, - "robustness": 51.1 + "accuracy": 78.2, + "cost": 0.0001, + "robustness": 63.8 } } } @@ -20454,5 +20454,937 @@ } } } + }, + "barouter": { + "metrics": { + "easy": { + "accuracy": 94.3, + "cost": 0.0004, + "robustness": 49.0 + }, + "medium": { + "accuracy": 62.9, + "cost": 0.0007, + "robustness": 60.2 + }, + "hard": { + "accuracy": 24.6, + "cost": 0.0009, + "robustness": 50.5 + }, + "all": { + "accuracy": 68.9, + "cost": 0.0006, + "robustness": 52.4 + } + }, + "categories": { + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 94.0, + "cost": 0.0005, + "robustness": 37.5 + }, + "medium": { + "accuracy": 51.0, + "cost": 0.001, + "robustness": 72.7 + }, + "hard": { + "accuracy": 6.8, + "cost": 0.0011, + "robustness": 40.0 + }, + "all": { + "accuracy": 60.0, + "cost": 0.0008, + "robustness": 54.2 + } + }, + "subcategories": { + "Arts": { + "metrics": { + "easy": { + "accuracy": 96.6, + "cost": 0.0003, + "robustness": 100.0 + }, + "medium": { + "accuracy": 64.1, + "cost": 0.0003, + "robustness": 50.0 + }, + "hard": { + "accuracy": 7.1, + "cost": 0.0003, + "robustness": 0.0 + }, + "all": { + "accuracy": 55.8, + "cost": 0.0003, + "robustness": 40.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 97.0, + "cost": 0.0003, + "robustness": 20.0 + }, + "medium": { + "accuracy": 35.3, + "cost": 0.0008, + "robustness": 60.0 + }, + "hard": { + "accuracy": 4.0, + "cost": 0.0005, + "robustness": 0.0 + }, + "all": { + "accuracy": 57.5, + "cost": 0.0006, + "robustness": 36.4 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 90.7, + "cost": 0.0008, + "robustness": 50.0 + }, + "medium": { + "accuracy": 64.0, + "cost": 0.0016, + "robustness": 100.0 + }, + "hard": { + "accuracy": 7.6, + "cost": 0.0021, + "robustness": 100.0 + }, + "all": { + "accuracy": 64.1, + "cost": 0.0013, + "robustness": 87.5 + } + } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 96.1, + "cost": 0.0005, + "robustness": 38.5 + }, + "medium": { + "accuracy": 74.5, + "cost": 0.0008, + "robustness": 47.6 + }, + "hard": { + "accuracy": 29.4, + "cost": 0.0019, + "robustness": 66.7 + }, + "all": { + "accuracy": 77.3, + "cost": 0.0008, + "robustness": 48.4 + } + }, + "subcategories": { + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0005, + "robustness": 30.8 + }, + "medium": { + "accuracy": 78.2, + "cost": 0.0009, + "robustness": 47.1 + }, + "hard": { + "accuracy": 33.0, + "cost": 0.0022, + "robustness": 61.5 + }, + "all": { + "accuracy": 76.5, + "cost": 0.001, + "robustness": 46.5 + } + } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 95.3, + "cost": 0.0005, + "robustness": 46.2 + }, + "medium": { + "accuracy": 57.6, + "cost": 0.0005, + "robustness": 50.0 + }, + "hard": { + "accuracy": 8.3, + "cost": 0.0005, + "robustness": 100.0 + }, + "all": { + "accuracy": 79.3, + "cost": 0.0005, + "robustness": 52.6 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0005, + "robustness": 69.6 + }, + "medium": { + "accuracy": 64.4, + "cost": 0.0005, + "robustness": 83.3 + }, + "hard": { + "accuracy": 8.7, + "cost": 0.0006, + "robustness": 50.0 + }, + "all": { + "accuracy": 70.3, + "cost": 0.0005, + "robustness": 66.7 + } + }, + "subcategories": { + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0005, + "robustness": 75.0 + }, + "medium": { + "accuracy": 66.7, + "cost": 0.0005, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0005, + "robustness": 0 + }, + "all": { + "accuracy": 92.3, + "cost": 0.0005, + "robustness": 75.0 + } + } + }, + "Geography": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0003, + "robustness": 50.0 + }, + "medium": { + "accuracy": 51.7, + "cost": 0.0004, + "robustness": 100.0 + }, + "hard": { + "accuracy": 8.3, + "cost": 0.0004, + "robustness": 0 + }, + "all": { + "accuracy": 82.0, + "cost": 0.0003, + "robustness": 55.6 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0005, + "robustness": 81.8 + }, + "medium": { + "accuracy": 66.9, + "cost": 0.0006, + "robustness": 80.0 + }, + "hard": { + "accuracy": 8.8, + "cost": 0.0006, + "robustness": 50.0 + }, + "all": { + "accuracy": 64.5, + "cost": 0.0006, + "robustness": 69.2 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 70.3, + "cost": 0.0003, + "robustness": 63.6 + }, + "medium": { + "accuracy": 28.1, + "cost": 0.0005, + "robustness": 77.8 + }, + "hard": { + "accuracy": 50.9, + "cost": 0.0004, + "robustness": 52.0 + }, + "all": { + "accuracy": 51.3, + "cost": 0.0004, + "robustness": 60.0 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 70.3, + "cost": 0.0003, + "robustness": 63.6 + }, + "medium": { + "accuracy": 28.1, + "cost": 0.0005, + "robustness": 77.8 + }, + "hard": { + "accuracy": 50.9, + "cost": 0.0004, + "robustness": 52.0 + }, + "all": { + "accuracy": 51.3, + "cost": 0.0004, + "robustness": 60.0 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0004, + "robustness": 66.7 + }, + "medium": { + "accuracy": 67.3, + "cost": 0.0004, + "robustness": 50.0 + }, + "hard": { + "accuracy": 30.3, + "cost": 0.0005, + "robustness": 50.0 + }, + "all": { + "accuracy": 44.9, + "cost": 0.0005, + "robustness": 51.7 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0004, + "robustness": 66.7 + }, + "medium": { + "accuracy": 67.3, + "cost": 0.0004, + "robustness": 50.0 + }, + "hard": { + "accuracy": 30.3, + "cost": 0.0005, + "robustness": 50.0 + }, + "all": { + "accuracy": 44.9, + "cost": 0.0005, + "robustness": 51.7 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 92.9, + "cost": 0.0004, + "robustness": 51.9 + }, + "medium": { + "accuracy": 61.1, + "cost": 0.0005, + "robustness": 30.0 + }, + "hard": { + "accuracy": 8.6, + "cost": 0.0006, + "robustness": 42.9 + }, + "all": { + "accuracy": 73.6, + "cost": 0.0004, + "robustness": 45.5 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 87.6, + "cost": 0.0002, + "robustness": 57.1 + }, + "medium": { + "accuracy": 38.8, + "cost": 0.0003, + "robustness": 40.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 0.0 + }, + "all": { + "accuracy": 68.9, + "cost": 0.0002, + "robustness": 47.6 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 98.5, + "cost": 0.0004, + "robustness": 25.0 + }, + "medium": { + "accuracy": 76.3, + "cost": 0.0008, + "robustness": 0.0 + }, + "hard": { + "accuracy": 40.0, + "cost": 0.0011, + "robustness": 0 + }, + "all": { + "accuracy": 88.0, + "cost": 0.0006, + "robustness": 14.3 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 94.6, + "cost": 0.0004, + "robustness": 66.7 + }, + "medium": { + "accuracy": 65.0, + "cost": 0.0007, + "robustness": 50.0 + }, + "hard": { + "accuracy": 8.9, + "cost": 0.0006, + "robustness": 33.3 + }, + "all": { + "accuracy": 53.3, + "cost": 0.0006, + "robustness": 50.0 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0006, + "robustness": 50.0 + }, + "medium": { + "accuracy": 85.4, + "cost": 0.0005, + "robustness": 0 + }, + "hard": { + "accuracy": 5.6, + "cost": 0.0006, + "robustness": 100.0 + }, + "all": { + "accuracy": 85.6, + "cost": 0.0006, + "robustness": 62.5 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0004, + "robustness": 59.0 + }, + "medium": { + "accuracy": 74.1, + "cost": 0.001, + "robustness": 66.7 + }, + "hard": { + "accuracy": 14.9, + "cost": 0.002, + "robustness": 44.4 + }, + "all": { + "accuracy": 78.7, + "cost": 0.0008, + "robustness": 59.4 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0004, + "robustness": 50.0 + }, + "medium": { + "accuracy": 80.0, + "cost": 0.0004, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 96.0, + "cost": 0.0004, + "robustness": 50.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 97.4, + "cost": 0.0005, + "robustness": 100.0 + }, + "medium": { + "accuracy": 70.6, + "cost": 0.0006, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0007, + "robustness": 0 + }, + "all": { + "accuracy": 88.1, + "cost": 0.0005, + "robustness": 100.0 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0004, + "robustness": 33.3 + }, + "medium": { + "accuracy": 85.3, + "cost": 0.0006, + "robustness": 50.0 + }, + "hard": { + "accuracy": 23.1, + "cost": 0.0005, + "robustness": 0 + }, + "all": { + "accuracy": 82.2, + "cost": 0.0005, + "robustness": 40.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0003, + "robustness": 25.0 + }, + "medium": { + "accuracy": 64.4, + "cost": 0.0004, + "robustness": 66.7 + }, + "hard": { + "accuracy": 7.7, + "cost": 0.0003, + "robustness": 0.0 + }, + "all": { + "accuracy": 87.9, + "cost": 0.0003, + "robustness": 40.0 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0004, + "robustness": 72.7 + }, + "medium": { + "accuracy": 78.0, + "cost": 0.0013, + "robustness": 63.6 + }, + "hard": { + "accuracy": 21.3, + "cost": 0.004, + "robustness": 60.0 + }, + "all": { + "accuracy": 77.9, + "cost": 0.0013, + "robustness": 66.7 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0003, + "robustness": 66.7 + }, + "medium": { + "accuracy": 73.8, + "cost": 0.0006, + "robustness": 100.0 + }, + "hard": { + "accuracy": 36.4, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 80.4, + "cost": 0.0004, + "robustness": 80.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0002, + "robustness": 40.0 + }, + "medium": { + "accuracy": 44.0, + "cost": 0.0003, + "robustness": 0 + }, + "hard": { + "accuracy": 3.4, + "cost": 0.0003, + "robustness": 33.3 + }, + "all": { + "accuracy": 36.8, + "cost": 0.0003, + "robustness": 37.5 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0005, + "robustness": 77.8 + }, + "medium": { + "accuracy": 59.7, + "cost": 0.0008, + "robustness": 75.0 + }, + "hard": { + "accuracy": 9.4, + "cost": 0.0009, + "robustness": 57.1 + }, + "all": { + "accuracy": 64.0, + "cost": 0.0007, + "robustness": 73.0 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0004, + "robustness": 83.3 + }, + "medium": { + "accuracy": 63.8, + "cost": 0.0008, + "robustness": 100.0 + }, + "hard": { + "accuracy": 8.8, + "cost": 0.0012, + "robustness": 66.7 + }, + "all": { + "accuracy": 67.6, + "cost": 0.0007, + "robustness": 85.7 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 92.3, + "cost": 0.0007, + "robustness": 100.0 + }, + "medium": { + "accuracy": 55.6, + "cost": 0.0009, + "robustness": 75.0 + }, + "hard": { + "accuracy": 16.7, + "cost": 0.0007, + "robustness": 50.0 + }, + "all": { + "accuracy": 63.3, + "cost": 0.0008, + "robustness": 77.8 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 93.6, + "cost": 0.0005, + "robustness": 33.3 + }, + "medium": { + "accuracy": 22.2, + "cost": 0.0005, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0005, + "robustness": 0.0 + }, + "all": { + "accuracy": 75.4, + "cost": 0.0005, + "robustness": 25.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 66.7, + "cost": 0.0003, + "robustness": 0 + }, + "medium": { + "accuracy": 55.6, + "cost": 0.0004, + "robustness": 0.0 + }, + "hard": { + "accuracy": 5.9, + "cost": 0.0005, + "robustness": 100.0 + }, + "all": { + "accuracy": 25.5, + "cost": 0.0005, + "robustness": 33.3 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 94.7, + "cost": 0.0004, + "robustness": 18.6 + }, + "medium": { + "accuracy": 61.0, + "cost": 0.0006, + "robustness": 52.4 + }, + "hard": { + "accuracy": 10.8, + "cost": 0.0005, + "robustness": 28.6 + }, + "all": { + "accuracy": 75.1, + "cost": 0.0005, + "robustness": 29.6 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 96.0, + "cost": 0.0006, + "robustness": 20.0 + }, + "medium": { + "accuracy": 77.8, + "cost": 0.0008, + "robustness": 69.2 + }, + "hard": { + "accuracy": 26.2, + "cost": 0.0009, + "robustness": 0.0 + }, + "all": { + "accuracy": 78.2, + "cost": 0.0007, + "robustness": 52.6 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0002, + "robustness": 75.0 + }, + "medium": { + "accuracy": 54.5, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 89.0, + "cost": 0.0002, + "robustness": 80.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 94.0, + "cost": 0.0004, + "robustness": 11.8 + }, + "medium": { + "accuracy": 49.6, + "cost": 0.0004, + "robustness": 14.3 + }, + "hard": { + "accuracy": 5.3, + "cost": 0.0004, + "robustness": 33.3 + }, + "all": { + "accuracy": 73.1, + "cost": 0.0004, + "robustness": 14.9 + } + } + } + } + } + } } } diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index a62dd77..1b646eb 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -1,4 +1,15 @@ [ + { + "Router Name": "Cross-Router", + "Arena Score": 76.12, + "Optimal Selection Score": 17.66, + "Optimal Cost Score": 45.49, + "Optimal Acc. Score": 90.31, + "Robustness Score": 67.14, + "Latency Score": null, + "Accuracy": 78.14, + "Cost per 1k": 0.3 + }, { "Router Name": "vllm", "Arena Score": 75.3, @@ -21,6 +32,17 @@ "Accuracy": 76.4, "Cost per 1k": 0.18 }, + { + "Router Name": "Nadir-Tumbler", + "Arena Score": 75.17, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 66.43, + "Latency Score": null, + "Accuracy": 75.34, + "Cost per 1k": 0.08 + }, { "Router Name": "AgentForge Router", "Arena Score": 74.13, @@ -32,17 +54,6 @@ "Accuracy": 74.72, "Cost per 1k": 0.13 }, - { - "Router Name": "Cross-Router", - "Arena Score": 73.71, - "Optimal Selection Score": 40.18, - "Optimal Cost Score": 44.03, - "Optimal Acc. Score": 89.58, - "Robustness Score": 59.05, - "Latency Score": null, - "Accuracy": 75.13, - "Cost per 1k": 0.26 - }, { "Router Name": "Weave Router", "Arena Score": 72.82, @@ -109,6 +120,17 @@ "Accuracy": 72.05, "Cost per 1k": 0.2 }, + { + "Router Name": "chuzom-solo-v32", + "Arena Score": 70.61, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 100.0, + "Latency Score": null, + "Accuracy": 70.59, + "Cost per 1k": 0.1 + }, { "Router Name": "azure", "Arena Score": 70.42, @@ -131,6 +153,17 @@ "Accuracy": 70.17, "Cost per 1k": 0.12 }, + { + "Router Name": "BARouter", + "Arena Score": 67.09, + "Optimal Selection Score": 58.56, + "Optimal Cost Score": 59.03, + "Optimal Acc. Score": 87.91, + "Robustness Score": 52.38, + "Latency Score": null, + "Accuracy": 68.8, + "Cost per 1k": 0.63 + }, { "Router Name": "mirt_bert", "Arena Score": 66.89, diff --git a/src/data/routers.json b/src/data/routers.json index 00e382a..0ce83e6 100644 --- a/src/data/routers.json +++ b/src/data/routers.json @@ -440,5 +440,39 @@ "google/gemini-3.1-flash-lite", "deepseek/deepseek-v4-flash" ] + }, + "Nadir-Tumbler": { + "name": "Nadir-Tumbler", + "affiliation": "@doramirdor", + "githubUrl": "https://github.com/doramirdor", + "type": "open-source", + "description": "Submitted by @doramirdor.", + "modelPool": [ + "qwen/qwen3-235b-a22b-2507", + "deepseek/deepseek-v3.2", + "grok-4-1-fast-reasoning" + ] + }, + "chuzom-solo-v32": { + "name": "chuzom-solo-v32", + "affiliation": "@ypollak2", + "githubUrl": "https://github.com/ypollak2", + "type": "open-source", + "description": "Submitted by @ypollak2.", + "modelPool": [ + "deepseek/deepseek-v3.2" + ] + }, + "BARouter": { + "name": "BARouter", + "affiliation": "@zulk2002", + "githubUrl": "https://github.com/zulk2002", + "type": "open-source", + "description": "Submitted by @zulk2002.", + "modelPool": [ + "qwen3-235b-a22b-instruct-2507", + "gpt-4.1", + "deepseek-v3.1" + ] } }