diff --git a/src/data/flip_labels/flip_labels_asiainfo.json b/src/data/flip_labels/flip_labels_asiainfo.json new file mode 100644 index 0000000..f6a1eb7 --- /dev/null +++ b/src/data/flip_labels/flip_labels_asiainfo.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 1 + }, + { + "global index": "ArcMMLU_123", + "flip": 1 + }, + { + "global index": "ArcMMLU_16", + "flip": 1 + }, + { + "global index": "ArcMMLU_182", + "flip": 1 + }, + { + "global index": "ArcMMLU_230", + "flip": 1 + }, + { + "global index": "ArcMMLU_293", + "flip": 1 + }, + { + "global index": "ArcMMLU_349", + "flip": 1 + }, + { + "global index": "ArcMMLU_378", + "flip": 1 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 1 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 1 + }, + { + "global index": "ArcMMLU_659", + "flip": 1 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 1 + }, + { + "global index": "ArcMMLU_689", + "flip": 1 + }, + { + "global index": "ArcMMLU_702", + "flip": 1 + }, + { + "global index": "ArcMMLU_713", + "flip": 1 + }, + { + "global index": "ArcMMLU_98", + "flip": 1 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 1 + }, + { + "global index": "AsDiv_472", + "flip": 1 + }, + { + "global index": "AsDiv_733", + "flip": 1 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 1 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 1 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 1 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 1 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 1 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 1 + }, + { + "global index": "GeoBench_502", + "flip": 1 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 1 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 1 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 1 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 1 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 1 + }, + { + "global index": "MMLUPro_business_507", + "flip": 1 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 1 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 1 + }, + { + "global index": "MMLU_management_77", + "flip": 1 + }, + { + "global index": "MMLU_management_91", + "flip": 1 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 1 + }, + { + "global index": "MathQA_2102", + "flip": 1 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 1 + }, + { + "global index": "MedMCQA_1005", + "flip": 1 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 1 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 1 + }, + { + "global index": "MedMCQA_145", + "flip": 1 + }, + { + "global index": "MedMCQA_2010", + "flip": 1 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 1 + }, + { + "global index": "MedMCQA_2581", + "flip": 1 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 1 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 1 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 1 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 1 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 1 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 1 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 1 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 1 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 1 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 1 + }, + { + "global index": "QANTA_Literature_408", + "flip": 0 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 1 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 1 + }, + { + "global index": "SocialiQA_26846", + "flip": 1 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] diff --git a/src/data/flip_labels/flip_labels_barouter.json b/src/data/flip_labels/flip_labels_barouter.json index d7844c0..6fa4bc4 100644 --- a/src/data/flip_labels/flip_labels_barouter.json +++ b/src/data/flip_labels/flip_labels_barouter.json @@ -5,15 +5,15 @@ }, { "global index": "AIME_58", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_12", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_123", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_16", @@ -21,7 +21,7 @@ }, { "global index": "ArcMMLU_182", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_230", @@ -29,7 +29,7 @@ }, { "global index": "ArcMMLU_293", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_349", @@ -37,7 +37,7 @@ }, { "global index": "ArcMMLU_378", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_443", @@ -49,11 +49,11 @@ }, { "global index": "ArcMMLU_631", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_646", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_659", @@ -61,7 +61,7 @@ }, { "global index": "ArcMMLU_676", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_685", @@ -69,11 +69,11 @@ }, { "global index": "ArcMMLU_689", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_702", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_713", @@ -81,7 +81,7 @@ }, { "global index": "ArcMMLU_98", - "flip": 1 + "flip": 0 }, { "global index": "AsDiv_1165", @@ -97,7 +97,7 @@ }, { "global index": "AsDiv_733", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_0", @@ -109,7 +109,7 @@ }, { "global index": "ChessInstruct_144", - "flip": 0 + "flip": 1 }, { "global index": "ChessInstruct_42", @@ -129,11 +129,11 @@ }, { "global index": "Ethics_commonsense_28", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_51", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_commonsense_6", @@ -141,7 +141,7 @@ }, { "global index": "Ethics_commonsense_62", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_70", @@ -149,7 +149,7 @@ }, { "global index": "Ethics_commonsense_85", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_90", @@ -157,7 +157,7 @@ }, { "global index": "Ethics_deontology_0", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_deontology_2", @@ -165,11 +165,11 @@ }, { "global index": "Ethics_deontology_31", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_deontology_32", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_deontology_56", @@ -177,7 +177,7 @@ }, { "global index": "Ethics_justice_1", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_justice_45", @@ -189,15 +189,15 @@ }, { "global index": "Ethics_justice_84", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_virtue_14", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_virtue_26", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_virtue_30", @@ -213,7 +213,7 @@ }, { "global index": "FinQA_149", - "flip": 0 + "flip": 1 }, { "global index": "FinQA_208", @@ -221,7 +221,7 @@ }, { "global index": "FinQA_56", - "flip": 0 + "flip": 1 }, { "global index": "FinQA_60", @@ -245,15 +245,15 @@ }, { "global index": "GeoBench_1113", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_124", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_1243", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_30", @@ -261,7 +261,7 @@ }, { "global index": "GeoBench_502", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_526", @@ -273,7 +273,7 @@ }, { "global index": "GeoBench_766", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_87", @@ -285,11 +285,11 @@ }, { "global index": "GeoBench_944", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_968", - "flip": 1 + "flip": 0 }, { "global index": "GeoGraphyData_100k_27", @@ -301,11 +301,11 @@ }, { "global index": "LiveCodeBench_105", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_114", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_118", @@ -313,15 +313,15 @@ }, { "global index": "LiveCodeBench_131", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_136", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_181", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_237", @@ -333,7 +333,7 @@ }, { "global index": "LiveCodeBench_350", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_386", @@ -341,11 +341,11 @@ }, { "global index": "LiveCodeBench_405", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_43", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_431", @@ -353,7 +353,7 @@ }, { "global index": "LiveCodeBench_437", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_476", @@ -361,11 +361,11 @@ }, { "global index": "LiveCodeBench_485", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_49", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_491", @@ -373,7 +373,7 @@ }, { "global index": "LiveCodeBench_499", - "flip": 0 + "flip": 1 }, { "global index": "MATH_108", @@ -381,7 +381,7 @@ }, { "global index": "MATH_442", - "flip": 0 + "flip": 1 }, { "global index": "MATH_53", @@ -389,7 +389,7 @@ }, { "global index": "MMLUPro_biology_2808", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_biology_2912", @@ -405,7 +405,7 @@ }, { "global index": "MMLUPro_biology_3188", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_biology_3215", @@ -433,11 +433,11 @@ }, { "global index": "MMLUPro_business_503", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_507", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_6", @@ -445,11 +445,11 @@ }, { "global index": "MMLUPro_business_784", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_3796", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_3837", @@ -465,11 +465,11 @@ }, { "global index": "MMLUPro_chemistry_4407", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9086", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9110", @@ -477,11 +477,11 @@ }, { "global index": "MMLUPro_computer science_9136", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9138", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9149", @@ -489,7 +489,7 @@ }, { "global index": "MMLUPro_computer science_9200", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9212", @@ -497,19 +497,19 @@ }, { "global index": "MMLUPro_computer science_9239", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9264", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9285", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9289", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9414", @@ -525,7 +525,7 @@ }, { "global index": "MMLUPro_computer science_9452", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9471", @@ -533,11 +533,11 @@ }, { "global index": "MMLUPro_computer science_9475", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5769", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5907", @@ -549,7 +549,7 @@ }, { "global index": "MMLUPro_economics_5965", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_6114", @@ -561,19 +561,19 @@ }, { "global index": "MMLUPro_economics_6135", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_6325", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_6353", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10076", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10125", @@ -581,7 +581,7 @@ }, { "global index": "MMLUPro_engineering_10179", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10195", @@ -597,7 +597,7 @@ }, { "global index": "MMLUPro_engineering_10342", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10395", @@ -609,7 +609,7 @@ }, { "global index": "MMLUPro_engineering_10432", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10473", @@ -621,7 +621,7 @@ }, { "global index": "MMLUPro_engineering_10701", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10823", @@ -633,15 +633,15 @@ }, { "global index": "MMLUPro_health_4885", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_4973", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5093", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5144", @@ -649,7 +649,7 @@ }, { "global index": "MMLUPro_health_5214", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5215", @@ -657,7 +657,7 @@ }, { "global index": "MMLUPro_health_5261", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5473", @@ -665,11 +665,11 @@ }, { "global index": "MMLUPro_health_5514", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4486", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4490", @@ -677,11 +677,11 @@ }, { "global index": "MMLUPro_history_4497", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4509", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4517", @@ -693,7 +693,7 @@ }, { "global index": "MMLUPro_history_4605", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4629", @@ -705,11 +705,11 @@ }, { "global index": "MMLUPro_history_4717", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4749", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4752", @@ -721,11 +721,11 @@ }, { "global index": "MMLUPro_history_4810", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4833", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4836", @@ -733,7 +733,7 @@ }, { "global index": "MMLUPro_history_4841", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1007", @@ -761,11 +761,11 @@ }, { "global index": "MMLUPro_law_1818", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_806", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_899", @@ -773,7 +773,7 @@ }, { "global index": "MMLUPro_math_6429", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6526", @@ -785,11 +785,11 @@ }, { "global index": "MMLUPro_math_6848", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7101", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7249", @@ -801,15 +801,15 @@ }, { "global index": "MMLUPro_math_7451", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7577", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_philosophy_9510", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_philosophy_9536", @@ -821,11 +821,11 @@ }, { "global index": "MMLUPro_philosophy_9672", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_philosophy_9943", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_7773", @@ -845,7 +845,7 @@ }, { "global index": "MMLUPro_physics_9017", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2005", @@ -861,7 +861,7 @@ }, { "global index": "MMLUPro_psychology_2406", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2420", @@ -869,7 +869,7 @@ }, { "global index": "MMLUPro_psychology_2450", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2457", @@ -881,7 +881,7 @@ }, { "global index": "MMLU_formal_logic_121", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_16", @@ -889,11 +889,11 @@ }, { "global index": "MMLU_formal_logic_32", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_63", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_7", @@ -905,7 +905,7 @@ }, { "global index": "MMLU_formal_logic_85", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_management_4", @@ -925,15 +925,15 @@ }, { "global index": "MMLU_management_93", - "flip": 0 + "flip": 1 }, { "global index": "MathQA_158", - "flip": 0 + "flip": 1 }, { "global index": "MathQA_1742", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_202", @@ -961,11 +961,11 @@ }, { "global index": "MedMCQA_1005", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_1054", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_1298", @@ -977,23 +977,23 @@ }, { "global index": "MedMCQA_1362", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_145", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2010", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2323", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2366", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_2581", @@ -1001,31 +1001,31 @@ }, { "global index": "MedMCQA_511", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_59", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_643", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_853", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_126", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_14", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_147", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_152", @@ -1033,15 +1033,15 @@ }, { "global index": "MusicTheoryBench_188", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_189", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_240", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_33", @@ -1069,7 +1069,7 @@ }, { "global index": "NarrativeQA_2474", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_2820", @@ -1085,11 +1085,11 @@ }, { "global index": "NarrativeQA_4128", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_4347", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_4540", @@ -1097,11 +1097,11 @@ }, { "global index": "NarrativeQA_5022", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_5259", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_533", @@ -1109,15 +1109,15 @@ }, { "global index": "NarrativeQA_5894", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_6829", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_7678", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_7964", @@ -1125,11 +1125,11 @@ }, { "global index": "NarrativeQA_8215", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_8598", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_927", @@ -1137,11 +1137,11 @@ }, { "global index": "OpenTDB_Animals_2545", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Animals_262", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Art_1429", @@ -1153,23 +1153,23 @@ }, { "global index": "OpenTDB_Celebrities_1456", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Celebrities_3577", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Celebrities_3968", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_1178", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_1807", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_1957", @@ -1177,11 +1177,11 @@ }, { "global index": "OpenTDB_General Knowledge_2181", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_3404", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_3727", @@ -1193,11 +1193,11 @@ }, { "global index": "OpenTDB_Geography_2099", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_2346", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Geography_3880", @@ -1205,7 +1205,7 @@ }, { "global index": "OpenTDB_Geography_792", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_1162", @@ -1217,11 +1217,11 @@ }, { "global index": "OpenTDB_History_3712", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_3902", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Science & Nature_1175", @@ -1233,7 +1233,7 @@ }, { "global index": "OpenTDB_Science & Nature_3716", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Science & Nature_476", @@ -1245,7 +1245,7 @@ }, { "global index": "OpenTDB_Vehicles_1173", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Vehicles_1419", @@ -1257,11 +1257,11 @@ }, { "global index": "OpenTDB_Vehicles_3234", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_0", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_154", @@ -1269,23 +1269,23 @@ }, { "global index": "PubMedQA_18", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_238", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_250", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_337", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_362", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_437", @@ -1297,19 +1297,19 @@ }, { "global index": "PubMedQA_520", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_575", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_582", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_588", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_610", @@ -1317,11 +1317,11 @@ }, { "global index": "PubMedQA_63", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_643", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_687", @@ -1333,23 +1333,23 @@ }, { "global index": "PubMedQA_73", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_755", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_8", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_81", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_854", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_905", @@ -1361,19 +1361,19 @@ }, { "global index": "QANTA_Fine Arts_1702", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Fine Arts_828", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Fine Arts_865", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Geography_1023", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Geography_1555", @@ -1381,11 +1381,11 @@ }, { "global index": "QANTA_Geography_304", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_1084", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_1154", @@ -1397,7 +1397,7 @@ }, { "global index": "QANTA_History_473", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_926", @@ -1417,7 +1417,7 @@ }, { "global index": "QANTA_Literature_1326", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1727", @@ -1425,11 +1425,11 @@ }, { "global index": "QANTA_Literature_1843", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_386", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_408", @@ -1437,15 +1437,15 @@ }, { "global index": "QANTA_Literature_475", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_833", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Philosophy_1270", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Philosophy_499", @@ -1457,7 +1457,7 @@ }, { "global index": "QANTA_Science_1360", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_1473", @@ -1469,7 +1469,7 @@ }, { "global index": "QANTA_Science_619", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_1847", @@ -1477,7 +1477,7 @@ }, { "global index": "QANTA_Social Science_2", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_77", @@ -1485,23 +1485,23 @@ }, { "global index": "SocialiQA_13810", - "flip": 1 + "flip": 0 }, { "global index": "SocialiQA_22095", - "flip": 0 + "flip": 1 }, { "global index": "SocialiQA_26846", - "flip": 1 + "flip": 0 }, { "global index": "SocialiQA_7839", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-CausalReasoning_4526", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-ClozeTest_12894", @@ -1517,11 +1517,11 @@ }, { "global index": "SuperGLUE-Entailment_19410", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Entailment_19567", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Entailment_522", @@ -1541,11 +1541,11 @@ }, { "global index": "SuperGLUE-QA_4046", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_4102", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-QA_4160", @@ -1561,11 +1561,11 @@ }, { "global index": "SuperGLUE-RC_8531", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_19695", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_19738", @@ -1577,7 +1577,7 @@ }, { "global index": "SuperGLUE-Wic_20189", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_20253", @@ -1585,15 +1585,15 @@ }, { "global index": "SuperGLUE-Wsc_20368", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wsc_20370", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-cs-en_156", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-cs-en_246", @@ -1605,15 +1605,15 @@ }, { "global index": "WMT19-de-en_46", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-de-en_715", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-de-en_883", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-fi-en_222", @@ -1629,15 +1629,15 @@ }, { "global index": "WMT19-gu-en_123", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-gu-en_191", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-gu-en_491", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-gu-en_968", @@ -1645,7 +1645,7 @@ }, { "global index": "WMT19-kk-en_528", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-kk-en_826", @@ -1653,11 +1653,11 @@ }, { "global index": "WMT19-lt-en_135", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-lt-en_269", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-lt-en_636", @@ -1677,6 +1677,6 @@ }, { "global index": "WMT19-zh-en_59", - "flip": 1 + "flip": 0 } ] diff --git a/src/data/flip_labels/flip_labels_divyam.json b/src/data/flip_labels/flip_labels_divyam.json new file mode 100644 index 0000000..1d7dc21 --- /dev/null +++ b/src/data/flip_labels/flip_labels_divyam.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 0 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 1 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 1 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 1 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 0 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 0 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 0 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 0 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 1 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 0 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] diff --git a/src/data/flip_labels/flip_labels_nadir-caliper.json b/src/data/flip_labels/flip_labels_nadir-caliper.json new file mode 100644 index 0000000..9b41741 --- /dev/null +++ b/src/data/flip_labels/flip_labels_nadir-caliper.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 1 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 1 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 1 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 1 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 1 + }, + { + "global index": "ChessInstruct_107", + "flip": 1 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 1 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 1 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 1 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 1 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 1 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 1 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 1 + }, + { + "global index": "GeoBench_1243", + "flip": 1 + }, + { + "global index": "GeoBench_30", + "flip": 1 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 1 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 1 + }, + { + "global index": "GeoBench_87", + "flip": 1 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 1 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 1 + }, + { + "global index": "LiveCodeBench_114", + "flip": 1 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 1 + }, + { + "global index": "LiveCodeBench_136", + "flip": 1 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 1 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 1 + }, + { + "global index": "LiveCodeBench_437", + "flip": 1 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 1 + }, + { + "global index": "LiveCodeBench_49", + "flip": 1 + }, + { + "global index": "LiveCodeBench_491", + "flip": 1 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 1 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 1 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 1 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 1 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 1 + }, + { + "global index": "NarrativeQA_4540", + "flip": 1 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 0 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 1 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 1 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 1 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 1 + }, + { + "global index": "QANTA_History_433", + "flip": 1 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 1 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 0 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 1 + }, + { + "global index": "QANTA_Science_1473", + "flip": 1 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] diff --git a/src/data/flip_labels/flip_labels_paix2.json b/src/data/flip_labels/flip_labels_paix2.json new file mode 100644 index 0000000..f0ead64 --- /dev/null +++ b/src/data/flip_labels/flip_labels_paix2.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 1 + }, + { + "global index": "ArcMMLU_12", + "flip": 1 + }, + { + "global index": "ArcMMLU_123", + "flip": 1 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 1 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 1 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 1 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 1 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 1 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 1 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 1 + }, + { + "global index": "ArcMMLU_98", + "flip": 1 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 1 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 1 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 1 + }, + { + "global index": "Ethics_justice_76", + "flip": 0 + }, + { + "global index": "Ethics_justice_84", + "flip": 1 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 1 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 1 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 1 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 0 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 1 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 1 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 1 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 1 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 1 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 1 + }, + { + "global index": "MedMCQA_1005", + "flip": 1 + }, + { + "global index": "MedMCQA_1054", + "flip": 1 + }, + { + "global index": "MedMCQA_1298", + "flip": 1 + }, + { + "global index": "MedMCQA_1309", + "flip": 1 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 1 + }, + { + "global index": "MedMCQA_2010", + "flip": 1 + }, + { + "global index": "MedMCQA_2323", + "flip": 1 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 1 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 1 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 1 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 1 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 1 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 1 + }, + { + "global index": "NarrativeQA_8598", + "flip": 1 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 0 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 1 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 1 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 1 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 1 + }, + { + "global index": "PubMedQA_582", + "flip": 1 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 1 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 1 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 1 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 1 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 0 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 1 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 1 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] diff --git a/src/data/flip_labels/flip_labels_sqwish-router.json b/src/data/flip_labels/flip_labels_sqwish-router.json index 9450917..f0250d3 100644 --- a/src/data/flip_labels/flip_labels_sqwish-router.json +++ b/src/data/flip_labels/flip_labels_sqwish-router.json @@ -1,7 +1,7 @@ [ { "global index": "AIME_112", - "flip": 0 + "flip": 1 }, { "global index": "AIME_58", @@ -17,7 +17,7 @@ }, { "global index": "ArcMMLU_16", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_182", @@ -33,15 +33,15 @@ }, { "global index": "ArcMMLU_349", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_378", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_443", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_496", @@ -69,23 +69,23 @@ }, { "global index": "ArcMMLU_689", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_702", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_713", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_98", - "flip": 0 + "flip": 1 }, { "global index": "AsDiv_1165", - "flip": 0 + "flip": 1 }, { "global index": "AsDiv_1347", @@ -97,15 +97,15 @@ }, { "global index": "AsDiv_733", - "flip": 0 + "flip": 1 }, { "global index": "ChessInstruct_0", - "flip": 0 + "flip": 1 }, { "global index": "ChessInstruct_107", - "flip": 0 + "flip": 1 }, { "global index": "ChessInstruct_144", @@ -113,43 +113,43 @@ }, { "global index": "ChessInstruct_42", - "flip": 0 + "flip": 1 }, { "global index": "ChessInstruct_58", - "flip": 0 + "flip": 1 }, { "global index": "ChessInstruct_71", - "flip": 0 + "flip": 1 }, { "global index": "ChessInstruct_84", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_28", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_51", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_6", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_62", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_70", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_85", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_90", @@ -157,11 +157,11 @@ }, { "global index": "Ethics_deontology_0", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_2", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_31", @@ -169,19 +169,19 @@ }, { "global index": "Ethics_deontology_32", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_56", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_justice_1", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_justice_45", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_justice_76", @@ -189,15 +189,15 @@ }, { "global index": "Ethics_justice_84", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_virtue_14", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_virtue_26", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_virtue_30", @@ -205,7 +205,7 @@ }, { "global index": "Ethics_virtue_48", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_virtue_51", @@ -233,7 +233,7 @@ }, { "global index": "GeoBench_1002", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_1094", @@ -245,11 +245,11 @@ }, { "global index": "GeoBench_1113", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_124", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_1243", @@ -257,19 +257,19 @@ }, { "global index": "GeoBench_30", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_502", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_526", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_591", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_766", @@ -277,7 +277,7 @@ }, { "global index": "GeoBench_87", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_915", @@ -293,15 +293,15 @@ }, { "global index": "GeoGraphyData_100k_27", - "flip": 0 + "flip": 1 }, { "global index": "GeoGraphyData_100k_42", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_105", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_114", @@ -313,7 +313,7 @@ }, { "global index": "LiveCodeBench_131", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_136", @@ -321,7 +321,7 @@ }, { "global index": "LiveCodeBench_181", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_237", @@ -361,7 +361,7 @@ }, { "global index": "LiveCodeBench_485", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_49", @@ -369,7 +369,7 @@ }, { "global index": "LiveCodeBench_491", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_499", @@ -377,11 +377,11 @@ }, { "global index": "MATH_108", - "flip": 0 + "flip": 1 }, { "global index": "MATH_442", - "flip": 0 + "flip": 1 }, { "global index": "MATH_53", @@ -389,11 +389,11 @@ }, { "global index": "MMLUPro_biology_2808", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_biology_2912", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_biology_2980", @@ -417,7 +417,7 @@ }, { "global index": "MMLUPro_business_226", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_294", @@ -425,7 +425,7 @@ }, { "global index": "MMLUPro_business_378", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_430", @@ -437,7 +437,7 @@ }, { "global index": "MMLUPro_business_507", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_6", @@ -449,11 +449,11 @@ }, { "global index": "MMLUPro_chemistry_3796", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_chemistry_3837", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_chemistry_3974", @@ -473,7 +473,7 @@ }, { "global index": "MMLUPro_computer science_9110", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9136", @@ -485,7 +485,7 @@ }, { "global index": "MMLUPro_computer science_9149", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9200", @@ -501,7 +501,7 @@ }, { "global index": "MMLUPro_computer science_9264", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9285", @@ -513,27 +513,27 @@ }, { "global index": "MMLUPro_computer science_9414", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9415", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9430", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9452", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9471", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9475", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_5769", @@ -541,19 +541,19 @@ }, { "global index": "MMLUPro_economics_5907", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_5931", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_5965", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_6114", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_6122", @@ -569,7 +569,7 @@ }, { "global index": "MMLUPro_economics_6353", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10076", @@ -581,7 +581,7 @@ }, { "global index": "MMLUPro_engineering_10179", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10195", @@ -593,7 +593,7 @@ }, { "global index": "MMLUPro_engineering_10298", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10342", @@ -617,7 +617,7 @@ }, { "global index": "MMLUPro_engineering_10537", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10701", @@ -633,7 +633,7 @@ }, { "global index": "MMLUPro_health_4885", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_4973", @@ -653,15 +653,15 @@ }, { "global index": "MMLUPro_health_5215", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_5261", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_5473", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_5514", @@ -677,7 +677,7 @@ }, { "global index": "MMLUPro_history_4497", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4509", @@ -685,11 +685,11 @@ }, { "global index": "MMLUPro_history_4517", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4523", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4605", @@ -697,11 +697,11 @@ }, { "global index": "MMLUPro_history_4629", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4638", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4717", @@ -721,11 +721,11 @@ }, { "global index": "MMLUPro_history_4810", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4833", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4836", @@ -733,7 +733,7 @@ }, { "global index": "MMLUPro_history_4841", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1007", @@ -749,7 +749,7 @@ }, { "global index": "MMLUPro_law_1462", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1484", @@ -757,7 +757,7 @@ }, { "global index": "MMLUPro_law_1518", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1818", @@ -765,19 +765,19 @@ }, { "global index": "MMLUPro_law_806", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_899", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_6429", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_6526", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_6623", @@ -793,7 +793,7 @@ }, { "global index": "MMLUPro_math_7249", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_7284", @@ -801,7 +801,7 @@ }, { "global index": "MMLUPro_math_7451", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_7577", @@ -809,7 +809,7 @@ }, { "global index": "MMLUPro_philosophy_9510", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_philosophy_9536", @@ -817,7 +817,7 @@ }, { "global index": "MMLUPro_philosophy_9663", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_philosophy_9672", @@ -829,15 +829,15 @@ }, { "global index": "MMLUPro_physics_7773", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_physics_7887", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_physics_7893", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_physics_8888", @@ -845,7 +845,7 @@ }, { "global index": "MMLUPro_physics_9017", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2005", @@ -857,31 +857,31 @@ }, { "global index": "MMLUPro_psychology_2329", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2406", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2420", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2450", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2457", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2524", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_formal_logic_121", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_formal_logic_16", @@ -897,7 +897,7 @@ }, { "global index": "MMLU_formal_logic_7", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_formal_logic_70", @@ -909,7 +909,7 @@ }, { "global index": "MMLU_management_4", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_management_41", @@ -917,7 +917,7 @@ }, { "global index": "MMLU_management_77", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_management_91", @@ -925,7 +925,7 @@ }, { "global index": "MMLU_management_93", - "flip": 0 + "flip": 1 }, { "global index": "MathQA_158", @@ -933,11 +933,11 @@ }, { "global index": "MathQA_1742", - "flip": 0 + "flip": 1 }, { "global index": "MathQA_202", - "flip": 0 + "flip": 1 }, { "global index": "MathQA_2092", @@ -945,7 +945,7 @@ }, { "global index": "MathQA_2102", - "flip": 0 + "flip": 1 }, { "global index": "MathQA_2851", @@ -997,7 +997,7 @@ }, { "global index": "MedMCQA_2581", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_511", @@ -1017,11 +1017,11 @@ }, { "global index": "MusicTheoryBench_126", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_14", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_147", @@ -1029,11 +1029,11 @@ }, { "global index": "MusicTheoryBench_152", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_188", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_189", @@ -1041,15 +1041,15 @@ }, { "global index": "MusicTheoryBench_240", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_33", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_337", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_340", @@ -1069,15 +1069,15 @@ }, { "global index": "NarrativeQA_2474", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_2820", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_3282", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_4102", @@ -1097,7 +1097,7 @@ }, { "global index": "NarrativeQA_5022", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_5259", @@ -1117,7 +1117,7 @@ }, { "global index": "NarrativeQA_7678", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_7964", @@ -1133,7 +1133,7 @@ }, { "global index": "NarrativeQA_927", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Animals_2545", @@ -1149,19 +1149,19 @@ }, { "global index": "OpenTDB_Celebrities_1078", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Celebrities_1456", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Celebrities_3577", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Celebrities_3968", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_1178", @@ -1173,27 +1173,27 @@ }, { "global index": "OpenTDB_General Knowledge_1957", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_2181", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_3404", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_3727", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_4019", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_2099", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_2346", @@ -1201,7 +1201,7 @@ }, { "global index": "OpenTDB_Geography_3880", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_792", @@ -1209,11 +1209,11 @@ }, { "global index": "OpenTDB_History_1162", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_History_2026", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_History_3712", @@ -1225,7 +1225,7 @@ }, { "global index": "OpenTDB_Science & Nature_1175", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Science & Nature_1560", @@ -1233,7 +1233,7 @@ }, { "global index": "OpenTDB_Science & Nature_3716", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Science & Nature_476", @@ -1241,7 +1241,7 @@ }, { "global index": "OpenTDB_Sports_2289", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Vehicles_1173", @@ -1249,11 +1249,11 @@ }, { "global index": "OpenTDB_Vehicles_1419", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Vehicles_2519", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Vehicles_3234", @@ -1261,7 +1261,7 @@ }, { "global index": "PubMedQA_0", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_154", @@ -1269,11 +1269,11 @@ }, { "global index": "PubMedQA_18", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_238", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_250", @@ -1285,7 +1285,7 @@ }, { "global index": "PubMedQA_362", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_437", @@ -1301,11 +1301,11 @@ }, { "global index": "PubMedQA_575", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_582", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_588", @@ -1313,7 +1313,7 @@ }, { "global index": "PubMedQA_610", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_63", @@ -1321,7 +1321,7 @@ }, { "global index": "PubMedQA_643", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_687", @@ -1329,11 +1329,11 @@ }, { "global index": "PubMedQA_722", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_73", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_755", @@ -1341,19 +1341,19 @@ }, { "global index": "PubMedQA_8", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_81", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_854", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_905", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Fine Arts_1212", @@ -1365,15 +1365,15 @@ }, { "global index": "QANTA_Fine Arts_828", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Fine Arts_865", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Geography_1023", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Geography_1555", @@ -1381,23 +1381,23 @@ }, { "global index": "QANTA_Geography_304", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_1084", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_1154", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_433", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_473", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_926", @@ -1405,15 +1405,15 @@ }, { "global index": "QANTA_Literature_1045", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_1073", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_1239", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_1326", @@ -1425,11 +1425,11 @@ }, { "global index": "QANTA_Literature_1843", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_386", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_408", @@ -1437,7 +1437,7 @@ }, { "global index": "QANTA_Literature_475", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_833", @@ -1449,19 +1449,19 @@ }, { "global index": "QANTA_Philosophy_499", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Philosophy_91", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Science_1360", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Science_1473", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Science_308", @@ -1477,15 +1477,15 @@ }, { "global index": "QANTA_Social Science_2", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Social Science_77", - "flip": 0 + "flip": 1 }, { "global index": "SocialiQA_13810", - "flip": 0 + "flip": 1 }, { "global index": "SocialiQA_22095", @@ -1493,7 +1493,7 @@ }, { "global index": "SocialiQA_26846", - "flip": 0 + "flip": 1 }, { "global index": "SocialiQA_7839", @@ -1501,19 +1501,19 @@ }, { "global index": "SuperGLUE-CausalReasoning_4526", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-ClozeTest_12894", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-ClozeTest_17965", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-ClozeTest_18766", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Entailment_19410", @@ -1533,7 +1533,7 @@ }, { "global index": "SuperGLUE-QA_1408", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-QA_3137", @@ -1541,15 +1541,15 @@ }, { "global index": "SuperGLUE-QA_4046", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-QA_4102", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-QA_4160", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-RC_7725", @@ -1561,7 +1561,7 @@ }, { "global index": "SuperGLUE-RC_8531", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_19695", @@ -1569,7 +1569,7 @@ }, { "global index": "SuperGLUE-Wic_19738", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_20079", @@ -1581,11 +1581,11 @@ }, { "global index": "SuperGLUE-Wic_20253", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wsc_20368", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wsc_20370", @@ -1593,7 +1593,7 @@ }, { "global index": "WMT19-cs-en_156", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-cs-en_246", @@ -1601,15 +1601,15 @@ }, { "global index": "WMT19-cs-en_568", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-de-en_46", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-de-en_715", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-de-en_883", @@ -1617,7 +1617,7 @@ }, { "global index": "WMT19-fi-en_222", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-fi-en_610", @@ -1629,7 +1629,7 @@ }, { "global index": "WMT19-gu-en_123", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-gu-en_191", @@ -1637,7 +1637,7 @@ }, { "global index": "WMT19-gu-en_491", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-gu-en_968", @@ -1653,11 +1653,11 @@ }, { "global index": "WMT19-lt-en_135", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-lt-en_269", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-lt-en_636", diff --git a/src/data/routerMetrics/category_scores.json b/src/data/routerMetrics/category_scores.json index 5ad1a0d..d166c68 100644 --- a/src/data/routerMetrics/category_scores.json +++ b/src/data/routerMetrics/category_scores.json @@ -13934,48 +13934,48 @@ "sqwish-router": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.3, + "cost": 0.0002, + "robustness": 51.0 }, "medium": { - "accuracy": 77.4, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 79.4, + "cost": 0.0008, + "robustness": 44.2 }, "hard": { - "accuracy": 34.1, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 44.7, + "cost": 0.0015, + "robustness": 60.6 }, "all": { - "accuracy": 76.4, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 79.8, + "cost": 0.0007, + "robustness": 51.7 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 94.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.0, + "cost": 0.0006, + "robustness": 50.0 }, "medium": { - "accuracy": 65.1, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 76.5, + "cost": 0.0019, + "robustness": 9.1 }, "hard": { - "accuracy": 14.3, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 33.3, + "cost": 0.0026, + "robustness": 60.0 }, "all": { - "accuracy": 66.7, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 76.1, + "cost": 0.0015, + "robustness": 33.3 } }, "subcategories": { @@ -13983,71 +13983,71 @@ "metrics": { "easy": { "accuracy": 98.3, - "cost": 0.0001, + "cost": 0.0004, "robustness": 100.0 }, "medium": { - "accuracy": 87.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 94.9, + "cost": 0.0014, + "robustness": 0.0 }, "hard": { - "accuracy": 12.5, - "cost": 0.0, + "accuracy": 33.9, + "cost": 0.0034, "robustness": 100.0 }, "all": { - "accuracy": 64.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 74.0, + "cost": 0.0018, + "robustness": 60.0 } } }, "Music": { "metrics": { "easy": { - "accuracy": 93.9, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 97.0, + "cost": 0.0004, + "robustness": 60.0 }, "medium": { - "accuracy": 53.4, - "cost": 0.0007, - "robustness": 100.0 + "accuracy": 58.6, + "cost": 0.0019, + "robustness": 20.0 }, "hard": { - "accuracy": 12.0, - "cost": 0.0013, - "robustness": 100.0 + "accuracy": 40.0, + "cost": 0.0029, + "robustness": 0.0 }, "all": { - "accuracy": 65.8, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 72.5, + "cost": 0.0014, + "robustness": 36.4 } } }, "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 92.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.4, + "cost": 0.0008, + "robustness": 0.0 }, "medium": { - "accuracy": 70.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 90.0, + "cost": 0.0022, + "robustness": 0.0 }, "hard": { - "accuracy": 16.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 30.3, + "cost": 0.0018, + "robustness": 50.0 }, "all": { - "accuracy": 68.6, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 80.1, + "cost": 0.0015, + "robustness": 12.5 } } } @@ -14056,72 +14056,72 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 97.5, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 98.0, + "cost": 0.0002, + "robustness": 53.8 }, "medium": { - "accuracy": 84.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 87.4, + "cost": 0.0005, + "robustness": 57.1 }, "hard": { - "accuracy": 34.7, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 55.1, + "cost": 0.0012, + "robustness": 60.0 }, "all": { - "accuracy": 82.1, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 87.0, + "cost": 0.0005, + "robustness": 56.5 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 98.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 98.6, + "cost": 0.0002, + "robustness": 53.8 }, "medium": { - "accuracy": 87.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 91.5, + "cost": 0.0006, + "robustness": 52.9 }, "hard": { - "accuracy": 38.8, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 61.7, + "cost": 0.0014, + "robustness": 61.5 }, "all": { - "accuracy": 81.9, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 88.2, + "cost": 0.0006, + "robustness": 55.8 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 95.6, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 97.1, + "cost": 0.0002, + "robustness": 53.8 }, "medium": { - "accuracy": 70.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 69.4, + "cost": 0.0003, + "robustness": 75.0 }, "hard": { - "accuracy": 11.1, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 16.7, + "cost": 0.0002, + "robustness": 50.0 }, "all": { - "accuracy": 82.6, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 83.8, + "cost": 0.0002, + "robustness": 57.9 } } } @@ -14130,37 +14130,37 @@ "History": { "metrics": { "easy": { - "accuracy": 97.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.6, + "cost": 0.0003, + "robustness": 39.1 }, "medium": { - "accuracy": 74.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 81.4, + "cost": 0.001, + "robustness": 33.3 }, "hard": { - "accuracy": 14.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 29.5, + "cost": 0.0021, + "robustness": 40.0 }, "all": { - "accuracy": 73.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 79.0, + "cost": 0.0009, + "robustness": 38.5 } }, "subcategories": { "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 0.0 }, "medium": { - "accuracy": 77.8, - "cost": 0.0001, + "accuracy": 100.0, + "cost": 0.0002, "robustness": 0 }, "hard": { @@ -14169,57 +14169,57 @@ "robustness": 0 }, "all": { - "accuracy": 92.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 98.1, + "cost": 0.0003, + "robustness": 0.0 } } }, "Geography": { "metrics": { "easy": { - "accuracy": 98.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.2, + "cost": 0.0003, + "robustness": 37.5 }, "medium": { - "accuracy": 62.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 65.5, + "cost": 0.0014, + "robustness": 0.0 }, "hard": { - "accuracy": 25.0, - "cost": 0.0, + "accuracy": 58.3, + "cost": 0.0027, "robustness": 0 }, "all": { - "accuracy": 85.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 88.0, + "cost": 0.0007, + "robustness": 33.3 } } }, "History": { "metrics": { "easy": { - "accuracy": 96.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.3, + "cost": 0.0004, + "robustness": 54.5 }, "medium": { - "accuracy": 76.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 83.5, + "cost": 0.0009, + "robustness": 40.0 }, "hard": { - "accuracy": 13.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 27.2, + "cost": 0.0021, + "robustness": 40.0 }, "all": { - "accuracy": 68.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 74.3, + "cost": 0.001, + "robustness": 46.2 } } } @@ -14228,48 +14228,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 91.9, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 93.3, + "cost": 0.0002, + "robustness": 54.5 }, "medium": { - "accuracy": 74.9, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 55.7, + "cost": 0.0002, + "robustness": 55.6 }, "hard": { - "accuracy": 59.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 65.1, + "cost": 0.0007, + "robustness": 52.0 }, "all": { - "accuracy": 72.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 71.3, + "cost": 0.0004, + "robustness": 53.3 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 91.9, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 93.3, + "cost": 0.0002, + "robustness": 54.5 }, "medium": { - "accuracy": 74.9, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 55.7, + "cost": 0.0002, + "robustness": 55.6 }, "hard": { - "accuracy": 59.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 65.1, + "cost": 0.0007, + "robustness": 52.0 }, "all": { - "accuracy": 72.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 71.3, + "cost": 0.0004, + "robustness": 53.3 } } } @@ -14279,23 +14279,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.0003, + "robustness": 66.7 }, "medium": { - "accuracy": 77.5, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 88.2, + "cost": 0.0014, + "robustness": 50.0 }, "hard": { - "accuracy": 35.9, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 47.1, + "cost": 0.0019, + "robustness": 58.3 }, "all": { - "accuracy": 50.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 59.7, + "cost": 0.0016, + "robustness": 58.6 } }, "subcategories": { @@ -14303,23 +14303,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.0003, + "robustness": 66.7 }, "medium": { - "accuracy": 77.5, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 88.2, + "cost": 0.0014, + "robustness": 50.0 }, "hard": { - "accuracy": 35.9, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 47.1, + "cost": 0.0019, + "robustness": 58.3 }, "all": { - "accuracy": 50.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 59.7, + "cost": 0.0016, + "robustness": 58.6 } } } @@ -14328,48 +14328,48 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 96.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.3, + "cost": 0.0002, + "robustness": 37.0 }, "medium": { - "accuracy": 84.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 76.8, + "cost": 0.0005, + "robustness": 30.0 }, "hard": { "accuracy": 38.3, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.002, + "robustness": 42.9 }, "all": { - "accuracy": 86.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 83.7, + "cost": 0.0005, + "robustness": 36.4 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 95.1, + "accuracy": 93.0, "cost": 0.0001, - "robustness": 100.0 + "robustness": 28.6 }, "medium": { - "accuracy": 78.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 55.3, + "cost": 0.0002, + "robustness": 0.0 }, "hard": { - "accuracy": 84.6, + "accuracy": 0.0, "cost": 0.0, - "robustness": 100.0 + "robustness": 50.0 }, "all": { - "accuracy": 89.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 77.4, + "cost": 0.0002, + "robustness": 23.8 } } }, @@ -14377,23 +14377,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.0002, + "robustness": 75.0 }, "medium": { "accuracy": 97.4, - "cost": 0.0002, - "robustness": 100.0 + "cost": 0.0004, + "robustness": 66.7 }, "hard": { - "accuracy": 80.0, - "cost": 0.0005, + "accuracy": 60.0, + "cost": 0.0004, "robustness": 0 }, "all": { - "accuracy": 98.1, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 97.2, + "cost": 0.0003, + "robustness": 71.4 } } }, @@ -14401,47 +14401,47 @@ "metrics": { "easy": { "accuracy": 97.3, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.0006, + "robustness": 66.7 }, "medium": { - "accuracy": 77.5, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 92.5, + "cost": 0.0012, + "robustness": 50.0 }, "hard": { - "accuracy": 26.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 48.9, + "cost": 0.0032, + "robustness": 33.3 }, "all": { - "accuracy": 64.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 77.9, + "cost": 0.0018, + "robustness": 50.0 } } }, "Psychology": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 99.2, + "cost": 0.0003, + "robustness": 16.7 }, "medium": { - "accuracy": 89.6, - "cost": 0.0001, + "accuracy": 85.4, + "cost": 0.0004, "robustness": 0 }, "hard": { - "accuracy": 22.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 33.3, + "cost": 0.0007, + "robustness": 50.0 }, "all": { - "accuracy": 87.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 89.3, + "cost": 0.0004, + "robustness": 25.0 } } } @@ -14450,24 +14450,24 @@ "Science": { "metrics": { "easy": { - "accuracy": 99.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 99.4, + "cost": 0.0002, + "robustness": 64.1 }, "medium": { - "accuracy": 82.6, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 82.9, + "cost": 0.0006, + "robustness": 23.8 }, "hard": { - "accuracy": 30.9, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 38.1, + "cost": 0.0014, + "robustness": 77.8 }, "all": { - "accuracy": 84.5, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 85.8, + "cost": 0.0005, + "robustness": 53.6 } }, "subcategories": { @@ -14475,12 +14475,12 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, + "cost": 0.0, "robustness": 100.0 }, "medium": { "accuracy": 60.0, - "cost": 0.0001, + "cost": 0.0002, "robustness": 0 }, "hard": { @@ -14499,95 +14499,95 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.0002, + "robustness": 71.4 }, "medium": { "accuracy": 76.5, - "cost": 0.0001, + "cost": 0.0006, "robustness": 0 }, "hard": { - "accuracy": 44.4, - "cost": 0.0003, + "accuracy": 33.3, + "cost": 0.0014, "robustness": 0 }, "all": { - "accuracy": 93.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 93.0, + "cost": 0.0003, + "robustness": 71.4 } } }, "Chemistry": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 66.7 }, "medium": { - "accuracy": 91.2, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 88.2, + "cost": 0.0004, + "robustness": 50.0 }, "hard": { "accuracy": 23.1, - "cost": 0.0003, + "cost": 0.0005, "robustness": 0 }, "all": { "accuracy": 84.4, - "cost": 0.0002, - "robustness": 100.0 + "cost": 0.0004, + "robustness": 60.0 } } }, "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 98.8, + "accuracy": 99.2, "cost": 0.0001, - "robustness": 100.0 + "robustness": 62.5 }, "medium": { - "accuracy": 72.9, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 76.3, + "cost": 0.0004, + "robustness": 16.7 }, "hard": { - "accuracy": 15.4, - "cost": 0.0001, + "accuracy": 23.1, + "cost": 0.0002, "robustness": 100.0 }, "all": { - "accuracy": 90.9, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 92.1, + "cost": 0.0002, + "robustness": 46.7 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 98.9, + "accuracy": 99.2, "cost": 0.0001, - "robustness": 100.0 + "robustness": 72.7 }, "medium": { - "accuracy": 84.8, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 85.2, + "cost": 0.0006, + "robustness": 27.3 }, "hard": { - "accuracy": 47.2, - "cost": 0.001, - "robustness": 100.0 + "accuracy": 57.3, + "cost": 0.0013, + "robustness": 80.0 }, "all": { - "accuracy": 85.4, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 87.2, + "cost": 0.0005, + "robustness": 55.6 } } }, @@ -14595,23 +14595,23 @@ "metrics": { "easy": { "accuracy": 97.7, - "cost": 0.0002, - "robustness": 100.0 + "cost": 0.0004, + "robustness": 33.3 }, "medium": { "accuracy": 88.1, - "cost": 0.0004, - "robustness": 100.0 + "cost": 0.0007, + "robustness": 0.0 }, "hard": { - "accuracy": 45.5, - "cost": 0.0003, + "accuracy": 54.5, + "cost": 0.0007, "robustness": 0 }, "all": { - "accuracy": 87.6, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 88.7, + "cost": 0.0006, + "robustness": 20.0 } } }, @@ -14620,22 +14620,22 @@ "easy": { "accuracy": 100.0, "cost": 0.0001, - "robustness": 100.0 + "robustness": 40.0 }, "medium": { - "accuracy": 72.0, - "cost": 0.0, + "accuracy": 68.0, + "cost": 0.0016, "robustness": 0 }, "hard": { - "accuracy": 6.8, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 13.6, + "cost": 0.0022, + "robustness": 66.7 }, "all": { - "accuracy": 45.6, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 48.2, + "cost": 0.0015, + "robustness": 50.0 } } } @@ -14644,96 +14644,96 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 96.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.0, + "cost": 0.0003, + "robustness": 50.0 }, "medium": { - "accuracy": 74.3, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 77.9, + "cost": 0.0008, + "robustness": 50.0 }, "hard": { - "accuracy": 17.4, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 25.5, + "cost": 0.0013, + "robustness": 85.7 }, "all": { - "accuracy": 71.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 74.9, + "cost": 0.0007, + "robustness": 56.8 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 97.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 98.4, + "cost": 0.0002, + "robustness": 58.3 }, "medium": { - "accuracy": 79.1, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 82.8, + "cost": 0.0006, + "robustness": 66.7 }, "hard": { - "accuracy": 18.8, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 25.0, + "cost": 0.0008, + "robustness": 66.7 }, "all": { - "accuracy": 75.8, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 78.6, + "cost": 0.0005, + "robustness": 61.9 } } }, "Law": { "metrics": { "easy": { - "accuracy": 95.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 93.8, + "cost": 0.0006, + "robustness": 33.3 }, "medium": { - "accuracy": 71.4, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 74.6, + "cost": 0.0011, + "robustness": 50.0 }, "hard": { "accuracy": 26.7, - "cost": 0.0003, + "cost": 0.0008, "robustness": 100.0 }, "all": { - "accuracy": 72.8, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 73.4, + "cost": 0.0009, + "robustness": 55.6 } } }, "Social problems": { "metrics": { "easy": { - "accuracy": 93.6, + "accuracy": 95.7, "cost": 0.0001, - "robustness": 100.0 + "robustness": 33.3 }, "medium": { - "accuracy": 33.3, + "accuracy": 22.2, "cost": 0.0, "robustness": 0 }, "hard": { - "accuracy": 0.0, + "accuracy": 20.0, "cost": 0.0, "robustness": 100.0 }, "all": { - "accuracy": 77.0, + "accuracy": 78.7, "cost": 0.0001, - "robustness": 100.0 + "robustness": 50.0 } } }, @@ -14741,23 +14741,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0, + "cost": 0.0008, "robustness": 0 }, "medium": { - "accuracy": 61.1, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 72.2, + "cost": 0.0014, + "robustness": 0.0 }, "hard": { - "accuracy": 8.8, - "cost": 0.0, + "accuracy": 26.5, + "cost": 0.0029, "robustness": 100.0 }, "all": { - "accuracy": 30.9, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 45.5, + "cost": 0.0023, + "robustness": 33.3 } } } @@ -14766,96 +14766,96 @@ "Technology": { "metrics": { "easy": { - "accuracy": 95.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 95.9, + "cost": 0.0002, + "robustness": 51.2 }, "medium": { - "accuracy": 72.9, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 78.1, + "cost": 0.0007, + "robustness": 71.4 }, "hard": { - "accuracy": 30.6, - "cost": 0.0003, + "accuracy": 33.1, + "cost": 0.0008, "robustness": 100.0 }, "all": { - "accuracy": 81.5, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 83.5, + "cost": 0.0004, + "robustness": 62.0 } }, "subcategories": { "Engineering": { "metrics": { "easy": { - "accuracy": 98.4, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 97.6, + "cost": 0.0004, + "robustness": 60.0 }, "medium": { - "accuracy": 88.0, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 93.4, + "cost": 0.0009, + "robustness": 76.9 }, "hard": { - "accuracy": 57.1, - "cost": 0.0009, + "accuracy": 64.3, + "cost": 0.0014, "robustness": 100.0 }, "all": { - "accuracy": 88.1, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 91.3, + "cost": 0.0008, + "robustness": 73.7 } } }, "Management and public relations": { "metrics": { "easy": { - "accuracy": 98.3, + "accuracy": 95.0, "cost": 0.0001, - "robustness": 100.0 + "robustness": 50.0 }, "medium": { - "accuracy": 63.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 45.5, + "cost": 0.0002, + "robustness": 0.0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, + "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 90.4, + "accuracy": 84.9, "cost": 0.0001, - "robustness": 100.0 + "robustness": 40.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 94.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 95.6, + "cost": 0.0002, + "robustness": 50.0 }, "medium": { - "accuracy": 62.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 69.0, + "cost": 0.0005, + "robustness": 71.4 }, "hard": { - "accuracy": 21.2, - "cost": 0.0001, + "accuracy": 22.1, + "cost": 0.0006, "robustness": 100.0 }, "all": { - "accuracy": 78.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 80.7, + "cost": 0.0003, + "robustness": 59.6 } } } @@ -17034,38 +17034,3766 @@ "robustness": 11.1 }, "hard": { - "accuracy": 61.6, - "cost": 0.0012, - "robustness": 16.0 + "accuracy": 61.6, + "cost": 0.0012, + "robustness": 16.0 + }, + "all": { + "accuracy": 70.5, + "cost": 0.0009, + "robustness": 15.6 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 93.8, + "cost": 0.0006, + "robustness": 18.2 + }, + "medium": { + "accuracy": 58.7, + "cost": 0.0007, + "robustness": 11.1 + }, + "hard": { + "accuracy": 61.6, + "cost": 0.0012, + "robustness": 16.0 + }, + "all": { + "accuracy": 70.5, + "cost": 0.0009, + "robustness": 15.6 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0006, + "robustness": 33.3 + }, + "medium": { + "accuracy": 76.7, + "cost": 0.0013, + "robustness": 0.0 + }, + "hard": { + "accuracy": 32.0, + "cost": 0.0015, + "robustness": 16.7 + }, + "all": { + "accuracy": 47.1, + "cost": 0.0013, + "robustness": 17.2 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0006, + "robustness": 33.3 + }, + "medium": { + "accuracy": 76.7, + "cost": 0.0013, + "robustness": 0.0 + }, + "hard": { + "accuracy": 32.0, + "cost": 0.0015, + "robustness": 16.7 + }, + "all": { + "accuracy": 47.1, + "cost": 0.0013, + "robustness": 17.2 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 96.1, + "cost": 0.0007, + "robustness": 33.3 + }, + "medium": { + "accuracy": 66.4, + "cost": 0.001, + "robustness": 20.0 + }, + "hard": { + "accuracy": 27.2, + "cost": 0.0013, + "robustness": 42.9 + }, + "all": { + "accuracy": 79.1, + "cost": 0.0009, + "robustness": 31.8 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 94.1, + "cost": 0.0006, + "robustness": 28.6 + }, + "medium": { + "accuracy": 40.0, + "cost": 0.0008, + "robustness": 20.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0006, + "robustness": 0.0 + }, + "all": { + "accuracy": 74.2, + "cost": 0.0006, + "robustness": 23.8 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0009, + "robustness": 0.0 + }, + "medium": { + "accuracy": 89.5, + "cost": 0.0012, + "robustness": 33.3 + }, + "hard": { + "accuracy": 60.0, + "cost": 0.0027, + "robustness": 0 + }, + "all": { + "accuracy": 94.4, + "cost": 0.0011, + "robustness": 14.3 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 89.2, + "cost": 0.001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0011, + "robustness": 0.0 + }, + "hard": { + "accuracy": 28.9, + "cost": 0.0016, + "robustness": 66.7 + }, + "all": { + "accuracy": 63.1, + "cost": 0.0013, + "robustness": 50.0 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 99.2, + "cost": 0.0007, + "robustness": 50.0 + }, + "medium": { + "accuracy": 85.4, + "cost": 0.001, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0007, + "robustness": 50.0 + }, + "all": { + "accuracy": 88.2, + "cost": 0.0008, + "robustness": 50.0 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0006, + "robustness": 10.3 + }, + "medium": { + "accuracy": 81.7, + "cost": 0.0012, + "robustness": 14.3 + }, + "hard": { + "accuracy": 28.4, + "cost": 0.0021, + "robustness": 44.4 + }, + "all": { + "accuracy": 83.4, + "cost": 0.001, + "robustness": 15.9 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0003, + "robustness": 0.0 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0011, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 88.0, + "cost": 0.0004, + "robustness": 0.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 97.4, + "cost": 0.0006, + "robustness": 14.3 + }, + "medium": { + "accuracy": 82.4, + "cost": 0.0007, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0011, + "robustness": 0 + }, + "all": { + "accuracy": 90.9, + "cost": 0.0006, + "robustness": 14.3 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0008, + "robustness": 33.3 + }, + "medium": { + "accuracy": 94.1, + "cost": 0.0015, + "robustness": 0.0 + }, + "hard": { + "accuracy": 30.8, + "cost": 0.001, + "robustness": 0 + }, + "all": { + "accuracy": 87.8, + "cost": 0.0011, + "robustness": 20.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 98.1, + "cost": 0.0005, + "robustness": 0.0 + }, + "medium": { + "accuracy": 74.6, + "cost": 0.0009, + "robustness": 0.0 + }, + "hard": { + "accuracy": 23.1, + "cost": 0.0005, + "robustness": 100.0 + }, + "all": { + "accuracy": 90.9, + "cost": 0.0006, + "robustness": 6.7 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 98.9, + "cost": 0.0007, + "robustness": 18.2 + }, + "medium": { + "accuracy": 84.0, + "cost": 0.0012, + "robustness": 27.3 + }, + "hard": { + "accuracy": 38.2, + "cost": 0.003, + "robustness": 60.0 + }, + "all": { + "accuracy": 83.7, + "cost": 0.0013, + "robustness": 29.6 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0005, + "robustness": 0.0 + }, + "medium": { + "accuracy": 76.2, + "cost": 0.0015, + "robustness": 0.0 + }, + "hard": { + "accuracy": 63.6, + "cost": 0.0021, + "robustness": 0 + }, + "all": { + "accuracy": 84.5, + "cost": 0.0011, + "robustness": 0.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0005, + "robustness": 0.0 + }, + "medium": { + "accuracy": 72.0, + "cost": 0.0015, + "robustness": 0 + }, + "hard": { + "accuracy": 8.5, + "cost": 0.0015, + "robustness": 0.0 + }, + "all": { + "accuracy": 45.6, + "cost": 0.0012, + "robustness": 0.0 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0008, + "robustness": 27.8 + }, + "medium": { + "accuracy": 77.1, + "cost": 0.0015, + "robustness": 8.3 + }, + "hard": { + "accuracy": 18.1, + "cost": 0.0016, + "robustness": 42.9 + }, + "all": { + "accuracy": 72.1, + "cost": 0.0012, + "robustness": 24.3 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 96.2, + "cost": 0.0008, + "robustness": 25.0 + }, + "medium": { + "accuracy": 79.8, + "cost": 0.0013, + "robustness": 16.7 + }, + "hard": { + "accuracy": 18.8, + "cost": 0.0019, + "robustness": 0.0 + }, + "all": { + "accuracy": 75.4, + "cost": 0.0012, + "robustness": 19.0 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 92.3, + "cost": 0.0013, + "robustness": 33.3 + }, + "medium": { + "accuracy": 77.8, + "cost": 0.002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 20.0, + "cost": 0.001, + "robustness": 100.0 + }, + "all": { + "accuracy": 72.8, + "cost": 0.0015, + "robustness": 33.3 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 95.7, + "cost": 0.0003, + "robustness": 33.3 + }, + "medium": { + "accuracy": 44.4, + "cost": 0.0007, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 0.0 + }, + "all": { + "accuracy": 80.3, + "cost": 0.0004, + "robustness": 25.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 66.7, + "cost": 0.0001, + "robustness": 0 + }, + "medium": { + "accuracy": 66.7, + "cost": 0.0016, + "robustness": 0.0 + }, + "hard": { + "accuracy": 17.6, + "cost": 0.0017, + "robustness": 100.0 + }, + "all": { + "accuracy": 36.4, + "cost": 0.0016, + "robustness": 33.3 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 96.6, + "cost": 0.0006, + "robustness": 25.6 + }, + "medium": { + "accuracy": 75.0, + "cost": 0.0011, + "robustness": 19.0 + }, + "hard": { + "accuracy": 24.2, + "cost": 0.0013, + "robustness": 28.6 + }, + "all": { + "accuracy": 82.0, + "cost": 0.0009, + "robustness": 23.9 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0009, + "robustness": 0.0 + }, + "medium": { + "accuracy": 88.0, + "cost": 0.0013, + "robustness": 15.4 + }, + "hard": { + "accuracy": 35.7, + "cost": 0.0021, + "robustness": 0.0 + }, + "all": { + "accuracy": 85.1, + "cost": 0.0013, + "robustness": 10.5 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0006, + "robustness": 25.0 + }, + "medium": { + "accuracy": 63.6, + "cost": 0.0005, + "robustness": 0.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 + }, + "all": { + "accuracy": 89.0, + "cost": 0.0006, + "robustness": 20.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 96.4, + "cost": 0.0006, + "robustness": 29.4 + }, + "medium": { + "accuracy": 66.5, + "cost": 0.0011, + "robustness": 28.6 + }, + "hard": { + "accuracy": 20.4, + "cost": 0.001, + "robustness": 33.3 + }, + "all": { + "accuracy": 80.4, + "cost": 0.0007, + "robustness": 29.8 + } + } + } + } + } + } + }, + "hybrid-router": { + "metrics": { + "easy": { + "accuracy": 96.2, + "cost": 0.0, + "robustness": 96.0 + }, + "medium": { + "accuracy": 67.7, + "cost": 0.0, + "robustness": 96.5 + }, + "hard": { + "accuracy": 25.7, + "cost": 0.0001, + "robustness": 98.2 + }, + "all": { + "accuracy": 71.4, + "cost": 0.0, + "robustness": 96.7 + } + }, + "categories": { + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 94.3, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 57.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 7.5, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 62.6, + "cost": 0.0, + "robustness": 100.0 + } + }, + "subcategories": { + "Arts": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 74.4, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 5.4, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 58.4, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 96.0, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 50.9, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 12.0, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 65.4, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 91.4, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 58.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 7.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 62.4, + "cost": 0.0001, + "robustness": 100.0 + } + } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0, + "robustness": 92.3 + }, + "medium": { + "accuracy": 80.3, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 33.5, + "cost": 0.0002, + "robustness": 100.0 + }, + "all": { + "accuracy": 80.6, + "cost": 0.0001, + "robustness": 96.8 + } + }, + "subcategories": { + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 98.6, + "cost": 0.0, + "robustness": 84.6 + }, + "medium": { + "accuracy": 82.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 37.8, + "cost": 0.0002, + "robustness": 100.0 + }, + "all": { + "accuracy": 80.0, + "cost": 0.0001, + "robustness": 95.3 + } + } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 96.0, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 69.4, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 8.3, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 82.3, + "cost": 0.0, + "robustness": 100.0 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.5, + "cost": 0.0, + "robustness": 95.7 + }, + "medium": { + "accuracy": 62.1, + "cost": 0.0, + "robustness": 83.3 + }, + "hard": { + "accuracy": 9.4, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 69.3, + "cost": 0.0, + "robustness": 94.9 + } + }, + "subcategories": { + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 92.9, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 66.7, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 88.5, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Geography": { + "metrics": { + "easy": { + "accuracy": 96.3, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 48.3, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 25.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 81.3, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0, + "robustness": 90.9 + }, + "medium": { + "accuracy": 64.7, + "cost": 0.0, + "robustness": 80.0 + }, + "hard": { + "accuracy": 7.4, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 63.7, + "cost": 0.0, + "robustness": 92.3 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 91.4, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 50.3, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 48.6, + "cost": 0.0, + "robustness": 96.0 + }, + "all": { + "accuracy": 61.8, + "cost": 0.0, + "robustness": 97.8 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 91.4, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 50.3, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 48.6, + "cost": 0.0, + "robustness": 96.0 + }, + "all": { + "accuracy": 61.8, + "cost": 0.0, + "robustness": 97.8 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 65.0, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 28.7, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 43.2, + "cost": 0.0, + "robustness": 100.0 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 65.0, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 28.7, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 43.2, + "cost": 0.0, + "robustness": 100.0 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 96.3, + "cost": 0.0, + "robustness": 96.3 + }, + "medium": { + "accuracy": 62.1, + "cost": 0.0, + "robustness": 80.0 + }, + "hard": { + "accuracy": 14.8, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 76.6, + "cost": 0.0, + "robustness": 93.2 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 94.6, + "cost": 0.0, + "robustness": 92.9 + }, + "medium": { + "accuracy": 37.6, + "cost": 0.0, + "robustness": 60.0 + }, + "hard": { + "accuracy": 7.7, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 73.5, + "cost": 0.0, + "robustness": 85.7 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 84.2, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 80.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 93.5, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 94.6, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 67.5, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 8.9, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 54.1, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 83.3, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 16.7, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 86.1, + "cost": 0.0, + "robustness": 100.0 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 97.9, + "cost": 0.0, + "robustness": 97.4 + }, + "medium": { + "accuracy": 75.9, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 21.1, + "cost": 0.0001, + "robustness": 88.9 + }, + "all": { + "accuracy": 80.5, + "cost": 0.0, + "robustness": 97.1 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 80.0, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 92.0, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 64.7, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 88.8, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 82.4, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 23.1, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 81.1, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 72.9, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0.0 + }, + "all": { + "accuracy": 89.4, + "cost": 0.0, + "robustness": 93.3 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 99.2, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 79.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 37.1, + "cost": 0.0002, + "robustness": 100.0 + }, + "all": { + "accuracy": 81.9, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 95.5, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 78.6, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 27.3, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 80.4, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 93.3, + "cost": 0.0, + "robustness": 80.0 + }, + "medium": { + "accuracy": 40.0, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 1.7, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 34.2, + "cost": 0.0, + "robustness": 87.5 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 95.3, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 62.5, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 7.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 64.7, + "cost": 0.0, + "robustness": 100.0 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 68.7, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 6.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 69.2, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 90.8, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 60.3, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 10.0, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 63.3, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 93.6, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 22.2, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 20.0, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 0 + }, + "medium": { + "accuracy": 33.3, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 5.9, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 20.0, + "cost": 0.0, + "robustness": 100.0 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 95.5, + "cost": 0.0, + "robustness": 93.0 + }, + "medium": { + "accuracy": 67.6, + "cost": 0.0, + "robustness": 95.2 + }, + "hard": { + "accuracy": 17.8, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 78.4, + "cost": 0.0, + "robustness": 94.4 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 96.0, + "cost": 0.0, + "robustness": 80.0 + }, + "medium": { + "accuracy": 80.8, + "cost": 0.0001, + "robustness": 92.3 + }, + "hard": { + "accuracy": 35.7, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 80.9, + "cost": 0.0001, + "robustness": 89.5 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 63.6, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 91.8, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0, + "robustness": 94.1 + }, + "medium": { + "accuracy": 58.7, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 11.5, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 76.6, + "cost": 0.0, + "robustness": 95.7 + } + } + } + } + } + } + }, + "cross-router": { + "metrics": { + "easy": { + "accuracy": 96.5, + "cost": 0.0001, + "robustness": 59.1 + }, + "medium": { + "accuracy": 78.3, + "cost": 0.0003, + "robustness": 69.9 + }, + "hard": { + "accuracy": 41.0, + "cost": 0.0006, + "robustness": 78.9 + }, + "all": { + "accuracy": 78.2, + "cost": 0.0003, + "robustness": 67.1 + } + }, + "categories": { + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 95.3, + "cost": 0.0002, + "robustness": 50.0 + }, + "medium": { + "accuracy": 67.5, + "cost": 0.0006, + "robustness": 72.7 + }, + "hard": { + "accuracy": 23.8, + "cost": 0.0014, + "robustness": 80.0 + }, + "all": { + "accuracy": 70.1, + "cost": 0.0006, + "robustness": 66.7 + } + }, + "subcategories": { + "Arts": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 79.5, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 28.6, + "cost": 0.0021, + "robustness": 100.0 + }, + "all": { + "accuracy": 68.2, + "cost": 0.0009, + "robustness": 100.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 92.9, + "cost": 0.0002, + "robustness": 40.0 + }, + "medium": { + "accuracy": 61.2, + "cost": 0.0008, + "robustness": 40.0 + }, + "hard": { + "accuracy": 12.0, + "cost": 0.0013, + "robustness": 0.0 + }, + "all": { + "accuracy": 69.2, + "cost": 0.0006, + "robustness": 36.4 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 95.7, + "cost": 0.0002, + "robustness": 50.0 + }, + "medium": { + "accuracy": 70.0, + "cost": 0.0005, + "robustness": 100.0 + }, + "hard": { + "accuracy": 24.2, + "cost": 0.0008, + "robustness": 100.0 + }, + "all": { + "accuracy": 71.9, + "cost": 0.0004, + "robustness": 87.5 + } + } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 53.8 + }, + "medium": { + "accuracy": 84.4, + "cost": 0.0003, + "robustness": 52.4 + }, + "hard": { + "accuracy": 54.3, + "cost": 0.0007, + "robustness": 60.0 + }, + "all": { + "accuracy": 85.5, + "cost": 0.0003, + "robustness": 54.8 + } + }, + "subcategories": { + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 97.8, + "cost": 0.0001, + "robustness": 30.8 + }, + "medium": { + "accuracy": 89.9, + "cost": 0.0004, + "robustness": 64.7 + }, + "hard": { + "accuracy": 62.2, + "cost": 0.0008, + "robustness": 53.8 + }, + "all": { + "accuracy": 87.5, + "cost": 0.0004, + "robustness": 51.2 + } + } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 96.4, + "cost": 0.0001, + "robustness": 76.9 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 8.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 80.6, + "cost": 0.0001, + "robustness": 63.2 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0001, + "robustness": 39.1 + }, + "medium": { + "accuracy": 76.3, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 18.1, + "cost": 0.0006, + "robustness": 70.0 + }, + "all": { + "accuracy": 75.3, + "cost": 0.0002, + "robustness": 56.4 + } + }, + "subcategories": { + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 88.9, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 + }, + "all": { + "accuracy": 94.2, + "cost": 0.0001, + "robustness": 50.0 + } + } + }, + "Geography": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0001, + "robustness": 62.5 + }, + "medium": { + "accuracy": 65.5, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 41.7, + "cost": 0.0006, + "robustness": 0 + }, + "all": { + "accuracy": 87.3, + "cost": 0.0001, + "robustness": 66.7 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 18.2 + }, + "medium": { + "accuracy": 77.7, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 16.2, + "cost": 0.0006, + "robustness": 70.0 + }, + "all": { + "accuracy": 69.7, + "cost": 0.0003, + "robustness": 53.8 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 95.7, + "cost": 0.0001, + "robustness": 72.7 + }, + "medium": { + "accuracy": 77.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 68.0, + "cost": 0.0002, + "robustness": 92.0 + }, + "all": { + "accuracy": 78.5, + "cost": 0.0001, + "robustness": 88.9 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 95.7, + "cost": 0.0001, + "robustness": 72.7 + }, + "medium": { + "accuracy": 77.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 68.0, + "cost": 0.0002, + "robustness": 92.0 + }, + "all": { + "accuracy": 78.5, + "cost": 0.0001, + "robustness": 88.9 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 88.7, + "cost": 0.0004, + "robustness": 100.0 + }, + "hard": { + "accuracy": 40.5, + "cost": 0.0008, + "robustness": 91.7 + }, + "all": { + "accuracy": 54.8, + "cost": 0.0006, + "robustness": 93.1 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 88.7, + "cost": 0.0004, + "robustness": 100.0 + }, + "hard": { + "accuracy": 40.5, + "cost": 0.0008, + "robustness": 91.7 + }, + "all": { + "accuracy": 54.8, + "cost": 0.0006, + "robustness": 93.1 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 94.4, + "cost": 0.0001, + "robustness": 63.0 + }, + "medium": { + "accuracy": 78.7, + "cost": 0.0001, + "robustness": 70.0 + }, + "hard": { + "accuracy": 34.6, + "cost": 0.0004, + "robustness": 100.0 + }, + "all": { + "accuracy": 82.7, + "cost": 0.0002, + "robustness": 70.5 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 92.4, + "cost": 0.0001, + "robustness": 71.4 + }, + "medium": { + "accuracy": 72.9, + "cost": 0.0001, + "robustness": 60.0 + }, + "hard": { + "accuracy": 61.5, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 85.2, + "cost": 0.0001, + "robustness": 71.4 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 98.5, + "cost": 0.0002, + "robustness": 100.0 + }, + "medium": { + "accuracy": 86.8, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 60.0, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 92.6, + "cost": 0.0002, + "robustness": 100.0 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 89.2, + "cost": 0.0001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0003, + "robustness": 50.0 + }, + "hard": { + "accuracy": 33.3, + "cost": 0.0006, + "robustness": 100.0 + }, + "all": { + "accuracy": 64.8, + "cost": 0.0004, + "robustness": 75.0 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0001, + "robustness": 16.7 + }, + "medium": { + "accuracy": 83.3, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0002, + "robustness": 100.0 + }, + "all": { + "accuracy": 85.0, + "cost": 0.0001, + "robustness": 37.5 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 59.0 + }, + "medium": { + "accuracy": 83.3, + "cost": 0.0003, + "robustness": 57.1 + }, + "hard": { + "accuracy": 32.5, + "cost": 0.0005, + "robustness": 77.8 + }, + "all": { + "accuracy": 84.6, + "cost": 0.0002, + "robustness": 60.9 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 92.0, + "cost": 0.0001, + "robustness": 50.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0002, + "robustness": 28.6 + }, + "medium": { + "accuracy": 76.5, + "cost": 0.0002, + "robustness": 0 + }, + "hard": { + "accuracy": 33.3, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 92.3, + "cost": 0.0002, + "robustness": 28.6 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 97.1, + "cost": 0.0003, + "robustness": 50.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 85.6, + "cost": 0.0002, + "robustness": 60.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0001, + "robustness": 75.0 + }, + "medium": { + "accuracy": 74.6, + "cost": 0.0001, + "robustness": 33.3 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 89.1, + "cost": 0.0001, + "robustness": 53.3 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 99.2, + "cost": 0.0001, + "robustness": 63.6 + }, + "medium": { + "accuracy": 85.2, + "cost": 0.0003, + "robustness": 63.6 + }, + "hard": { + "accuracy": 48.3, + "cost": 0.001, + "robustness": 80.0 + }, + "all": { + "accuracy": 85.9, + "cost": 0.0003, + "robustness": 66.7 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 85.7, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 36.4, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 85.6, + "cost": 0.0002, + "robustness": 40.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 72.0, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 18.6, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 51.8, + "cost": 0.0001, + "robustness": 100.0 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 96.3, + "cost": 0.0002, + "robustness": 50.0 + }, + "medium": { + "accuracy": 76.3, + "cost": 0.0004, + "robustness": 66.7 + }, + "hard": { + "accuracy": 30.9, + "cost": 0.0007, + "robustness": 71.4 + }, + "all": { + "accuracy": 75.1, + "cost": 0.0004, + "robustness": 59.5 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 97.8, + "cost": 0.0001, + "robustness": 58.3 + }, + "medium": { + "accuracy": 82.2, + "cost": 0.0004, + "robustness": 66.7 + }, + "hard": { + "accuracy": 40.0, + "cost": 0.0008, + "robustness": 66.7 + }, + "all": { + "accuracy": 81.0, + "cost": 0.0004, + "robustness": 61.9 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 92.3, + "cost": 0.0003, + "robustness": 0.0 + }, + "medium": { + "accuracy": 68.3, + "cost": 0.0004, + "robustness": 50.0 + }, + "hard": { + "accuracy": 26.7, + "cost": 0.0004, + "robustness": 50.0 + }, + "all": { + "accuracy": 70.3, + "cost": 0.0003, + "robustness": 33.3 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 95.7, + "cost": 0.0001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 22.2, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 75.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0002, + "robustness": 0 + }, + "medium": { + "accuracy": 77.8, + "cost": 0.0006, + "robustness": 100.0 + }, + "hard": { + "accuracy": 17.6, + "cost": 0.001, + "robustness": 100.0 + }, + "all": { + "accuracy": 41.8, + "cost": 0.0008, + "robustness": 100.0 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 94.9, + "cost": 0.0001, + "robustness": 69.8 + }, + "medium": { + "accuracy": 73.6, + "cost": 0.0003, + "robustness": 76.2 + }, + "hard": { + "accuracy": 27.4, + "cost": 0.0004, + "robustness": 28.6 + }, + "all": { + "accuracy": 80.9, + "cost": 0.0002, + "robustness": 67.6 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 96.8, + "cost": 0.0002, + "robustness": 80.0 + }, + "medium": { + "accuracy": 87.4, + "cost": 0.0006, + "robustness": 76.9 + }, + "hard": { + "accuracy": 57.1, + "cost": 0.001, + "robustness": 0.0 + }, + "all": { + "accuracy": 87.2, + "cost": 0.0005, + "robustness": 73.7 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0001, + "robustness": 75.0 + }, + "medium": { + "accuracy": 63.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 + }, + "all": { + "accuracy": 89.0, + "cost": 0.0001, + "robustness": 80.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 94.3, + "cost": 0.0001, + "robustness": 67.6 + }, + "medium": { + "accuracy": 64.5, + "cost": 0.0001, + "robustness": 71.4 + }, + "hard": { + "accuracy": 16.8, + "cost": 0.0002, + "robustness": 33.3 + }, + "all": { + "accuracy": 78.2, + "cost": 0.0001, + "robustness": 63.8 + } + } + } + } + } + } + }, + "llm-router": { + "metrics": { + "easy": { + "accuracy": 95.6, + "cost": 0.0001, + "robustness": 30.8 + }, + "medium": { + "accuracy": 69.1, + "cost": 0.0002, + "robustness": 23.9 + }, + "hard": { + "accuracy": 27.0, + "cost": 0.0002, + "robustness": 34.9 + }, + "all": { + "accuracy": 71.8, + "cost": 0.0001, + "robustness": 30.0 + } + }, + "categories": { + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 92.6, + "cost": 0.0001, + "robustness": 37.5 + }, + "medium": { + "accuracy": 52.2, + "cost": 0.0003, + "robustness": 27.3 + }, + "hard": { + "accuracy": 10.2, + "cost": 0.0004, + "robustness": 20.0 + }, + "all": { + "accuracy": 60.6, + "cost": 0.0002, + "robustness": 29.2 + } + }, + "subcategories": { + "Arts": { + "metrics": { + "easy": { + "accuracy": 93.2, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 64.1, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 8.9, + "cost": 0.0004, + "robustness": 0.0 + }, + "all": { + "accuracy": 55.2, + "cost": 0.0002, + "robustness": 0.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 97.0, + "cost": 0.0001, + "robustness": 20.0 + }, + "medium": { + "accuracy": 47.4, + "cost": 0.0005, + "robustness": 0.0 + }, + "hard": { + "accuracy": 12.0, + "cost": 0.0008, + "robustness": 0.0 + }, + "all": { + "accuracy": 64.2, + "cost": 0.0003, + "robustness": 9.1 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 89.3, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 53.0, + "cost": 0.0001, + "robustness": 75.0 + }, + "hard": { + "accuracy": 10.6, + "cost": 0.0003, + "robustness": 50.0 + }, + "all": { + "accuracy": 60.5, + "cost": 0.0001, + "robustness": 75.0 + } + } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 96.2, + "cost": 0.0001, + "robustness": 23.1 + }, + "medium": { + "accuracy": 76.6, + "cost": 0.0001, + "robustness": 14.3 + }, + "hard": { + "accuracy": 28.6, + "cost": 0.0001, + "robustness": 20.0 + }, + "all": { + "accuracy": 77.9, + "cost": 0.0001, + "robustness": 19.4 + } + }, + "subcategories": { + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0001, + "robustness": 23.1 + }, + "medium": { + "accuracy": 81.4, + "cost": 0.0001, + "robustness": 11.8 + }, + "hard": { + "accuracy": 31.1, + "cost": 0.0002, + "robustness": 23.1 + }, + "all": { + "accuracy": 77.4, + "cost": 0.0001, + "robustness": 18.6 + } + } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 95.3, + "cost": 0.0001, + "robustness": 23.1 + }, + "medium": { + "accuracy": 55.3, + "cost": 0.0001, + "robustness": 25.0 + }, + "hard": { + "accuracy": 13.9, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 79.3, + "cost": 0.0001, + "robustness": 21.1 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.8, + "cost": 0.0001, + "robustness": 34.8 + }, + "medium": { + "accuracy": 75.1, + "cost": 0.0002, + "robustness": 16.7 + }, + "hard": { + "accuracy": 17.4, + "cost": 0.0002, + "robustness": 10.0 + }, + "all": { + "accuracy": 74.4, + "cost": 0.0001, + "robustness": 25.6 + } + }, + "subcategories": { + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0, + "robustness": 75.0 + }, + "medium": { + "accuracy": 77.8, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 94.2, + "cost": 0.0, + "robustness": 75.0 + } + } + }, + "Geography": { + "metrics": { + "easy": { + "accuracy": 96.3, + "cost": 0.0001, + "robustness": 12.5 + }, + "medium": { + "accuracy": 55.2, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 25.0, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 82.7, + "cost": 0.0001, + "robustness": 11.1 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0002, + "robustness": 36.4 + }, + "medium": { + "accuracy": 79.1, + "cost": 0.0002, + "robustness": 20.0 + }, + "hard": { + "accuracy": 16.2, + "cost": 0.0002, + "robustness": 10.0 + }, + "all": { + "accuracy": 69.9, + "cost": 0.0002, + "robustness": 23.1 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 88.5, + "cost": 0.0, + "robustness": 45.5 + }, + "medium": { + "accuracy": 48.5, + "cost": 0.0001, + "robustness": 66.7 + }, + "hard": { + "accuracy": 48.3, + "cost": 0.0, + "robustness": 84.0 + }, + "all": { + "accuracy": 60.3, + "cost": 0.0, + "robustness": 71.1 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 88.5, + "cost": 0.0, + "robustness": 45.5 + }, + "medium": { + "accuracy": 48.5, + "cost": 0.0001, + "robustness": 66.7 + }, + "hard": { + "accuracy": 48.3, + "cost": 0.0, + "robustness": 84.0 + }, + "all": { + "accuracy": 60.3, + "cost": 0.0, + "robustness": 71.1 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 72.5, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 31.9, + "cost": 0.0002, + "robustness": 0.0 + }, + "all": { + "accuracy": 46.7, + "cost": 0.0002, + "robustness": 0.0 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 72.5, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 31.9, + "cost": 0.0002, + "robustness": 0.0 + }, + "all": { + "accuracy": 46.7, + "cost": 0.0002, + "robustness": 0.0 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 93.6, + "cost": 0.0, + "robustness": 85.2 + }, + "medium": { + "accuracy": 66.8, + "cost": 0.0, + "robustness": 70.0 + }, + "hard": { + "accuracy": 16.0, + "cost": 0.0002, + "robustness": 57.1 + }, + "all": { + "accuracy": 76.6, + "cost": 0.0001, + "robustness": 77.3 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 90.8, + "cost": 0.0, + "robustness": 92.9 + }, + "medium": { + "accuracy": 43.5, + "cost": 0.0, + "robustness": 80.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 72.4, + "cost": 0.0, + "robustness": 90.5 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 95.4, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 92.1, + "cost": 0.0, + "robustness": 66.7 + }, + "hard": { + "accuracy": 60.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 92.6, + "cost": 0.0, + "robustness": 85.7 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 72.5, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 17.8, + "cost": 0.0003, + "robustness": 33.3 + }, + "all": { + "accuracy": 59.8, + "cost": 0.0001, + "robustness": 62.5 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 95.9, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 83.3, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0001, + "robustness": 50.0 + }, + "all": { + "accuracy": 84.5, + "cost": 0.0001, + "robustness": 50.0 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 98.6, + "cost": 0.0001, + "robustness": 7.7 + }, + "medium": { + "accuracy": 79.4, + "cost": 0.0002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 18.0, + "cost": 0.0005, + "robustness": 0.0 + }, + "all": { + "accuracy": 81.5, + "cost": 0.0002, + "robustness": 4.3 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 88.0, + "cost": 0.0001, + "robustness": 50.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0001, + "robustness": 14.3 + }, + "medium": { + "accuracy": 76.5, + "cost": 0.0002, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 91.6, + "cost": 0.0001, + "robustness": 14.3 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 95.3, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 85.3, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 23.1, + "cost": 0.0004, + "robustness": 0 + }, + "all": { + "accuracy": 81.1, + "cost": 0.0001, + "robustness": 0.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 99.6, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 59.3, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 23.1, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 89.4, + "cost": 0.0001, + "robustness": 0.0 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 98.1, + "cost": 0.0001, + "robustness": 9.1 + }, + "medium": { + "accuracy": 83.6, + "cost": 0.0002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 22.5, + "cost": 0.0008, + "robustness": 0.0 + }, + "all": { + "accuracy": 80.9, + "cost": 0.0002, + "robustness": 3.7 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0002, + "robustness": 0.0 + }, + "medium": { + "accuracy": 92.9, + "cost": 0.0002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 45.5, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 89.7, + "cost": 0.0002, + "robustness": 0.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 3.4, + "cost": 0.0002, + "robustness": 0.0 + }, + "all": { + "accuracy": 41.2, + "cost": 0.0001, + "robustness": 0.0 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 97.0, + "cost": 0.0001, + "robustness": 16.7 + }, + "medium": { + "accuracy": 67.6, + "cost": 0.0002, + "robustness": 41.7 + }, + "hard": { + "accuracy": 11.4, + "cost": 0.0002, + "robustness": 42.9 + }, + "all": { + "accuracy": 68.1, + "cost": 0.0001, + "robustness": 29.7 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 8.3 + }, + "medium": { + "accuracy": 71.2, + "cost": 0.0002, + "robustness": 66.7 + }, + "hard": { + "accuracy": 8.8, + "cost": 0.0002, + "robustness": 33.3 + }, + "all": { + "accuracy": 70.7, + "cost": 0.0001, + "robustness": 28.6 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 95.4, + "cost": 0.0002, + "robustness": 33.3 + }, + "medium": { + "accuracy": 73.0, + "cost": 0.0002, + "robustness": 25.0 + }, + "hard": { + "accuracy": 23.3, + "cost": 0.0002, + "robustness": 50.0 + }, + "all": { + "accuracy": 72.8, + "cost": 0.0002, + "robustness": 33.3 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 97.9, + "cost": 0.0, + "robustness": 33.3 + }, + "medium": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 20.0, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0, + "robustness": 50.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 0 + }, + "medium": { + "accuracy": 50.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 5.9, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 25.5, + "cost": 0.0001, + "robustness": 0.0 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 94.8, + "cost": 0.0001, + "robustness": 23.3 + }, + "medium": { + "accuracy": 67.4, + "cost": 0.0002, + "robustness": 9.5 + }, + "hard": { + "accuracy": 21.0, + "cost": 0.0002, + "robustness": 71.4 + }, + "all": { + "accuracy": 78.3, + "cost": 0.0002, + "robustness": 23.9 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 95.2, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 84.4, + "cost": 0.0003, + "robustness": 7.7 + }, + "hard": { + "accuracy": 47.6, + "cost": 0.0005, + "robustness": 0.0 + }, + "all": { + "accuracy": 83.9, + "cost": 0.0003, + "robustness": 5.3 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0, + "robustness": 50.0 + }, + "medium": { + "accuracy": 63.6, + "cost": 0.0, + "robustness": 0.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 87.7, + "cost": 0.0, + "robustness": 40.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 94.7, + "cost": 0.0001, + "robustness": 23.5 + }, + "medium": { + "accuracy": 55.8, + "cost": 0.0001, + "robustness": 14.3 + }, + "hard": { + "accuracy": 11.5, + "cost": 0.0001, + "robustness": 83.3 + }, + "all": { + "accuracy": 75.7, + "cost": 0.0001, + "robustness": 29.8 + } + } + } + } + } + } + }, + "barouter": { + "metrics": { + "easy": { + "accuracy": 95.9, + "cost": 0.0001, + "robustness": 71.2 + }, + "medium": { + "accuracy": 74.7, + "cost": 0.0004, + "robustness": 76.1 + }, + "hard": { + "accuracy": 36.0, + "cost": 0.0008, + "robustness": 56.9 + }, + "all": { + "accuracy": 75.7, + "cost": 0.0004, + "robustness": 68.8 + } + }, + "categories": { + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 95.6, + "cost": 0.0003, + "robustness": 75.0 + }, + "medium": { + "accuracy": 69.0, + "cost": 0.001, + "robustness": 90.9 + }, + "hard": { + "accuracy": 22.4, + "cost": 0.0024, + "robustness": 60.0 + }, + "all": { + "accuracy": 70.6, + "cost": 0.001, + "robustness": 79.2 + } + }, + "subcategories": { + "Arts": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 79.5, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 17.9, + "cost": 0.0016, + "robustness": 50.0 + }, + "all": { + "accuracy": 64.3, + "cost": 0.0006, + "robustness": 60.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 96.0, + "cost": 0.0002, + "robustness": 80.0 + }, + "medium": { + "accuracy": 53.4, + "cost": 0.001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 24.0, + "cost": 0.0014, + "robustness": 100.0 + }, + "all": { + "accuracy": 67.9, + "cost": 0.0007, + "robustness": 90.9 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 94.3, + "cost": 0.0004, + "robustness": 50.0 + }, + "medium": { + "accuracy": 83.0, + "cost": 0.0014, + "robustness": 100.0 + }, + "hard": { + "accuracy": 25.8, + "cost": 0.0034, + "robustness": 50.0 + }, + "all": { + "accuracy": 75.8, + "cost": 0.0014, + "robustness": 75.0 + } + } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 96.8, + "cost": 0.0001, + "robustness": 61.5 + }, + "medium": { + "accuracy": 85.9, + "cost": 0.0004, + "robustness": 71.4 + }, + "hard": { + "accuracy": 57.1, + "cost": 0.0014, + "robustness": 13.3 + }, + "all": { + "accuracy": 86.3, + "cost": 0.0004, + "robustness": 53.2 + } + }, + "subcategories": { + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0001, + "robustness": 76.9 + }, + "medium": { + "accuracy": 89.4, + "cost": 0.0005, + "robustness": 70.6 + }, + "hard": { + "accuracy": 64.6, + "cost": 0.0016, + "robustness": 15.4 + }, + "all": { + "accuracy": 87.6, + "cost": 0.0006, + "robustness": 55.8 + } + } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 95.6, + "cost": 0.0001, + "robustness": 46.2 + }, + "medium": { + "accuracy": 70.6, + "cost": 0.0001, + "robustness": 75.0 + }, + "hard": { + "accuracy": 13.9, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 82.8, + "cost": 0.0001, + "robustness": 47.4 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 98.7, + "cost": 0.0001, + "robustness": 65.2 + }, + "medium": { + "accuracy": 75.1, + "cost": 0.0002, + "robustness": 50.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0005, + "robustness": 70.0 + }, + "all": { + "accuracy": 75.0, + "cost": 0.0002, + "robustness": 64.1 + } + }, + "subcategories": { + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 100.0, + "cost": 0.0002, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 + }, + "all": { + "accuracy": 98.1, + "cost": 0.0001, + "robustness": 50.0 + } + } + }, + "Geography": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0001, + "robustness": 75.0 + }, + "medium": { + "accuracy": 69.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 41.7, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 88.7, + "cost": 0.0001, + "robustness": 66.7 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0002, + "robustness": 63.6 + }, + "medium": { + "accuracy": 74.8, + "cost": 0.0002, + "robustness": 60.0 + }, + "hard": { + "accuracy": 13.2, + "cost": 0.0006, + "robustness": 70.0 + }, + "all": { + "accuracy": 68.5, + "cost": 0.0003, + "robustness": 65.4 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 86.6, + "cost": 0.0001, + "robustness": 63.6 + }, + "medium": { + "accuracy": 41.9, + "cost": 0.0001, + "robustness": 77.8 + }, + "hard": { + "accuracy": 53.3, + "cost": 0.0003, + "robustness": 72.0 }, "all": { - "accuracy": 70.5, - "cost": 0.0009, - "robustness": 15.6 + "accuracy": 60.5, + "cost": 0.0002, + "robustness": 71.1 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 93.8, - "cost": 0.0006, - "robustness": 18.2 + "accuracy": 86.6, + "cost": 0.0001, + "robustness": 63.6 }, "medium": { - "accuracy": 58.7, - "cost": 0.0007, - "robustness": 11.1 + "accuracy": 41.9, + "cost": 0.0001, + "robustness": 77.8 }, "hard": { - "accuracy": 61.6, - "cost": 0.0012, - "robustness": 16.0 + "accuracy": 53.3, + "cost": 0.0003, + "robustness": 72.0 }, "all": { - "accuracy": 70.5, - "cost": 0.0009, - "robustness": 15.6 + "accuracy": 60.5, + "cost": 0.0002, + "robustness": 71.1 } } } @@ -17075,23 +20803,23 @@ "metrics": { "easy": { "accuracy": 99.1, - "cost": 0.0006, - "robustness": 33.3 + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 76.7, - "cost": 0.0013, - "robustness": 0.0 + "accuracy": 84.6, + "cost": 0.0002, + "robustness": 50.0 }, "hard": { - "accuracy": 32.0, - "cost": 0.0015, - "robustness": 16.7 + "accuracy": 34.7, + "cost": 0.0006, + "robustness": 50.0 }, "all": { - "accuracy": 47.1, - "cost": 0.0013, - "robustness": 17.2 + "accuracy": 49.9, + "cost": 0.0005, + "robustness": 55.2 } }, "subcategories": { @@ -17099,23 +20827,23 @@ "metrics": { "easy": { "accuracy": 99.1, - "cost": 0.0006, - "robustness": 33.3 + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 76.7, - "cost": 0.0013, - "robustness": 0.0 + "accuracy": 84.6, + "cost": 0.0002, + "robustness": 50.0 }, "hard": { - "accuracy": 32.0, - "cost": 0.0015, - "robustness": 16.7 + "accuracy": 34.7, + "cost": 0.0006, + "robustness": 50.0 }, "all": { - "accuracy": 47.1, - "cost": 0.0013, - "robustness": 17.2 + "accuracy": 49.9, + "cost": 0.0005, + "robustness": 55.2 } } } @@ -17124,120 +20852,120 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 96.1, - "cost": 0.0007, - "robustness": 33.3 + "accuracy": 95.1, + "cost": 0.0001, + "robustness": 70.4 }, "medium": { - "accuracy": 66.4, - "cost": 0.001, - "robustness": 20.0 + "accuracy": 70.1, + "cost": 0.0002, + "robustness": 80.0 }, "hard": { - "accuracy": 27.2, - "cost": 0.0013, - "robustness": 42.9 + "accuracy": 19.8, + "cost": 0.0006, + "robustness": 57.1 }, "all": { - "accuracy": 79.1, - "cost": 0.0009, - "robustness": 31.8 + "accuracy": 78.9, + "cost": 0.0002, + "robustness": 70.5 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 94.1, - "cost": 0.0006, - "robustness": 28.6 + "accuracy": 92.4, + "cost": 0.0001, + "robustness": 64.3 }, "medium": { - "accuracy": 40.0, - "cost": 0.0008, - "robustness": 20.0 + "accuracy": 44.7, + "cost": 0.0001, + "robustness": 80.0 }, "hard": { - "accuracy": 15.4, - "cost": 0.0006, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 0.0 }, "all": { - "accuracy": 74.2, - "cost": 0.0006, - "robustness": 23.8 + "accuracy": 73.9, + "cost": 0.0001, + "robustness": 61.9 } } }, "Philosophical logic": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0009, - "robustness": 0.0 + "accuracy": 95.4, + "cost": 0.0001, + "robustness": 75.0 }, "medium": { - "accuracy": 89.5, - "cost": 0.0012, - "robustness": 33.3 + "accuracy": 92.1, + "cost": 0.0003, + "robustness": 66.7 }, "hard": { - "accuracy": 60.0, - "cost": 0.0027, + "accuracy": 40.0, + "cost": 0.0005, "robustness": 0 }, "all": { - "accuracy": 94.4, - "cost": 0.0011, - "robustness": 14.3 + "accuracy": 91.7, + "cost": 0.0002, + "robustness": 71.4 } } }, "Philosophy": { "metrics": { "easy": { - "accuracy": 89.2, - "cost": 0.001, + "accuracy": 97.3, + "cost": 0.0001, "robustness": 66.7 }, "medium": { - "accuracy": 77.5, - "cost": 0.0011, - "robustness": 0.0 + "accuracy": 82.5, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 28.9, - "cost": 0.0016, + "accuracy": 24.4, + "cost": 0.0008, "robustness": 66.7 }, "all": { - "accuracy": 63.1, - "cost": 0.0013, - "robustness": 50.0 + "accuracy": 65.6, + "cost": 0.0004, + "robustness": 75.0 } } }, "Psychology": { "metrics": { "easy": { - "accuracy": 99.2, - "cost": 0.0007, - "robustness": 50.0 + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 83.3 }, "medium": { - "accuracy": 85.4, - "cost": 0.001, + "accuracy": 87.5, + "cost": 0.0002, "robustness": 0 }, "hard": { - "accuracy": 22.2, - "cost": 0.0007, - "robustness": 50.0 + "accuracy": 16.7, + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 88.2, - "cost": 0.0008, - "robustness": 50.0 + "accuracy": 87.7, + "cost": 0.0001, + "robustness": 87.5 } } } @@ -17246,24 +20974,24 @@ "Science": { "metrics": { "easy": { - "accuracy": 98.2, - "cost": 0.0006, - "robustness": 10.3 + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 66.7 }, "medium": { - "accuracy": 81.7, - "cost": 0.0012, - "robustness": 14.3 + "accuracy": 83.6, + "cost": 0.0003, + "robustness": 81.0 }, "hard": { - "accuracy": 28.4, - "cost": 0.0021, - "robustness": 44.4 + "accuracy": 37.1, + "cost": 0.0009, + "robustness": 77.8 }, "all": { - "accuracy": 83.4, - "cost": 0.001, - "robustness": 15.9 + "accuracy": 85.3, + "cost": 0.0003, + "robustness": 72.5 } }, "subcategories": { @@ -17271,12 +20999,12 @@ "metrics": { "easy": { "accuracy": 95.0, - "cost": 0.0003, - "robustness": 0.0 + "cost": 0.0001, + "robustness": 50.0 }, "medium": { - "accuracy": 60.0, - "cost": 0.0011, + "accuracy": 80.0, + "cost": 0.0001, "robustness": 0 }, "hard": { @@ -17285,57 +21013,57 @@ "robustness": 0 }, "all": { - "accuracy": 88.0, - "cost": 0.0004, - "robustness": 0.0 + "accuracy": 92.0, + "cost": 0.0001, + "robustness": 50.0 } } }, "Biology": { "metrics": { "easy": { - "accuracy": 97.4, - "cost": 0.0006, - "robustness": 14.3 + "accuracy": 100.0, + "cost": 0.0002, + "robustness": 71.4 }, "medium": { - "accuracy": 82.4, - "cost": 0.0007, + "accuracy": 76.5, + "cost": 0.0002, "robustness": 0 }, "hard": { - "accuracy": 22.2, - "cost": 0.0011, + "accuracy": 33.3, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 90.9, - "cost": 0.0006, - "robustness": 14.3 + "accuracy": 93.0, + "cost": 0.0002, + "robustness": 71.4 } } }, "Chemistry": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0008, - "robustness": 33.3 + "accuracy": 97.7, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { "accuracy": 94.1, - "cost": 0.0015, - "robustness": 0.0 + "cost": 0.0003, + "robustness": 50.0 }, "hard": { "accuracy": 30.8, - "cost": 0.001, + "cost": 0.0006, "robustness": 0 }, "all": { - "accuracy": 87.8, - "cost": 0.0011, - "robustness": 20.0 + "accuracy": 86.7, + "cost": 0.0003, + "robustness": 80.0 } } }, @@ -17343,23 +21071,23 @@ "metrics": { "easy": { "accuracy": 98.1, - "cost": 0.0005, - "robustness": 0.0 + "cost": 0.0001, + "robustness": 25.0 }, "medium": { "accuracy": 74.6, - "cost": 0.0009, - "robustness": 0.0 + "cost": 0.0002, + "robustness": 66.7 }, "hard": { - "accuracy": 23.1, - "cost": 0.0005, + "accuracy": 7.7, + "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 90.9, - "cost": 0.0006, - "robustness": 6.7 + "accuracy": 90.3, + "cost": 0.0001, + "robustness": 46.7 } } }, @@ -17367,71 +21095,71 @@ "metrics": { "easy": { "accuracy": 98.9, - "cost": 0.0007, - "robustness": 18.2 + "cost": 0.0002, + "robustness": 81.8 }, "medium": { - "accuracy": 84.0, - "cost": 0.0012, - "robustness": 27.3 + "accuracy": 86.4, + "cost": 0.0004, + "robustness": 90.9 }, "hard": { - "accuracy": 38.2, - "cost": 0.003, + "accuracy": 58.4, + "cost": 0.0012, "robustness": 60.0 }, "all": { - "accuracy": 83.7, - "cost": 0.0013, - "robustness": 29.6 + "accuracy": 87.7, + "cost": 0.0004, + "robustness": 81.5 } } }, "Physics": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0005, - "robustness": 0.0 + "accuracy": 93.2, + "cost": 0.0002, + "robustness": 100.0 }, "medium": { - "accuracy": 76.2, - "cost": 0.0015, - "robustness": 0.0 + "accuracy": 88.1, + "cost": 0.0005, + "robustness": 100.0 }, "hard": { - "accuracy": 63.6, - "cost": 0.0021, + "accuracy": 45.5, + "cost": 0.0006, "robustness": 0 }, "all": { - "accuracy": 84.5, - "cost": 0.0011, - "robustness": 0.0 + "accuracy": 85.6, + "cost": 0.0004, + "robustness": 100.0 } } }, "Science": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0005, - "robustness": 0.0 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 60.0 }, "medium": { - "accuracy": 72.0, - "cost": 0.0015, + "accuracy": 60.0, + "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 8.5, - "cost": 0.0015, - "robustness": 0.0 + "accuracy": 11.9, + "cost": 0.0008, + "robustness": 100.0 }, "all": { "accuracy": 45.6, - "cost": 0.0012, - "robustness": 0.0 + "cost": 0.0004, + "robustness": 75.0 } } } @@ -17440,72 +21168,72 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 95.0, - "cost": 0.0008, - "robustness": 27.8 + "accuracy": 96.0, + "cost": 0.0001, + "robustness": 77.8 }, "medium": { - "accuracy": 77.1, - "cost": 0.0015, - "robustness": 8.3 + "accuracy": 74.7, + "cost": 0.0003, + "robustness": 50.0 }, "hard": { "accuracy": 18.1, - "cost": 0.0016, - "robustness": 42.9 + "cost": 0.0005, + "robustness": 57.1 }, "all": { - "accuracy": 72.1, - "cost": 0.0012, - "robustness": 24.3 + "accuracy": 71.7, + "cost": 0.0003, + "robustness": 64.9 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 96.2, - "cost": 0.0008, - "robustness": 25.0 + "accuracy": 96.7, + "cost": 0.0001, + "robustness": 75.0 }, "medium": { - "accuracy": 79.8, - "cost": 0.0013, - "robustness": 16.7 + "accuracy": 77.3, + "cost": 0.0003, + "robustness": 33.3 }, "hard": { - "accuracy": 18.8, - "cost": 0.0019, - "robustness": 0.0 + "accuracy": 21.2, + "cost": 0.0007, + "robustness": 33.3 }, "all": { - "accuracy": 75.4, - "cost": 0.0012, - "robustness": 19.0 + "accuracy": 75.1, + "cost": 0.0003, + "robustness": 57.1 } } }, "Law": { "metrics": { "easy": { - "accuracy": 92.3, - "cost": 0.0013, - "robustness": 33.3 + "accuracy": 93.8, + "cost": 0.0002, + "robustness": 100.0 }, "medium": { - "accuracy": 77.8, - "cost": 0.002, - "robustness": 0.0 + "accuracy": 76.2, + "cost": 0.0004, + "robustness": 75.0 }, "hard": { "accuracy": 20.0, - "cost": 0.001, - "robustness": 100.0 + "cost": 0.0003, + "robustness": 50.0 }, "all": { "accuracy": 72.8, - "cost": 0.0015, - "robustness": 33.3 + "cost": 0.0003, + "robustness": 77.8 } } }, @@ -17513,47 +21241,47 @@ "metrics": { "easy": { "accuracy": 95.7, - "cost": 0.0003, - "robustness": 33.3 + "cost": 0.0, + "robustness": 66.7 }, "medium": { - "accuracy": 44.4, - "cost": 0.0007, + "accuracy": 22.2, + "cost": 0.0001, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0002, - "robustness": 0.0 + "cost": 0.0001, + "robustness": 100.0 }, "all": { - "accuracy": 80.3, - "cost": 0.0004, - "robustness": 25.0 + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 75.0 } } }, "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 66.7, + "accuracy": 100.0, "cost": 0.0001, "robustness": 0 }, "medium": { - "accuracy": 66.7, - "cost": 0.0016, - "robustness": 0.0 + "accuracy": 72.2, + "cost": 0.0001, + "robustness": 50.0 }, "hard": { - "accuracy": 17.6, - "cost": 0.0017, + "accuracy": 11.8, + "cost": 0.0002, "robustness": 100.0 }, "all": { "accuracy": 36.4, - "cost": 0.0016, - "robustness": 33.3 + "cost": 0.0002, + "robustness": 66.7 } } } @@ -17562,24 +21290,24 @@ "Technology": { "metrics": { "easy": { - "accuracy": 96.6, - "cost": 0.0006, - "robustness": 25.6 + "accuracy": 94.0, + "cost": 0.0001, + "robustness": 81.4 }, "medium": { - "accuracy": 75.0, - "cost": 0.0011, - "robustness": 19.0 + "accuracy": 70.2, + "cost": 0.0004, + "robustness": 90.5 }, "hard": { - "accuracy": 24.2, - "cost": 0.0013, - "robustness": 28.6 + "accuracy": 28.0, + "cost": 0.0005, + "robustness": 71.4 }, "all": { - "accuracy": 82.0, - "cost": 0.0009, - "robustness": 23.9 + "accuracy": 79.5, + "cost": 0.0002, + "robustness": 83.1 } }, "subcategories": { @@ -17587,36 +21315,36 @@ "metrics": { "easy": { "accuracy": 97.6, - "cost": 0.0009, - "robustness": 0.0 + "cost": 0.0002, + "robustness": 80.0 }, "medium": { - "accuracy": 88.0, - "cost": 0.0013, - "robustness": 15.4 + "accuracy": 90.4, + "cost": 0.0007, + "robustness": 100.0 }, "hard": { - "accuracy": 35.7, - "cost": 0.0021, + "accuracy": 59.5, + "cost": 0.0008, "robustness": 0.0 }, "all": { - "accuracy": 85.1, - "cost": 0.0013, - "robustness": 10.5 + "accuracy": 89.3, + "cost": 0.0005, + "robustness": 89.5 } } }, "Management and public relations": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0006, - "robustness": 25.0 + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 75.0 }, "medium": { - "accuracy": 63.6, - "cost": 0.0005, + "accuracy": 54.5, + "cost": 0.0001, "robustness": 0.0 }, "hard": { @@ -17626,32 +21354,32 @@ }, "all": { "accuracy": 89.0, - "cost": 0.0006, - "robustness": 20.0 + "cost": 0.0001, + "robustness": 60.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 96.4, - "cost": 0.0006, - "robustness": 29.4 + "accuracy": 92.9, + "cost": 0.0001, + "robustness": 82.4 }, "medium": { - "accuracy": 66.5, - "cost": 0.0011, - "robustness": 28.6 + "accuracy": 57.0, + "cost": 0.0002, + "robustness": 85.7 }, "hard": { - "accuracy": 20.4, - "cost": 0.001, - "robustness": 33.3 + "accuracy": 16.8, + "cost": 0.0003, + "robustness": 83.3 }, "all": { - "accuracy": 80.4, - "cost": 0.0007, - "robustness": 29.8 + "accuracy": 75.5, + "cost": 0.0002, + "robustness": 83.0 } } } @@ -17659,74 +21387,74 @@ } } }, - "hybrid-router": { + "lynkr": { "metrics": { "easy": { - "accuracy": 96.2, - "cost": 0.0, + "accuracy": 93.5, + "cost": 0.0001, "robustness": 96.0 }, "medium": { - "accuracy": 67.7, - "cost": 0.0, - "robustness": 96.5 + "accuracy": 60.9, + "cost": 0.0003, + "robustness": 89.4 }, "hard": { - "accuracy": 25.7, - "cost": 0.0001, - "robustness": 98.2 + "accuracy": 26.7, + "cost": 0.0005, + "robustness": 89.0 }, "all": { - "accuracy": 71.4, - "cost": 0.0, - "robustness": 96.7 + "accuracy": 68.4, + "cost": 0.0003, + "robustness": 92.4 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 94.3, - "cost": 0.0, + "accuracy": 90.6, + "cost": 0.0003, "robustness": 100.0 }, "medium": { - "accuracy": 57.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 42.4, + "cost": 0.0005, + "robustness": 90.9 }, "hard": { - "accuracy": 7.5, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 3.4, + "cost": 0.0004, + "robustness": 80.0 }, "all": { - "accuracy": 62.6, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 54.7, + "cost": 0.0004, + "robustness": 91.7 } }, "subcategories": { "Arts": { "metrics": { "easy": { - "accuracy": 98.3, - "cost": 0.0, + "accuracy": 96.6, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 74.4, - "cost": 0.0, + "accuracy": 41.0, + "cost": 0.0003, "robustness": 100.0 }, "hard": { - "accuracy": 5.4, - "cost": 0.0, + "accuracy": 1.8, + "cost": 0.0004, "robustness": 100.0 }, "all": { - "accuracy": 58.4, - "cost": 0.0, + "accuracy": 48.1, + "cost": 0.0003, "robustness": 100.0 } } @@ -17734,23 +21462,23 @@ "Music": { "metrics": { "easy": { - "accuracy": 96.0, - "cost": 0.0, + "accuracy": 90.9, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 50.9, - "cost": 0.0001, + "accuracy": 38.8, + "cost": 0.0005, "robustness": 100.0 }, "hard": { - "accuracy": 12.0, - "cost": 0.0, + "accuracy": 8.0, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 65.4, - "cost": 0.0, + "accuracy": 57.1, + "cost": 0.0003, "robustness": 100.0 } } @@ -17758,24 +21486,24 @@ "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 91.4, - "cost": 0.0, + "accuracy": 87.9, + "cost": 0.0004, "robustness": 100.0 }, "medium": { - "accuracy": 58.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 47.0, + "cost": 0.0006, + "robustness": 75.0 }, "hard": { - "accuracy": 7.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 3.0, + "cost": 0.0004, + "robustness": 50.0 }, "all": { - "accuracy": 62.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 56.2, + "cost": 0.0005, + "robustness": 75.0 } } } @@ -17784,72 +21512,72 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 97.5, - "cost": 0.0, - "robustness": 92.3 + "accuracy": 95.5, + "cost": 0.0001, + "robustness": 96.2 }, "medium": { - "accuracy": 80.3, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 77.7, + "cost": 0.0006, + "robustness": 85.7 }, "hard": { - "accuracy": 33.5, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 51.8, + "cost": 0.0019, + "robustness": 53.3 }, "all": { - "accuracy": 80.6, - "cost": 0.0001, - "robustness": 96.8 + "accuracy": 82.0, + "cost": 0.0006, + "robustness": 82.3 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 98.6, - "cost": 0.0, - "robustness": 84.6 + "accuracy": 96.2, + "cost": 0.0002, + "robustness": 92.3 }, "medium": { - "accuracy": 82.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 83.8, + "cost": 0.0006, + "robustness": 88.2 }, "hard": { - "accuracy": 37.8, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 59.3, + "cost": 0.0022, + "robustness": 46.2 }, "all": { - "accuracy": 80.0, - "cost": 0.0001, - "robustness": 95.3 + "accuracy": 83.9, + "cost": 0.0008, + "robustness": 76.7 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 96.0, - "cost": 0.0, + "accuracy": 94.5, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 69.4, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 50.6, + "cost": 0.0001, + "robustness": 75.0 }, "hard": { "accuracy": 8.3, - "cost": 0.0, + "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 82.3, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 77.3, + "cost": 0.0001, + "robustness": 94.7 } } } @@ -17858,47 +21586,47 @@ "History": { "metrics": { "easy": { - "accuracy": 96.5, - "cost": 0.0, - "robustness": 95.7 + "accuracy": 93.3, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 62.1, - "cost": 0.0, - "robustness": 83.3 + "accuracy": 46.3, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 9.4, - "cost": 0.0, + "accuracy": 4.0, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 69.3, - "cost": 0.0, - "robustness": 94.9 + "accuracy": 62.4, + "cost": 0.0002, + "robustness": 100.0 } }, "subcategories": { "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 92.9, - "cost": 0.0, + "accuracy": 95.2, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 66.7, - "cost": 0.0, + "accuracy": 33.3, + "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 100.0, - "cost": 0.0, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 88.5, - "cost": 0.0, + "accuracy": 82.7, + "cost": 0.0001, "robustness": 100.0 } } @@ -17906,23 +21634,23 @@ "Geography": { "metrics": { "easy": { - "accuracy": 96.3, - "cost": 0.0, + "accuracy": 95.4, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 48.3, - "cost": 0.0, + "accuracy": 44.8, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 25.0, - "cost": 0.0, + "accuracy": 0.0, + "cost": 0.0004, "robustness": 0 }, "all": { - "accuracy": 81.3, - "cost": 0.0, + "accuracy": 78.0, + "cost": 0.0001, "robustness": 100.0 } } @@ -17930,24 +21658,24 @@ "History": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0, - "robustness": 90.9 + "accuracy": 91.9, + "cost": 0.0002, + "robustness": 100.0 }, "medium": { - "accuracy": 64.7, - "cost": 0.0, - "robustness": 80.0 + "accuracy": 47.5, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 7.4, - "cost": 0.0, + "accuracy": 4.4, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 63.7, - "cost": 0.0, - "robustness": 92.3 + "accuracy": 55.6, + "cost": 0.0002, + "robustness": 100.0 } } } @@ -17956,48 +21684,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 91.4, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 90.9 }, "medium": { - "accuracy": 50.3, - "cost": 0.0, + "accuracy": 35.3, + "cost": 0.0002, "robustness": 100.0 }, - "hard": { - "accuracy": 48.6, - "cost": 0.0, + "hard": { + "accuracy": 49.4, + "cost": 0.0002, "robustness": 96.0 }, "all": { - "accuracy": 61.8, - "cost": 0.0, - "robustness": 97.8 + "accuracy": 54.3, + "cost": 0.0002, + "robustness": 95.6 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 91.4, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 90.9 }, "medium": { - "accuracy": 50.3, - "cost": 0.0, + "accuracy": 35.3, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 48.6, - "cost": 0.0, + "accuracy": 49.4, + "cost": 0.0002, "robustness": 96.0 }, "all": { - "accuracy": 61.8, - "cost": 0.0, - "robustness": 97.8 + "accuracy": 54.3, + "cost": 0.0002, + "robustness": 95.6 } } } @@ -18006,48 +21734,48 @@ "Literature": { "metrics": { "easy": { - "accuracy": 96.9, - "cost": 0.0, + "accuracy": 94.2, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 65.0, - "cost": 0.0, + "accuracy": 28.2, + "cost": 0.0004, "robustness": 100.0 }, "hard": { - "accuracy": 28.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 24.8, + "cost": 0.0003, + "robustness": 95.8 }, "all": { - "accuracy": 43.2, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 36.3, + "cost": 0.0003, + "robustness": 96.6 } }, "subcategories": { "Literature, rhetoric and criticism": { "metrics": { "easy": { - "accuracy": 96.9, - "cost": 0.0, + "accuracy": 94.2, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 65.0, - "cost": 0.0, + "accuracy": 28.2, + "cost": 0.0004, "robustness": 100.0 }, "hard": { - "accuracy": 28.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 24.8, + "cost": 0.0003, + "robustness": 95.8 }, "all": { - "accuracy": 43.2, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 36.3, + "cost": 0.0003, + "robustness": 96.6 } } } @@ -18056,72 +21784,72 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 96.3, - "cost": 0.0, + "accuracy": 94.6, + "cost": 0.0001, "robustness": 96.3 }, "medium": { - "accuracy": 62.1, - "cost": 0.0, + "accuracy": 58.3, + "cost": 0.0002, "robustness": 80.0 }, "hard": { - "accuracy": 14.8, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 7.4, + "cost": 0.0003, + "robustness": 85.7 }, "all": { - "accuracy": 76.6, - "cost": 0.0, - "robustness": 93.2 + "accuracy": 73.6, + "cost": 0.0002, + "robustness": 90.9 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 94.6, - "cost": 0.0, - "robustness": 92.9 + "accuracy": 93.0, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 37.6, - "cost": 0.0, - "robustness": 60.0 + "accuracy": 31.8, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 7.7, - "cost": 0.0, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 73.5, - "cost": 0.0, - "robustness": 85.7 + "accuracy": 70.3, + "cost": 0.0001, + "robustness": 100.0 } } }, "Philosophical logic": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0, + "accuracy": 98.5, + "cost": 0.0002, "robustness": 100.0 }, "medium": { - "accuracy": 84.2, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 92.1, + "cost": 0.0003, + "robustness": 66.7 }, "hard": { "accuracy": 80.0, - "cost": 0.0, + "cost": 0.0006, "robustness": 0 }, "all": { - "accuracy": 93.5, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 95.4, + "cost": 0.0002, + "robustness": 85.7 } } }, @@ -18129,47 +21857,47 @@ "metrics": { "easy": { "accuracy": 94.6, - "cost": 0.0, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 67.5, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 57.5, + "cost": 0.0003, + "robustness": 50.0 }, "hard": { - "accuracy": 8.9, - "cost": 0.0, + "accuracy": 4.4, + "cost": 0.0004, "robustness": 100.0 }, "all": { - "accuracy": 54.1, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 49.2, + "cost": 0.0003, + "robustness": 87.5 } } }, "Psychology": { "metrics": { "easy": { - "accuracy": 97.5, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 95.0, + "cost": 0.0001, + "robustness": 83.3 }, "medium": { - "accuracy": 83.3, - "cost": 0.0, + "accuracy": 79.2, + "cost": 0.0002, "robustness": 0 }, "hard": { - "accuracy": 16.7, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 50.0 }, "all": { - "accuracy": 86.1, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 81.8, + "cost": 0.0002, + "robustness": 75.0 } } } @@ -18178,23 +21906,23 @@ "Science": { "metrics": { "easy": { - "accuracy": 97.9, - "cost": 0.0, - "robustness": 97.4 + "accuracy": 97.0, + "cost": 0.0001, + "robustness": 94.9 }, "medium": { - "accuracy": 75.9, - "cost": 0.0, + "accuracy": 72.5, + "cost": 0.0003, "robustness": 100.0 }, "hard": { - "accuracy": 21.1, - "cost": 0.0001, - "robustness": 88.9 + "accuracy": 25.3, + "cost": 0.0006, + "robustness": 100.0 }, "all": { - "accuracy": 80.5, - "cost": 0.0, + "accuracy": 79.5, + "cost": 0.0002, "robustness": 97.1 } }, @@ -18203,12 +21931,12 @@ "metrics": { "easy": { "accuracy": 95.0, - "cost": 0.0, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 80.0, - "cost": 0.0, + "accuracy": 60.0, + "cost": 0.0001, "robustness": 0 }, "hard": { @@ -18217,8 +21945,8 @@ "robustness": 0 }, "all": { - "accuracy": 92.0, - "cost": 0.0, + "accuracy": 88.0, + "cost": 0.0001, "robustness": 100.0 } } @@ -18226,47 +21954,47 @@ "Biology": { "metrics": { "easy": { - "accuracy": 98.3, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 99.1, + "cost": 0.0001, + "robustness": 71.4 }, "medium": { - "accuracy": 64.7, - "cost": 0.0, + "accuracy": 70.6, + "cost": 0.0002, "robustness": 0 }, "hard": { - "accuracy": 11.1, - "cost": 0.0, + "accuracy": 22.2, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 88.8, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 90.9, + "cost": 0.0002, + "robustness": 71.4 } } }, "Chemistry": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0, + "accuracy": 100.0, + "cost": 0.0002, "robustness": 100.0 }, "medium": { - "accuracy": 82.4, - "cost": 0.0, + "accuracy": 76.5, + "cost": 0.0003, "robustness": 100.0 }, "hard": { "accuracy": 23.1, - "cost": 0.0, + "cost": 0.0003, "robustness": 0 }, "all": { - "accuracy": 81.1, - "cost": 0.0, + "accuracy": 80.0, + "cost": 0.0002, "robustness": 100.0 } } @@ -18274,47 +22002,47 @@ "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0, + "accuracy": 96.5, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 72.9, - "cost": 0.0, + "accuracy": 59.3, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 7.7, + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 89.4, - "cost": 0.0, - "robustness": 93.3 + "accuracy": 86.4, + "cost": 0.0001, + "robustness": 100.0 } } }, "Mathematics": { "metrics": { - "easy": { - "accuracy": 99.2, - "cost": 0.0, + "easy": { + "accuracy": 96.2, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 79.6, - "cost": 0.0001, + "accuracy": 78.8, + "cost": 0.0004, "robustness": 100.0 }, "hard": { - "accuracy": 37.1, - "cost": 0.0002, + "accuracy": 43.8, + "cost": 0.0011, "robustness": 100.0 }, "all": { - "accuracy": 81.9, - "cost": 0.0001, + "accuracy": 81.2, + "cost": 0.0004, "robustness": 100.0 } } @@ -18323,22 +22051,22 @@ "metrics": { "easy": { "accuracy": 95.5, - "cost": 0.0, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 78.6, - "cost": 0.0, + "accuracy": 69.0, + "cost": 0.0003, "robustness": 100.0 }, "hard": { - "accuracy": 27.3, - "cost": 0.0, + "accuracy": 18.2, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 80.4, - "cost": 0.0, + "accuracy": 75.3, + "cost": 0.0002, "robustness": 100.0 } } @@ -18346,24 +22074,24 @@ "Science": { "metrics": { "easy": { - "accuracy": 93.3, - "cost": 0.0, - "robustness": 80.0 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 40.0, - "cost": 0.0, + "accuracy": 44.0, + "cost": 0.0002, "robustness": 0 }, "hard": { - "accuracy": 1.7, - "cost": 0.0, + "accuracy": 3.4, + "cost": 0.0002, "robustness": 100.0 }, "all": { - "accuracy": 34.2, - "cost": 0.0, - "robustness": 87.5 + "accuracy": 37.7, + "cost": 0.0002, + "robustness": 100.0 } } } @@ -18372,95 +22100,95 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 95.3, - "cost": 0.0, + "accuracy": 94.0, + "cost": 0.0002, "robustness": 100.0 }, "medium": { - "accuracy": 62.5, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 64.8, + "cost": 0.0003, + "robustness": 66.7 }, "hard": { "accuracy": 7.4, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.0003, + "robustness": 85.7 }, "all": { - "accuracy": 64.7, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 65.0, + "cost": 0.0002, + "robustness": 86.5 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0, + "accuracy": 96.2, + "cost": 0.0002, "robustness": 100.0 }, "medium": { - "accuracy": 68.7, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 72.4, + "cost": 0.0003, + "robustness": 66.7 }, "hard": { - "accuracy": 6.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 10.0, + "cost": 0.0003, + "robustness": 66.7 }, "all": { - "accuracy": 69.2, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 70.9, + "cost": 0.0002, + "robustness": 85.7 } } }, "Law": { "metrics": { "easy": { - "accuracy": 90.8, - "cost": 0.0, + "accuracy": 86.2, + "cost": 0.0002, "robustness": 100.0 }, "medium": { - "accuracy": 60.3, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 52.4, + "cost": 0.0003, + "robustness": 50.0 }, "hard": { - "accuracy": 10.0, - "cost": 0.0, + "accuracy": 6.7, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 63.3, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 57.6, + "cost": 0.0002, + "robustness": 77.8 } } }, "Social problems": { "metrics": { "easy": { - "accuracy": 93.6, - "cost": 0.0, + "accuracy": 97.9, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 22.2, - "cost": 0.0, + "accuracy": 66.7, + "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 20.0, - "cost": 0.0, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 77.0, - "cost": 0.0, + "accuracy": 85.2, + "cost": 0.0001, "robustness": 100.0 } } @@ -18468,23 +22196,23 @@ "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0, + "accuracy": 66.7, + "cost": 0.0001, "robustness": 0 }, "medium": { - "accuracy": 33.3, - "cost": 0.0, + "accuracy": 38.9, + "cost": 0.0003, "robustness": 100.0 }, "hard": { - "accuracy": 5.9, - "cost": 0.0, + "accuracy": 2.9, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 20.0, - "cost": 0.0, + "accuracy": 18.2, + "cost": 0.0003, "robustness": 100.0 } } @@ -18494,24 +22222,24 @@ "Technology": { "metrics": { "easy": { - "accuracy": 95.5, - "cost": 0.0, + "accuracy": 93.2, + "cost": 0.0001, "robustness": 93.0 }, "medium": { - "accuracy": 67.6, - "cost": 0.0, - "robustness": 95.2 + "accuracy": 62.6, + "cost": 0.0003, + "robustness": 90.5 }, "hard": { - "accuracy": 17.8, - "cost": 0.0, + "accuracy": 19.7, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 78.4, - "cost": 0.0, - "robustness": 94.4 + "accuracy": 75.8, + "cost": 0.0002, + "robustness": 93.0 } }, "subcategories": { @@ -18519,22 +22247,22 @@ "metrics": { "easy": { "accuracy": 96.0, - "cost": 0.0, - "robustness": 80.0 + "cost": 0.0002, + "robustness": 100.0 }, "medium": { - "accuracy": 80.8, - "cost": 0.0001, - "robustness": 92.3 + "accuracy": 79.0, + "cost": 0.0005, + "robustness": 84.6 }, "hard": { - "accuracy": 35.7, - "cost": 0.0001, + "accuracy": 40.5, + "cost": 0.0007, "robustness": 100.0 }, "all": { - "accuracy": 80.9, - "cost": 0.0001, + "accuracy": 80.6, + "cost": 0.0004, "robustness": 89.5 } } @@ -18542,47 +22270,47 @@ "Management and public relations": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 96.7, + "cost": 0.0001, + "robustness": 75.0 }, "medium": { - "accuracy": 63.6, - "cost": 0.0, + "accuracy": 36.4, + "cost": 0.0001, "robustness": 100.0 }, "hard": { "accuracy": 0.0, - "cost": 0.0, + "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 91.8, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 84.9, + "cost": 0.0001, + "robustness": 80.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 95.0, - "cost": 0.0, + "accuracy": 92.3, + "cost": 0.0001, "robustness": 94.1 }, "medium": { - "accuracy": 58.7, - "cost": 0.0, + "accuracy": 52.5, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 11.5, - "cost": 0.0, + "accuracy": 12.4, + "cost": 0.0002, "robustness": 100.0 }, "all": { - "accuracy": 76.6, - "cost": 0.0, + "accuracy": 73.5, + "cost": 0.0001, "robustness": 95.7 } } @@ -18591,123 +22319,123 @@ } } }, - "cross-router": { + "nadir-caliper": { "metrics": { "easy": { - "accuracy": 96.5, - "cost": 0.0001, - "robustness": 59.1 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 89.4 }, "medium": { - "accuracy": 78.3, - "cost": 0.0003, - "robustness": 69.9 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 72.6 }, "hard": { - "accuracy": 41.0, - "cost": 0.0006, - "robustness": 78.9 + "accuracy": 0.0, + "cost": 0.0007, + "robustness": 69.7 }, "all": { - "accuracy": 78.2, - "cost": 0.0003, - "robustness": 67.1 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 79.8 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 95.3, - "cost": 0.0002, - "robustness": 50.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 75.0 }, "medium": { - "accuracy": 67.5, - "cost": 0.0006, - "robustness": 72.7 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 54.5 }, "hard": { - "accuracy": 23.8, - "cost": 0.0014, - "robustness": 80.0 + "accuracy": 0.0, + "cost": 0.0007, + "robustness": 60.0 }, - "all": { - "accuracy": 70.1, - "cost": 0.0006, - "robustness": 66.7 + "all": { + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 62.5 } }, "subcategories": { "Arts": { "metrics": { "easy": { - "accuracy": 98.3, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0, "robustness": 100.0 }, "medium": { - "accuracy": 79.5, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 50.0 }, "hard": { - "accuracy": 28.6, - "cost": 0.0021, + "accuracy": 0.0, + "cost": 0.0005, "robustness": 100.0 }, "all": { - "accuracy": 68.2, - "cost": 0.0009, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 80.0 } } }, "Music": { "metrics": { "easy": { - "accuracy": 92.9, - "cost": 0.0002, - "robustness": 40.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 80.0 }, "medium": { - "accuracy": 61.2, - "cost": 0.0008, + "accuracy": 0.0, + "cost": 0.0002, "robustness": 40.0 }, "hard": { - "accuracy": 12.0, - "cost": 0.0013, + "accuracy": 0.0, + "cost": 0.0002, "robustness": 0.0 }, "all": { - "accuracy": 69.2, - "cost": 0.0006, - "robustness": 36.4 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 54.5 } } }, "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 95.7, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0, "robustness": 50.0 }, "medium": { - "accuracy": 70.0, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 75.0 }, "hard": { - "accuracy": 24.2, - "cost": 0.0008, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0012, + "robustness": 50.0 }, "all": { - "accuracy": 71.9, + "accuracy": 0.0, "cost": 0.0004, - "robustness": 87.5 + "robustness": 62.5 } } } @@ -18716,72 +22444,72 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0001, - "robustness": 53.8 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 84.4, + "accuracy": 0.0, "cost": 0.0003, - "robustness": 52.4 + "robustness": 71.4 }, "hard": { - "accuracy": 54.3, - "cost": 0.0007, - "robustness": 60.0 + "accuracy": 0.0, + "cost": 0.0009, + "robustness": 33.3 }, "all": { - "accuracy": 85.5, + "accuracy": 0.0, "cost": 0.0003, - "robustness": 54.8 + "robustness": 74.2 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 97.8, + "accuracy": 0.0, "cost": 0.0001, - "robustness": 30.8 + "robustness": 100.0 }, "medium": { - "accuracy": 89.9, - "cost": 0.0004, - "robustness": 64.7 + "accuracy": 0.0, + "cost": 0.0003, + "robustness": 70.6 }, "hard": { - "accuracy": 62.2, - "cost": 0.0008, - "robustness": 53.8 + "accuracy": 0.0, + "cost": 0.0011, + "robustness": 38.5 }, "all": { - "accuracy": 87.5, + "accuracy": 0.0, "cost": 0.0004, - "robustness": 51.2 + "robustness": 69.8 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 96.4, - "cost": 0.0001, - "robustness": 76.9 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 60.0, + "accuracy": 0.0, "cost": 0.0001, - "robustness": 0.0 + "robustness": 75.0 }, "hard": { - "accuracy": 8.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0.0 }, "all": { - "accuracy": 80.6, - "cost": 0.0001, - "robustness": 63.2 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 84.2 } } } @@ -18790,96 +22518,96 @@ "History": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0001, - "robustness": 39.1 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 95.7 }, "medium": { - "accuracy": 76.3, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 50.0 }, "hard": { - "accuracy": 18.1, - "cost": 0.0006, + "accuracy": 0.0, + "cost": 0.0003, "robustness": 70.0 }, "all": { - "accuracy": 75.3, - "cost": 0.0002, - "robustness": 56.4 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 82.1 } }, "subcategories": { "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 88.9, + "accuracy": 0.0, "cost": 0.0001, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, + "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 94.2, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 } } }, "Geography": { "metrics": { "easy": { - "accuracy": 98.2, - "cost": 0.0001, - "robustness": 62.5 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 87.5 }, "medium": { - "accuracy": 65.5, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 41.7, - "cost": 0.0006, + "accuracy": 0.0, + "cost": 0.0005, "robustness": 0 }, "all": { - "accuracy": 87.3, + "accuracy": 0.0, "cost": 0.0001, - "robustness": 66.7 + "robustness": 88.9 } } }, "History": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0001, - "robustness": 18.2 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 77.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 40.0 }, "hard": { - "accuracy": 16.2, - "cost": 0.0006, + "accuracy": 0.0, + "cost": 0.0003, "robustness": 70.0 }, "all": { - "accuracy": 69.7, - "cost": 0.0003, - "robustness": 53.8 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 76.9 } } } @@ -18888,48 +22616,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 95.7, - "cost": 0.0001, - "robustness": 72.7 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 81.8 }, "medium": { - "accuracy": 77.2, + "accuracy": 0.0, "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 68.0, - "cost": 0.0002, - "robustness": 92.0 + "accuracy": 0.0, + "cost": 0.0005, + "robustness": 76.0 }, "all": { - "accuracy": 78.5, - "cost": 0.0001, - "robustness": 88.9 + "accuracy": 0.0, + "cost": 0.0003, + "robustness": 82.2 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 95.7, - "cost": 0.0001, - "robustness": 72.7 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 81.8 }, "medium": { - "accuracy": 77.2, + "accuracy": 0.0, "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 68.0, - "cost": 0.0002, - "robustness": 92.0 + "accuracy": 0.0, + "cost": 0.0005, + "robustness": 76.0 }, "all": { - "accuracy": 78.5, - "cost": 0.0001, - "robustness": 88.9 + "accuracy": 0.0, + "cost": 0.0003, + "robustness": 82.2 } } } @@ -18938,48 +22666,48 @@ "Literature": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 0.0, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 88.7, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 50.0 }, "hard": { - "accuracy": 40.5, - "cost": 0.0008, - "robustness": 91.7 + "accuracy": 0.0, + "cost": 0.001, + "robustness": 83.3 }, "all": { - "accuracy": 54.8, - "cost": 0.0006, - "robustness": 93.1 + "accuracy": 0.0, + "cost": 0.0008, + "robustness": 82.8 } }, "subcategories": { "Literature, rhetoric and criticism": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 0.0, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 88.7, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 50.0 }, "hard": { - "accuracy": 40.5, - "cost": 0.0008, - "robustness": 91.7 + "accuracy": 0.0, + "cost": 0.001, + "robustness": 83.3 }, "all": { - "accuracy": 54.8, - "cost": 0.0006, - "robustness": 93.1 + "accuracy": 0.0, + "cost": 0.0008, + "robustness": 82.8 } } } @@ -18988,23 +22716,23 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 94.4, - "cost": 0.0001, - "robustness": 63.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 81.5 }, "medium": { - "accuracy": 78.7, + "accuracy": 0.0, "cost": 0.0001, - "robustness": 70.0 + "robustness": 60.0 }, "hard": { - "accuracy": 34.6, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0003, + "robustness": 42.9 }, "all": { - "accuracy": 82.7, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 70.5 } }, @@ -19012,23 +22740,23 @@ "Ethics": { "metrics": { "easy": { - "accuracy": 92.4, - "cost": 0.0001, - "robustness": 71.4 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 85.7 }, "medium": { - "accuracy": 72.9, + "accuracy": 0.0, "cost": 0.0001, - "robustness": 60.0 + "robustness": 40.0 }, "hard": { - "accuracy": 61.5, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 50.0 }, "all": { - "accuracy": 85.2, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0, "robustness": 71.4 } } @@ -19036,23 +22764,23 @@ "Philosophical logic": { "metrics": { "easy": { - "accuracy": 98.5, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0, "robustness": 100.0 }, "medium": { - "accuracy": 86.8, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 60.0, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0003, "robustness": 0 }, "all": { - "accuracy": 92.6, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 100.0 } } @@ -19060,48 +22788,48 @@ "Philosophy": { "metrics": { "easy": { - "accuracy": 89.2, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0, "robustness": 66.7 }, "medium": { - "accuracy": 77.5, - "cost": 0.0003, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 50.0 }, "hard": { - "accuracy": 33.3, - "cost": 0.0006, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0005, + "robustness": 33.3 }, "all": { - "accuracy": 64.8, - "cost": 0.0004, - "robustness": 75.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 50.0 } } }, "Psychology": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0001, - "robustness": 16.7 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 66.7 }, "medium": { - "accuracy": 83.3, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0, "robustness": 0 }, "hard": { - "accuracy": 11.1, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 50.0 }, "all": { - "accuracy": 85.0, - "cost": 0.0001, - "robustness": 37.5 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 62.5 } } } @@ -19110,36 +22838,36 @@ "Science": { "metrics": { "easy": { - "accuracy": 98.3, - "cost": 0.0001, - "robustness": 59.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 84.6 }, "medium": { - "accuracy": 83.3, - "cost": 0.0003, - "robustness": 57.1 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 66.7 }, "hard": { - "accuracy": 32.5, + "accuracy": 0.0, "cost": 0.0005, "robustness": 77.8 }, "all": { - "accuracy": 84.6, - "cost": 0.0002, - "robustness": 60.9 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 78.3 } }, "subcategories": { "Animals (Zoology)": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0, "robustness": 50.0 }, "medium": { - "accuracy": 60.0, + "accuracy": 0.0, "cost": 0.0001, "robustness": 0 }, @@ -19149,8 +22877,8 @@ "robustness": 0 }, "all": { - "accuracy": 92.0, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0, "robustness": 50.0 } } @@ -19158,144 +22886,144 @@ "Biology": { "metrics": { "easy": { - "accuracy": 99.1, - "cost": 0.0002, - "robustness": 28.6 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 76.5, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 33.3, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 92.3, - "cost": 0.0002, - "robustness": 28.6 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 } } }, "Chemistry": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0001, - "robustness": 66.7 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 97.1, - "cost": 0.0003, + "accuracy": 0.0, + "cost": 0.0001, "robustness": 50.0 }, "hard": { - "accuracy": 15.4, - "cost": 0.0003, + "accuracy": 0.0, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 85.6, - "cost": 0.0002, - "robustness": 60.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 80.0 } } }, "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 96.9, - "cost": 0.0001, - "robustness": 75.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 50.0 }, "medium": { - "accuracy": 74.6, + "accuracy": 0.0, "cost": 0.0001, - "robustness": 33.3 + "robustness": 50.0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, + "cost": 0.0003, "robustness": 0.0 }, "all": { - "accuracy": 89.1, - "cost": 0.0001, - "robustness": 53.3 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 46.7 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 99.2, - "cost": 0.0001, - "robustness": 63.6 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 85.2, - "cost": 0.0003, - "robustness": 63.6 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 81.8 }, "hard": { - "accuracy": 48.3, - "cost": 0.001, - "robustness": 80.0 + "accuracy": 0.0, + "cost": 0.0006, + "robustness": 100.0 }, "all": { - "accuracy": 85.9, - "cost": 0.0003, - "robustness": 66.7 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 92.6 } } }, "Physics": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 85.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 50.0 }, "hard": { - "accuracy": 36.4, - "cost": 0.0003, + "accuracy": 0.0, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 85.6, - "cost": 0.0002, - "robustness": 40.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 80.0 } } }, "Science": { "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 100.0 + "easy": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 80.0 }, "medium": { - "accuracy": 72.0, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0003, "robustness": 0 }, "hard": { - "accuracy": 18.6, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0004, + "robustness": 66.7 }, "all": { - "accuracy": 51.8, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0003, + "robustness": 75.0 } } } @@ -19304,119 +23032,119 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 96.3, - "cost": 0.0002, - "robustness": 50.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 94.4 }, "medium": { - "accuracy": 76.3, - "cost": 0.0004, - "robustness": 66.7 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 83.3 }, "hard": { - "accuracy": 30.9, - "cost": 0.0007, - "robustness": 71.4 + "accuracy": 0.0, + "cost": 0.0004, + "robustness": 85.7 }, "all": { - "accuracy": 75.1, - "cost": 0.0004, - "robustness": 59.5 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 89.2 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 97.8, - "cost": 0.0001, - "robustness": 58.3 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 82.2, - "cost": 0.0004, - "robustness": 66.7 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 40.0, - "cost": 0.0008, + "accuracy": 0.0, + "cost": 0.0005, "robustness": 66.7 }, "all": { - "accuracy": 81.0, - "cost": 0.0004, - "robustness": 61.9 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 95.2 } } }, "Law": { "metrics": { "easy": { - "accuracy": 92.3, - "cost": 0.0003, - "robustness": 0.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 66.7 }, "medium": { - "accuracy": 68.3, - "cost": 0.0004, + "accuracy": 0.0, + "cost": 0.0003, "robustness": 50.0 }, "hard": { - "accuracy": 26.7, - "cost": 0.0004, - "robustness": 50.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 70.3, - "cost": 0.0003, - "robustness": 33.3 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 66.7 } } }, "Social problems": { "metrics": { "easy": { - "accuracy": 95.7, - "cost": 0.0001, - "robustness": 66.7 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 22.2, + "accuracy": 0.0, "cost": 0.0001, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, + "cost": 0.0, "robustness": 100.0 }, "all": { - "accuracy": 77.0, - "cost": 0.0001, - "robustness": 75.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 } } }, "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0, "robustness": 0 }, "medium": { - "accuracy": 77.8, - "cost": 0.0006, + "accuracy": 0.0, + "cost": 0.0003, "robustness": 100.0 }, "hard": { - "accuracy": 17.6, - "cost": 0.001, + "accuracy": 0.0, + "cost": 0.0005, "robustness": 100.0 }, "all": { - "accuracy": 41.8, - "cost": 0.0008, + "accuracy": 0.0, + "cost": 0.0004, "robustness": 100.0 } } @@ -19426,96 +23154,96 @@ "Technology": { "metrics": { "easy": { - "accuracy": 94.9, - "cost": 0.0001, - "robustness": 69.8 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 90.7 }, "medium": { - "accuracy": 73.6, - "cost": 0.0003, - "robustness": 76.2 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 85.7 }, "hard": { - "accuracy": 27.4, - "cost": 0.0004, - "robustness": 28.6 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 85.7 }, "all": { - "accuracy": 80.9, - "cost": 0.0002, - "robustness": 67.6 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 88.7 } }, "subcategories": { "Engineering": { "metrics": { "easy": { - "accuracy": 96.8, - "cost": 0.0002, + "accuracy": 0.0, + "cost": 0.0, "robustness": 80.0 }, "medium": { - "accuracy": 87.4, - "cost": 0.0006, - "robustness": 76.9 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 92.3 }, "hard": { - "accuracy": 57.1, - "cost": 0.001, - "robustness": 0.0 + "accuracy": 0.0, + "cost": 0.0003, + "robustness": 100.0 }, "all": { - "accuracy": 87.2, - "cost": 0.0005, - "robustness": 73.7 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 89.5 } } }, "Management and public relations": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0001, - "robustness": 75.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 63.6, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0, "robustness": 100.0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, + "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 89.0, - "cost": 0.0001, - "robustness": 80.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 94.3, - "cost": 0.0001, - "robustness": 67.6 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 91.2 }, "medium": { - "accuracy": 64.5, + "accuracy": 0.0, "cost": 0.0001, "robustness": 71.4 }, "hard": { - "accuracy": 16.8, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 83.3 }, "all": { - "accuracy": 78.2, + "accuracy": 0.0, "cost": 0.0001, - "robustness": 63.8 + "robustness": 87.2 } } } @@ -19523,75 +23251,75 @@ } } }, - "llm-router": { + "paix2": { "metrics": { "easy": { - "accuracy": 95.6, + "accuracy": 99.3, "cost": 0.0001, - "robustness": 30.8 + "robustness": 76.8 }, "medium": { - "accuracy": 69.1, - "cost": 0.0002, - "robustness": 23.9 + "accuracy": 83.6, + "cost": 0.0004, + "robustness": 79.6 }, "hard": { - "accuracy": 27.0, - "cost": 0.0002, - "robustness": 34.9 + "accuracy": 35.2, + "cost": 0.0005, + "robustness": 78.0 }, "all": { - "accuracy": 71.8, - "cost": 0.0001, - "robustness": 30.0 + "accuracy": 79.7, + "cost": 0.0003, + "robustness": 77.9 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 92.6, + "accuracy": 98.3, "cost": 0.0001, - "robustness": 37.5 + "robustness": 87.5 }, "medium": { - "accuracy": 52.2, - "cost": 0.0003, - "robustness": 27.3 + "accuracy": 78.0, + "cost": 0.0006, + "robustness": 81.8 }, "hard": { - "accuracy": 10.2, + "accuracy": 15.6, "cost": 0.0004, - "robustness": 20.0 + "robustness": 100.0 }, "all": { - "accuracy": 60.6, - "cost": 0.0002, - "robustness": 29.2 + "accuracy": 73.6, + "cost": 0.0004, + "robustness": 87.5 } }, "subcategories": { "Arts": { "metrics": { "easy": { - "accuracy": 93.2, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 64.1, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 82.1, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 8.9, - "cost": 0.0004, - "robustness": 0.0 + "accuracy": 3.6, + "cost": 0.0001, + "robustness": 100.0 }, - "all": { - "accuracy": 55.2, - "cost": 0.0002, - "robustness": 0.0 + "all": { + "accuracy": 60.4, + "cost": 0.0001, + "robustness": 100.0 } } }, @@ -19599,46 +23327,46 @@ "metrics": { "easy": { "accuracy": 97.0, - "cost": 0.0001, - "robustness": 20.0 + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 47.4, - "cost": 0.0005, - "robustness": 0.0 + "accuracy": 71.6, + "cost": 0.0009, + "robustness": 80.0 }, "hard": { - "accuracy": 12.0, - "cost": 0.0008, - "robustness": 0.0 + "accuracy": 48.0, + "cost": 0.001, + "robustness": 100.0 }, "all": { - "accuracy": 64.2, - "cost": 0.0003, - "robustness": 9.1 + "accuracy": 79.6, + "cost": 0.0006, + "robustness": 90.9 } } }, "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 89.3, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 98.6, + "cost": 0.0002, + "robustness": 50.0 }, "medium": { - "accuracy": 53.0, - "cost": 0.0001, + "accuracy": 84.0, + "cost": 0.0004, "robustness": 75.0 }, "hard": { - "accuracy": 10.6, - "cost": 0.0003, - "robustness": 50.0 + "accuracy": 13.6, + "cost": 0.0005, + "robustness": 100.0 }, "all": { - "accuracy": 60.5, - "cost": 0.0001, + "accuracy": 75.5, + "cost": 0.0003, "robustness": 75.0 } } @@ -19648,72 +23376,72 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 96.2, + "accuracy": 99.6, "cost": 0.0001, - "robustness": 23.1 + "robustness": 73.1 }, "medium": { - "accuracy": 76.6, - "cost": 0.0001, - "robustness": 14.3 + "accuracy": 90.5, + "cost": 0.0003, + "robustness": 66.7 }, "hard": { - "accuracy": 28.6, - "cost": 0.0001, - "robustness": 20.0 + "accuracy": 53.9, + "cost": 0.0011, + "robustness": 80.0 }, "all": { - "accuracy": 77.9, - "cost": 0.0001, - "robustness": 19.4 + "accuracy": 88.6, + "cost": 0.0003, + "robustness": 72.6 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 96.9, + "accuracy": 99.5, "cost": 0.0001, - "robustness": 23.1 + "robustness": 92.3 }, "medium": { - "accuracy": 81.4, - "cost": 0.0001, - "robustness": 11.8 + "accuracy": 94.7, + "cost": 0.0004, + "robustness": 70.6 }, "hard": { - "accuracy": 31.1, - "cost": 0.0002, - "robustness": 23.1 + "accuracy": 59.8, + "cost": 0.0013, + "robustness": 92.3 }, "all": { - "accuracy": 77.4, - "cost": 0.0001, - "robustness": 18.6 + "accuracy": 89.4, + "cost": 0.0004, + "robustness": 83.7 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 95.3, - "cost": 0.0001, - "robustness": 23.1 + "accuracy": 99.6, + "cost": 0.0, + "robustness": 53.8 }, "medium": { - "accuracy": 55.3, - "cost": 0.0001, - "robustness": 25.0 + "accuracy": 71.8, + "cost": 0.0002, + "robustness": 50.0 }, "hard": { - "accuracy": 13.9, + "accuracy": 19.4, "cost": 0.0001, "robustness": 0.0 }, "all": { - "accuracy": 79.3, + "accuracy": 86.4, "cost": 0.0001, - "robustness": 21.1 + "robustness": 47.4 } } } @@ -19722,24 +23450,24 @@ "History": { "metrics": { "easy": { - "accuracy": 96.8, + "accuracy": 98.9, "cost": 0.0001, - "robustness": 34.8 + "robustness": 78.3 }, "medium": { - "accuracy": 75.1, - "cost": 0.0002, - "robustness": 16.7 + "accuracy": 78.5, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 17.4, + "accuracy": 19.5, "cost": 0.0002, - "robustness": 10.0 + "robustness": 70.0 }, "all": { - "accuracy": 74.4, + "accuracy": 76.9, "cost": 0.0001, - "robustness": 25.6 + "robustness": 79.5 } }, "subcategories": { @@ -19752,16 +23480,16 @@ }, "medium": { "accuracy": 77.8, - "cost": 0.0, + "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 100.0, + "accuracy": 0.0, "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 94.2, + "accuracy": 92.3, "cost": 0.0, "robustness": 75.0 } @@ -19770,14 +23498,14 @@ "Geography": { "metrics": { "easy": { - "accuracy": 96.3, - "cost": 0.0001, - "robustness": 12.5 + "accuracy": 97.2, + "cost": 0.0, + "robustness": 75.0 }, "medium": { - "accuracy": 55.2, + "accuracy": 72.4, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "hard": { "accuracy": 25.0, @@ -19785,33 +23513,33 @@ "robustness": 0 }, "all": { - "accuracy": 82.7, + "accuracy": 86.7, "cost": 0.0001, - "robustness": 11.1 + "robustness": 77.8 } } }, "History": { "metrics": { "easy": { - "accuracy": 96.9, - "cost": 0.0002, - "robustness": 36.4 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 81.8 }, "medium": { - "accuracy": 79.1, - "cost": 0.0002, - "robustness": 20.0 + "accuracy": 79.9, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 16.2, + "accuracy": 19.1, "cost": 0.0002, - "robustness": 10.0 + "robustness": 70.0 }, "all": { - "accuracy": 69.9, - "cost": 0.0002, - "robustness": 23.1 + "accuracy": 72.3, + "cost": 0.0001, + "robustness": 80.8 } } } @@ -19820,48 +23548,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 88.5, + "accuracy": 97.1, "cost": 0.0, - "robustness": 45.5 + "robustness": 63.6 }, "medium": { - "accuracy": 48.5, + "accuracy": 71.9, "cost": 0.0001, - "robustness": 66.7 + "robustness": 88.9 }, "hard": { - "accuracy": 48.3, - "cost": 0.0, - "robustness": 84.0 + "accuracy": 51.7, + "cost": 0.0001, + "robustness": 92.0 }, "all": { - "accuracy": 60.3, - "cost": 0.0, - "robustness": 71.1 + "accuracy": 70.1, + "cost": 0.0001, + "robustness": 84.4 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 88.5, + "accuracy": 97.1, "cost": 0.0, - "robustness": 45.5 + "robustness": 63.6 }, "medium": { - "accuracy": 48.5, + "accuracy": 71.9, "cost": 0.0001, - "robustness": 66.7 + "robustness": 88.9 }, "hard": { - "accuracy": 48.3, - "cost": 0.0, - "robustness": 84.0 + "accuracy": 51.7, + "cost": 0.0001, + "robustness": 92.0 }, "all": { - "accuracy": 60.3, - "cost": 0.0, - "robustness": 71.1 + "accuracy": 70.1, + "cost": 0.0001, + "robustness": 84.4 } } } @@ -19872,22 +23600,22 @@ "easy": { "accuracy": 99.1, "cost": 0.0, - "robustness": 0.0 + "robustness": 66.7 }, "medium": { - "accuracy": 72.5, + "accuracy": 61.1, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "hard": { - "accuracy": 31.9, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 32.0, + "cost": 0.0001, + "robustness": 79.2 }, "all": { - "accuracy": 46.7, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 45.7, + "cost": 0.0001, + "robustness": 79.3 } }, "subcategories": { @@ -19896,22 +23624,22 @@ "easy": { "accuracy": 99.1, "cost": 0.0, - "robustness": 0.0 + "robustness": 66.7 }, "medium": { - "accuracy": 72.5, + "accuracy": 61.1, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "hard": { - "accuracy": 31.9, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 32.0, + "cost": 0.0001, + "robustness": 79.2 }, "all": { - "accuracy": 46.7, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 45.7, + "cost": 0.0001, + "robustness": 79.3 } } } @@ -19920,120 +23648,120 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 93.6, - "cost": 0.0, - "robustness": 85.2 + "accuracy": 99.3, + "cost": 0.0001, + "robustness": 70.4 }, "medium": { - "accuracy": 66.8, - "cost": 0.0, + "accuracy": 82.9, + "cost": 0.0003, "robustness": 70.0 }, "hard": { - "accuracy": 16.0, - "cost": 0.0002, + "accuracy": 23.5, + "cost": 0.0003, "robustness": 57.1 }, "all": { - "accuracy": 76.6, - "cost": 0.0001, - "robustness": 77.3 + "accuracy": 85.6, + "cost": 0.0002, + "robustness": 68.2 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 90.8, - "cost": 0.0, - "robustness": 92.9 + "accuracy": 99.5, + "cost": 0.0001, + "robustness": 64.3 }, "medium": { - "accuracy": 43.5, - "cost": 0.0, + "accuracy": 68.2, + "cost": 0.0001, "robustness": 80.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 23.1, + "cost": 0.0001, + "robustness": 50.0 }, "all": { - "accuracy": 72.4, - "cost": 0.0, - "robustness": 90.5 + "accuracy": 86.6, + "cost": 0.0001, + "robustness": 66.7 } } }, "Philosophical logic": { "metrics": { "easy": { - "accuracy": 95.4, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 98.5, + "cost": 0.0005, + "robustness": 75.0 }, "medium": { - "accuracy": 92.1, - "cost": 0.0, - "robustness": 66.7 + "accuracy": 94.7, + "cost": 0.0008, + "robustness": 33.3 }, "hard": { "accuracy": 60.0, - "cost": 0.0, + "cost": 0.002, "robustness": 0 }, "all": { - "accuracy": 92.6, - "cost": 0.0, - "robustness": 85.7 + "accuracy": 95.4, + "cost": 0.0007, + "robustness": 57.1 } } }, "Philosophy": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 66.7 }, "medium": { - "accuracy": 72.5, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 92.5, + "cost": 0.0004, + "robustness": 100.0 }, "hard": { - "accuracy": 17.8, - "cost": 0.0003, - "robustness": 33.3 + "accuracy": 20.0, + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 59.8, - "cost": 0.0001, - "robustness": 62.5 + "accuracy": 68.0, + "cost": 0.0002, + "robustness": 87.5 } } }, "Psychology": { "metrics": { "easy": { - "accuracy": 95.9, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 99.2, + "cost": 0.0, + "robustness": 83.3 }, "medium": { - "accuracy": 83.3, + "accuracy": 91.7, "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 11.1, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 22.2, + "cost": 0.0002, + "robustness": 0.0 }, "all": { - "accuracy": 84.5, + "accuracy": 89.8, "cost": 0.0001, - "robustness": 50.0 + "robustness": 62.5 } } } @@ -20042,37 +23770,37 @@ "Science": { "metrics": { "easy": { - "accuracy": 98.6, + "accuracy": 99.9, "cost": 0.0001, - "robustness": 7.7 + "robustness": 87.2 }, "medium": { - "accuracy": 79.4, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 87.7, + "cost": 0.0007, + "robustness": 85.7 }, "hard": { - "accuracy": 18.0, - "cost": 0.0005, - "robustness": 0.0 + "accuracy": 35.6, + "cost": 0.0021, + "robustness": 88.9 }, "all": { - "accuracy": 81.5, - "cost": 0.0002, - "robustness": 4.3 + "accuracy": 87.2, + "cost": 0.0005, + "robustness": 87.0 } }, "subcategories": { "Animals (Zoology)": { "metrics": { "easy": { - "accuracy": 95.0, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 100.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { "accuracy": 60.0, - "cost": 0.0, + "cost": 0.0001, "robustness": 0 }, "hard": { @@ -20081,18 +23809,18 @@ "robustness": 0 }, "all": { - "accuracy": 88.0, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 92.0, + "cost": 0.0, + "robustness": 100.0 } } }, "Biology": { "metrics": { "easy": { - "accuracy": 99.1, - "cost": 0.0001, - "robustness": 14.3 + "accuracy": 100.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { "accuracy": 76.5, @@ -20100,38 +23828,38 @@ "robustness": 0 }, "hard": { - "accuracy": 22.2, - "cost": 0.0002, + "accuracy": 55.6, + "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 91.6, + "accuracy": 94.4, "cost": 0.0001, - "robustness": 14.3 + "robustness": 100.0 } } }, "Chemistry": { "metrics": { "easy": { - "accuracy": 95.3, + "accuracy": 100.0, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "medium": { - "accuracy": 85.3, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 91.2, + "cost": 0.0008, + "robustness": 100.0 }, "hard": { - "accuracy": 23.1, - "cost": 0.0004, + "accuracy": 53.8, + "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 81.1, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 90.0, + "cost": 0.0003, + "robustness": 100.0 } } }, @@ -20139,71 +23867,71 @@ "metrics": { "easy": { "accuracy": 99.6, - "cost": 0.0001, - "robustness": 0.0 + "cost": 0.0, + "robustness": 87.5 }, "medium": { - "accuracy": 59.3, + "accuracy": 81.4, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "hard": { - "accuracy": 23.1, + "accuracy": 7.7, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "all": { - "accuracy": 89.4, + "accuracy": 92.7, "cost": 0.0001, - "robustness": 0.0 + "robustness": 93.3 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 98.1, - "cost": 0.0001, - "robustness": 9.1 + "accuracy": 100.0, + "cost": 0.0002, + "robustness": 81.8 }, "medium": { - "accuracy": 83.6, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 90.8, + "cost": 0.0009, + "robustness": 72.7 }, "hard": { - "accuracy": 22.5, - "cost": 0.0008, - "robustness": 0.0 + "accuracy": 52.8, + "cost": 0.0043, + "robustness": 80.0 }, "all": { - "accuracy": 80.9, - "cost": 0.0002, - "robustness": 3.7 + "accuracy": 89.2, + "cost": 0.0011, + "robustness": 77.8 } } }, "Physics": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 100.0, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 92.9, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 95.2, + "cost": 0.0003, + "robustness": 100.0 }, "hard": { - "accuracy": 45.5, - "cost": 0.0003, + "accuracy": 54.5, + "cost": 0.0008, "robustness": 0 }, "all": { - "accuracy": 89.7, + "accuracy": 92.8, "cost": 0.0002, - "robustness": 0.0 + "robustness": 100.0 } } }, @@ -20211,23 +23939,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 0.0 + "cost": 0.0, + "robustness": 60.0 }, "medium": { - "accuracy": 60.0, + "accuracy": 68.0, "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 3.4, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 5.1, + "cost": 0.0001, + "robustness": 100.0 }, "all": { - "accuracy": 41.2, + "accuracy": 43.9, "cost": 0.0001, - "robustness": 0.0 + "robustness": 75.0 } } } @@ -20236,94 +23964,94 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 97.0, + "accuracy": 99.7, "cost": 0.0001, - "robustness": 16.7 + "robustness": 94.4 }, "medium": { - "accuracy": 67.6, - "cost": 0.0002, - "robustness": 41.7 + "accuracy": 85.0, + "cost": 0.0004, + "robustness": 75.0 }, "hard": { - "accuracy": 11.4, - "cost": 0.0002, - "robustness": 42.9 + "accuracy": 22.1, + "cost": 0.0005, + "robustness": 71.4 }, "all": { - "accuracy": 68.1, - "cost": 0.0001, - "robustness": 29.7 + "accuracy": 77.9, + "cost": 0.0003, + "robustness": 83.8 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 97.3, + "accuracy": 99.5, "cost": 0.0001, - "robustness": 8.3 + "robustness": 100.0 }, "medium": { - "accuracy": 71.2, - "cost": 0.0002, - "robustness": 66.7 + "accuracy": 89.0, + "cost": 0.0005, + "robustness": 100.0 }, "hard": { - "accuracy": 8.8, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 26.2, + "cost": 0.0007, + "robustness": 100.0 }, "all": { - "accuracy": 70.7, - "cost": 0.0001, - "robustness": 28.6 + "accuracy": 81.7, + "cost": 0.0004, + "robustness": 100.0 } } }, "Law": { "metrics": { "easy": { - "accuracy": 95.4, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 73.0, - "cost": 0.0002, - "robustness": 25.0 + "accuracy": 84.1, + "cost": 0.0003, + "robustness": 50.0 }, "hard": { - "accuracy": 23.3, + "accuracy": 30.0, "cost": 0.0002, "robustness": 50.0 }, "all": { - "accuracy": 72.8, + "accuracy": 80.4, "cost": 0.0002, - "robustness": 33.3 + "robustness": 66.7 } } }, "Social problems": { "metrics": { "easy": { - "accuracy": 97.9, + "accuracy": 100.0, "cost": 0.0, - "robustness": 33.3 + "robustness": 66.7 }, "medium": { - "accuracy": 0.0, + "accuracy": 66.7, "cost": 0.0, "robustness": 0 }, "hard": { - "accuracy": 20.0, + "accuracy": 40.0, "cost": 0.0, - "robustness": 100.0 + "robustness": 0.0 }, "all": { - "accuracy": 77.0, + "accuracy": 90.2, "cost": 0.0, "robustness": 50.0 } @@ -20333,23 +24061,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, + "cost": 0.0, "robustness": 0 }, "medium": { - "accuracy": 50.0, + "accuracy": 61.1, "cost": 0.0001, - "robustness": 0.0 + "robustness": 50.0 }, "hard": { - "accuracy": 5.9, + "accuracy": 2.9, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "all": { - "accuracy": 25.5, + "accuracy": 27.3, "cost": 0.0001, - "robustness": 0.0 + "robustness": 66.7 } } } @@ -20358,96 +24086,96 @@ "Technology": { "metrics": { "easy": { - "accuracy": 94.8, + "accuracy": 99.4, "cost": 0.0001, - "robustness": 23.3 + "robustness": 67.4 }, "medium": { - "accuracy": 67.4, - "cost": 0.0002, - "robustness": 9.5 + "accuracy": 85.0, + "cost": 0.0003, + "robustness": 81.0 }, "hard": { - "accuracy": 21.0, - "cost": 0.0002, - "robustness": 71.4 + "accuracy": 33.8, + "cost": 0.0006, + "robustness": 28.6 }, "all": { - "accuracy": 78.3, + "accuracy": 87.7, "cost": 0.0002, - "robustness": 23.9 + "robustness": 67.6 } }, "subcategories": { "Engineering": { "metrics": { "easy": { - "accuracy": 95.2, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 99.2, + "cost": 0.0003, + "robustness": 80.0 }, "medium": { - "accuracy": 84.4, - "cost": 0.0003, - "robustness": 7.7 + "accuracy": 92.2, + "cost": 0.0005, + "robustness": 100.0 }, "hard": { - "accuracy": 47.6, - "cost": 0.0005, + "accuracy": 61.9, + "cost": 0.0019, "robustness": 0.0 }, "all": { - "accuracy": 83.9, - "cost": 0.0003, - "robustness": 5.3 + "accuracy": 91.0, + "cost": 0.0006, + "robustness": 89.5 } } }, "Management and public relations": { "metrics": { "easy": { - "accuracy": 95.0, + "accuracy": 100.0, "cost": 0.0, - "robustness": 50.0 + "robustness": 75.0 }, "medium": { - "accuracy": 63.6, - "cost": 0.0, + "accuracy": 81.8, + "cost": 0.0001, "robustness": 0.0 }, "hard": { "accuracy": 0.0, - "cost": 0.0, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 87.7, - "cost": 0.0, - "robustness": 40.0 + "accuracy": 94.5, + "cost": 0.0001, + "robustness": 60.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 94.7, + "accuracy": 99.4, "cost": 0.0001, - "robustness": 23.5 + "robustness": 64.7 }, "medium": { - "accuracy": 55.8, + "accuracy": 80.2, "cost": 0.0001, - "robustness": 14.3 + "robustness": 57.1 }, "hard": { - "accuracy": 11.5, + "accuracy": 23.9, "cost": 0.0001, - "robustness": 83.3 + "robustness": 33.3 }, "all": { - "accuracy": 75.7, + "accuracy": 86.1, "cost": 0.0001, - "robustness": 29.8 + "robustness": 59.6 } } } @@ -20455,51 +24183,51 @@ } } }, - "barouter": { + "divyam": { "metrics": { "easy": { - "accuracy": 94.3, - "cost": 0.0004, - "robustness": 49.0 + "accuracy": 97.6, + "cost": 0.0003, + "robustness": 98.0 }, "medium": { - "accuracy": 62.9, - "cost": 0.0007, - "robustness": 60.2 + "accuracy": 78.3, + "cost": 0.0006, + "robustness": 99.1 }, "hard": { - "accuracy": 24.6, - "cost": 0.0009, - "robustness": 50.5 + "accuracy": 40.5, + "cost": 0.0007, + "robustness": 98.2 }, "all": { - "accuracy": 68.9, - "cost": 0.0006, - "robustness": 52.4 + "accuracy": 78.6, + "cost": 0.0005, + "robustness": 98.3 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 94.0, - "cost": 0.0005, - "robustness": 37.5 + "accuracy": 97.3, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 51.0, - "cost": 0.001, - "robustness": 72.7 + "accuracy": 71.8, + "cost": 0.0005, + "robustness": 100.0 }, "hard": { - "accuracy": 6.8, - "cost": 0.0011, - "robustness": 40.0 + "accuracy": 26.5, + "cost": 0.0006, + "robustness": 100.0 }, "all": { - "accuracy": 60.0, - "cost": 0.0008, - "robustness": 54.2 + "accuracy": 73.1, + "cost": 0.0005, + "robustness": 100.0 } }, "subcategories": { @@ -20507,71 +24235,71 @@ "metrics": { "easy": { "accuracy": 96.6, - "cost": 0.0003, + "cost": 0.0002, "robustness": 100.0 }, "medium": { - "accuracy": 64.1, + "accuracy": 94.9, "cost": 0.0003, - "robustness": 50.0 + "robustness": 100.0 }, "hard": { - "accuracy": 7.1, + "accuracy": 26.8, "cost": 0.0003, - "robustness": 0.0 + "robustness": 100.0 }, "all": { - "accuracy": 55.8, + "accuracy": 70.8, "cost": 0.0003, - "robustness": 40.0 + "robustness": 100.0 } } }, "Music": { "metrics": { "easy": { - "accuracy": 97.0, + "accuracy": 96.0, "cost": 0.0003, - "robustness": 20.0 + "robustness": 100.0 }, "medium": { - "accuracy": 35.3, - "cost": 0.0008, - "robustness": 60.0 + "accuracy": 56.0, + "cost": 0.0006, + "robustness": 100.0 }, "hard": { - "accuracy": 4.0, - "cost": 0.0005, - "robustness": 0.0 + "accuracy": 24.0, + "cost": 0.0006, + "robustness": 100.0 }, "all": { - "accuracy": 57.5, - "cost": 0.0006, - "robustness": 36.4 + "accuracy": 69.2, + "cost": 0.0005, + "robustness": 100.0 } } }, "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 90.7, - "cost": 0.0008, - "robustness": 50.0 + "accuracy": 98.6, + "cost": 0.0004, + "robustness": 100.0 }, "medium": { - "accuracy": 64.0, - "cost": 0.0016, + "accuracy": 81.0, + "cost": 0.0006, "robustness": 100.0 }, "hard": { - "accuracy": 7.6, - "cost": 0.0021, + "accuracy": 27.3, + "cost": 0.0009, "robustness": 100.0 }, "all": { - "accuracy": 64.1, - "cost": 0.0013, - "robustness": 87.5 + "accuracy": 77.5, + "cost": 0.0006, + "robustness": 100.0 } } } @@ -20580,72 +24308,72 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 96.1, - "cost": 0.0005, - "robustness": 38.5 + "accuracy": 99.0, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 74.5, + "accuracy": 90.9, "cost": 0.0008, - "robustness": 47.6 + "robustness": 100.0 }, "hard": { - "accuracy": 29.4, - "cost": 0.0019, - "robustness": 66.7 + "accuracy": 65.7, + "cost": 0.0016, + "robustness": 93.3 }, "all": { - "accuracy": 77.3, - "cost": 0.0008, - "robustness": 48.4 + "accuracy": 90.5, + "cost": 0.0007, + "robustness": 98.4 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0005, - "robustness": 30.8 + "accuracy": 99.8, + "cost": 0.0004, + "robustness": 100.0 }, "medium": { - "accuracy": 78.2, + "accuracy": 93.6, "cost": 0.0009, - "robustness": 47.1 + "robustness": 100.0 }, "hard": { - "accuracy": 33.0, - "cost": 0.0022, - "robustness": 61.5 + "accuracy": 73.7, + "cost": 0.0018, + "robustness": 92.3 }, "all": { - "accuracy": 76.5, - "cost": 0.001, - "robustness": 46.5 + "accuracy": 92.0, + "cost": 0.0009, + "robustness": 97.7 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 95.3, - "cost": 0.0005, - "robustness": 46.2 + "accuracy": 97.8, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 57.6, - "cost": 0.0005, - "robustness": 50.0 + "accuracy": 78.8, + "cost": 0.0003, + "robustness": 100.0 }, "hard": { - "accuracy": 8.3, - "cost": 0.0005, + "accuracy": 19.4, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 79.3, - "cost": 0.0005, - "robustness": 52.6 + "accuracy": 86.6, + "cost": 0.0003, + "robustness": 100.0 } } } @@ -20654,96 +24382,96 @@ "History": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0005, - "robustness": 69.6 + "accuracy": 98.1, + "cost": 0.0003, + "robustness": 95.7 }, "medium": { - "accuracy": 64.4, - "cost": 0.0005, - "robustness": 83.3 + "accuracy": 83.1, + "cost": 0.0004, + "robustness": 100.0 }, - "hard": { - "accuracy": 8.7, - "cost": 0.0006, - "robustness": 50.0 + "hard": { + "accuracy": 28.9, + "cost": 0.0004, + "robustness": 100.0 }, "all": { - "accuracy": 70.3, - "cost": 0.0005, - "robustness": 66.7 + "accuracy": 79.6, + "cost": 0.0003, + "robustness": 97.4 } }, "subcategories": { "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0005, - "robustness": 75.0 + "accuracy": 97.6, + "cost": 0.0002, + "robustness": 100.0 }, "medium": { - "accuracy": 66.7, - "cost": 0.0005, + "accuracy": 100.0, + "cost": 0.0002, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0005, + "cost": 0.0003, "robustness": 0 }, "all": { - "accuracy": 92.3, - "cost": 0.0005, - "robustness": 75.0 + "accuracy": 96.2, + "cost": 0.0002, + "robustness": 100.0 } } }, "Geography": { "metrics": { "easy": { - "accuracy": 98.2, - "cost": 0.0003, - "robustness": 50.0 + "accuracy": 99.1, + "cost": 0.0002, + "robustness": 100.0 }, "medium": { - "accuracy": 51.7, - "cost": 0.0004, + "accuracy": 62.1, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 8.3, - "cost": 0.0004, + "accuracy": 58.3, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 82.0, - "cost": 0.0003, - "robustness": 55.6 + "accuracy": 88.7, + "cost": 0.0002, + "robustness": 100.0 } } }, "History": { "metrics": { "easy": { - "accuracy": 96.9, - "cost": 0.0005, - "robustness": 81.8 + "accuracy": 97.8, + "cost": 0.0004, + "robustness": 90.9 }, "medium": { - "accuracy": 66.9, - "cost": 0.0006, - "robustness": 80.0 + "accuracy": 86.3, + "cost": 0.0004, + "robustness": 100.0 }, "hard": { - "accuracy": 8.8, - "cost": 0.0006, - "robustness": 50.0 + "accuracy": 26.5, + "cost": 0.0004, + "robustness": 100.0 }, "all": { - "accuracy": 64.5, - "cost": 0.0006, - "robustness": 69.2 + "accuracy": 75.1, + "cost": 0.0004, + "robustness": 96.2 } } } @@ -20752,48 +24480,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 70.3, + "accuracy": 93.3, "cost": 0.0003, - "robustness": 63.6 + "robustness": 100.0 }, "medium": { - "accuracy": 28.1, - "cost": 0.0005, - "robustness": 77.8 + "accuracy": 50.3, + "cost": 0.0004, + "robustness": 100.0 }, "hard": { - "accuracy": 50.9, - "cost": 0.0004, - "robustness": 52.0 + "accuracy": 56.2, + "cost": 0.0005, + "robustness": 96.0 }, "all": { - "accuracy": 51.3, + "accuracy": 65.9, "cost": 0.0004, - "robustness": 60.0 + "robustness": 97.8 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 70.3, + "accuracy": 93.3, "cost": 0.0003, - "robustness": 63.6 + "robustness": 100.0 }, "medium": { - "accuracy": 28.1, - "cost": 0.0005, - "robustness": 77.8 + "accuracy": 50.3, + "cost": 0.0004, + "robustness": 100.0 }, "hard": { - "accuracy": 50.9, - "cost": 0.0004, - "robustness": 52.0 + "accuracy": 56.2, + "cost": 0.0005, + "robustness": 96.0 }, "all": { - "accuracy": 51.3, + "accuracy": 65.9, "cost": 0.0004, - "robustness": 60.0 + "robustness": 97.8 } } } @@ -20802,48 +24530,48 @@ "Literature": { "metrics": { "easy": { - "accuracy": 98.2, - "cost": 0.0004, - "robustness": 66.7 + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 67.3, - "cost": 0.0004, - "robustness": 50.0 + "accuracy": 83.4, + "cost": 0.0003, + "robustness": 100.0 }, "hard": { - "accuracy": 30.3, + "accuracy": 40.1, "cost": 0.0005, - "robustness": 50.0 + "robustness": 100.0 }, "all": { - "accuracy": 44.9, + "accuracy": 53.9, "cost": 0.0005, - "robustness": 51.7 + "robustness": 100.0 } }, "subcategories": { "Literature, rhetoric and criticism": { "metrics": { "easy": { - "accuracy": 98.2, - "cost": 0.0004, - "robustness": 66.7 + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 67.3, - "cost": 0.0004, - "robustness": 50.0 + "accuracy": 83.4, + "cost": 0.0003, + "robustness": 100.0 }, "hard": { - "accuracy": 30.3, + "accuracy": 40.1, "cost": 0.0005, - "robustness": 50.0 + "robustness": 100.0 }, "all": { - "accuracy": 44.9, + "accuracy": 53.9, "cost": 0.0005, - "robustness": 51.7 + "robustness": 100.0 } } } @@ -20852,48 +24580,48 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 92.9, - "cost": 0.0004, - "robustness": 51.9 + "accuracy": 97.1, + "cost": 0.0003, + "robustness": 92.6 }, "medium": { - "accuracy": 61.1, - "cost": 0.0005, - "robustness": 30.0 + "accuracy": 71.1, + "cost": 0.0004, + "robustness": 100.0 }, "hard": { - "accuracy": 8.6, - "cost": 0.0006, - "robustness": 42.9 + "accuracy": 35.8, + "cost": 0.0004, + "robustness": 100.0 }, "all": { - "accuracy": 73.6, - "cost": 0.0004, - "robustness": 45.5 + "accuracy": 82.1, + "cost": 0.0003, + "robustness": 95.5 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 87.6, - "cost": 0.0002, - "robustness": 57.1 + "accuracy": 95.7, + "cost": 0.0003, + "robustness": 92.9 }, "medium": { - "accuracy": 38.8, + "accuracy": 42.4, "cost": 0.0003, - "robustness": 40.0 + "robustness": 100.0 }, "hard": { "accuracy": 0.0, - "cost": 0.0002, - "robustness": 0.0 + "cost": 0.0003, + "robustness": 100.0 }, "all": { - "accuracy": 68.9, - "cost": 0.0002, - "robustness": 47.6 + "accuracy": 75.3, + "cost": 0.0003, + "robustness": 95.2 } } }, @@ -20902,70 +24630,70 @@ "easy": { "accuracy": 98.5, "cost": 0.0004, - "robustness": 25.0 + "robustness": 100.0 }, "medium": { - "accuracy": 76.3, - "cost": 0.0008, - "robustness": 0.0 + "accuracy": 84.2, + "cost": 0.0005, + "robustness": 100.0 }, "hard": { - "accuracy": 40.0, - "cost": 0.0011, + "accuracy": 60.0, + "cost": 0.0007, "robustness": 0 }, "all": { - "accuracy": 88.0, - "cost": 0.0006, - "robustness": 14.3 + "accuracy": 91.7, + "cost": 0.0004, + "robustness": 100.0 } } }, "Philosophy": { "metrics": { "easy": { - "accuracy": 94.6, - "cost": 0.0004, - "robustness": 66.7 + "accuracy": 97.3, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 65.0, - "cost": 0.0007, - "robustness": 50.0 + "accuracy": 95.0, + "cost": 0.0004, + "robustness": 100.0 }, "hard": { - "accuracy": 8.9, - "cost": 0.0006, - "robustness": 33.3 + "accuracy": 42.2, + "cost": 0.0004, + "robustness": 100.0 }, "all": { - "accuracy": 53.3, - "cost": 0.0006, - "robustness": 50.0 + "accuracy": 76.2, + "cost": 0.0004, + "robustness": 100.0 } } }, "Psychology": { "metrics": { "easy": { - "accuracy": 97.5, - "cost": 0.0006, - "robustness": 50.0 + "accuracy": 98.3, + "cost": 0.0003, + "robustness": 83.3 }, "medium": { - "accuracy": 85.4, - "cost": 0.0005, + "accuracy": 91.7, + "cost": 0.0004, "robustness": 0 }, "hard": { - "accuracy": 5.6, - "cost": 0.0006, + "accuracy": 38.9, + "cost": 0.0004, "robustness": 100.0 }, "all": { - "accuracy": 85.6, - "cost": 0.0006, - "robustness": 62.5 + "accuracy": 90.9, + "cost": 0.0004, + "robustness": 87.5 } } } @@ -20974,24 +24702,24 @@ "Science": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0004, - "robustness": 59.0 + "accuracy": 98.6, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 74.1, - "cost": 0.001, - "robustness": 66.7 + "accuracy": 83.3, + "cost": 0.0006, + "robustness": 100.0 }, "hard": { - "accuracy": 14.9, - "cost": 0.002, - "robustness": 44.4 + "accuracy": 26.3, + "cost": 0.0008, + "robustness": 100.0 }, "all": { - "accuracy": 78.7, - "cost": 0.0008, - "robustness": 59.4 + "accuracy": 83.9, + "cost": 0.0005, + "robustness": 100.0 } }, "subcategories": { @@ -20999,12 +24727,12 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0004, - "robustness": 50.0 + "cost": 0.0002, + "robustness": 100.0 }, "medium": { "accuracy": 80.0, - "cost": 0.0004, + "cost": 0.0003, "robustness": 0 }, "hard": { @@ -21014,31 +24742,31 @@ }, "all": { "accuracy": 96.0, - "cost": 0.0004, - "robustness": 50.0 + "cost": 0.0002, + "robustness": 100.0 } } }, "Biology": { "metrics": { "easy": { - "accuracy": 97.4, - "cost": 0.0005, + "accuracy": 100.0, + "cost": 0.0004, "robustness": 100.0 }, "medium": { - "accuracy": 70.6, - "cost": 0.0006, + "accuracy": 100.0, + "cost": 0.0005, "robustness": 0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0007, + "accuracy": 44.4, + "cost": 0.0005, "robustness": 0 }, "all": { - "accuracy": 88.1, - "cost": 0.0005, + "accuracy": 96.5, + "cost": 0.0004, "robustness": 100.0 } } @@ -21046,14 +24774,14 @@ "Chemistry": { "metrics": { "easy": { - "accuracy": 97.7, + "accuracy": 100.0, "cost": 0.0004, - "robustness": 33.3 + "robustness": 100.0 }, "medium": { - "accuracy": 85.3, + "accuracy": 94.1, "cost": 0.0006, - "robustness": 50.0 + "robustness": 100.0 }, "hard": { "accuracy": 23.1, @@ -21061,57 +24789,57 @@ "robustness": 0 }, "all": { - "accuracy": 82.2, + "accuracy": 86.7, "cost": 0.0005, - "robustness": 40.0 + "robustness": 100.0 } } }, "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 97.3, + "accuracy": 98.4, "cost": 0.0003, - "robustness": 25.0 + "robustness": 100.0 }, "medium": { - "accuracy": 64.4, - "cost": 0.0004, - "robustness": 66.7 + "accuracy": 69.5, + "cost": 0.0003, + "robustness": 100.0 }, "hard": { - "accuracy": 7.7, + "accuracy": 23.1, "cost": 0.0003, - "robustness": 0.0 + "robustness": 100.0 }, "all": { - "accuracy": 87.9, + "accuracy": 90.3, "cost": 0.0003, - "robustness": 40.0 + "robustness": 100.0 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 96.9, + "accuracy": 97.7, "cost": 0.0004, - "robustness": 72.7 + "robustness": 100.0 }, "medium": { - "accuracy": 78.0, - "cost": 0.0013, - "robustness": 63.6 + "accuracy": 85.2, + "cost": 0.0007, + "robustness": 100.0 }, "hard": { - "accuracy": 21.3, - "cost": 0.004, - "robustness": 60.0 + "accuracy": 31.5, + "cost": 0.0013, + "robustness": 100.0 }, "all": { - "accuracy": 77.9, - "cost": 0.0013, - "robustness": 66.7 + "accuracy": 82.7, + "cost": 0.0007, + "robustness": 100.0 } } }, @@ -21120,46 +24848,46 @@ "easy": { "accuracy": 97.7, "cost": 0.0003, - "robustness": 66.7 + "robustness": 100.0 }, "medium": { - "accuracy": 73.8, + "accuracy": 88.1, "cost": 0.0006, "robustness": 100.0 }, "hard": { - "accuracy": 36.4, - "cost": 0.0003, + "accuracy": 45.5, + "cost": 0.0005, "robustness": 0 }, "all": { - "accuracy": 80.4, - "cost": 0.0004, - "robustness": 80.0 + "accuracy": 87.6, + "cost": 0.0005, + "robustness": 100.0 } } }, "Science": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0002, - "robustness": 40.0 + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 44.0, + "accuracy": 64.0, "cost": 0.0003, "robustness": 0 }, "hard": { - "accuracy": 3.4, + "accuracy": 13.6, "cost": 0.0003, - "robustness": 33.3 + "robustness": 100.0 }, "all": { - "accuracy": 36.8, + "accuracy": 47.4, "cost": 0.0003, - "robustness": 37.5 + "robustness": 100.0 } } } @@ -21168,120 +24896,120 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 95.0, - "cost": 0.0005, - "robustness": 77.8 + "accuracy": 96.6, + "cost": 0.0004, + "robustness": 100.0 }, "medium": { - "accuracy": 59.7, - "cost": 0.0008, - "robustness": 75.0 + "accuracy": 77.9, + "cost": 0.0005, + "robustness": 91.7 }, "hard": { - "accuracy": 9.4, - "cost": 0.0009, - "robustness": 57.1 + "accuracy": 22.8, + "cost": 0.0005, + "robustness": 100.0 }, "all": { - "accuracy": 64.0, - "cost": 0.0007, - "robustness": 73.0 + "accuracy": 74.1, + "cost": 0.0005, + "robustness": 97.3 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 96.7, + "accuracy": 96.2, "cost": 0.0004, - "robustness": 83.3 + "robustness": 100.0 }, "medium": { - "accuracy": 63.8, - "cost": 0.0008, - "robustness": 100.0 + "accuracy": 77.9, + "cost": 0.0006, + "robustness": 83.3 }, "hard": { - "accuracy": 8.8, - "cost": 0.0012, - "robustness": 66.7 + "accuracy": 22.5, + "cost": 0.0006, + "robustness": 100.0 }, "all": { - "accuracy": 67.6, - "cost": 0.0007, - "robustness": 85.7 + "accuracy": 75.4, + "cost": 0.0005, + "robustness": 95.2 } } }, "Law": { "metrics": { "easy": { - "accuracy": 92.3, - "cost": 0.0007, + "accuracy": 98.5, + "cost": 0.0005, "robustness": 100.0 }, "medium": { - "accuracy": 55.6, - "cost": 0.0009, - "robustness": 75.0 + "accuracy": 82.5, + "cost": 0.0005, + "robustness": 100.0 }, "hard": { - "accuracy": 16.7, - "cost": 0.0007, - "robustness": 50.0 + "accuracy": 33.3, + "cost": 0.0005, + "robustness": 100.0 }, "all": { - "accuracy": 63.3, - "cost": 0.0008, - "robustness": 77.8 + "accuracy": 79.7, + "cost": 0.0005, + "robustness": 100.0 } } }, "Social problems": { "metrics": { "easy": { - "accuracy": 93.6, - "cost": 0.0005, - "robustness": 33.3 + "accuracy": 95.7, + "cost": 0.0002, + "robustness": 100.0 }, "medium": { - "accuracy": 22.2, - "cost": 0.0005, + "accuracy": 33.3, + "cost": 0.0002, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0005, - "robustness": 0.0 + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 75.4, - "cost": 0.0005, - "robustness": 25.0 + "accuracy": 78.7, + "cost": 0.0002, + "robustness": 100.0 } } }, "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 66.7, - "cost": 0.0003, + "accuracy": 100.0, + "cost": 0.0002, "robustness": 0 }, "medium": { - "accuracy": 55.6, - "cost": 0.0004, - "robustness": 0.0 + "accuracy": 83.3, + "cost": 0.0003, + "robustness": 100.0 }, "hard": { - "accuracy": 5.9, - "cost": 0.0005, + "accuracy": 17.6, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 25.5, - "cost": 0.0005, - "robustness": 33.3 + "accuracy": 43.6, + "cost": 0.0003, + "robustness": 100.0 } } } @@ -21290,96 +25018,96 @@ "Technology": { "metrics": { "easy": { - "accuracy": 94.7, + "accuracy": 97.0, "cost": 0.0004, - "robustness": 18.6 + "robustness": 97.7 }, "medium": { - "accuracy": 61.0, + "accuracy": 75.5, "cost": 0.0006, - "robustness": 52.4 + "robustness": 100.0 }, "hard": { - "accuracy": 10.8, + "accuracy": 30.6, "cost": 0.0005, - "robustness": 28.6 + "robustness": 100.0 }, "all": { - "accuracy": 75.1, + "accuracy": 83.1, "cost": 0.0005, - "robustness": 29.6 + "robustness": 98.6 } }, "subcategories": { "Engineering": { "metrics": { "easy": { - "accuracy": 96.0, - "cost": 0.0006, - "robustness": 20.0 + "accuracy": 97.6, + "cost": 0.0005, + "robustness": 100.0 }, "medium": { - "accuracy": 77.8, + "accuracy": 89.2, "cost": 0.0008, - "robustness": 69.2 + "robustness": 100.0 }, "hard": { - "accuracy": 26.2, - "cost": 0.0009, - "robustness": 0.0 + "accuracy": 54.8, + "cost": 0.0008, + "robustness": 100.0 }, "all": { - "accuracy": 78.2, + "accuracy": 88.1, "cost": 0.0007, - "robustness": 52.6 + "robustness": 100.0 } } }, "Management and public relations": { "metrics": { "easy": { - "accuracy": 98.3, - "cost": 0.0002, - "robustness": 75.0 + "accuracy": 96.7, + "cost": 0.0003, + "robustness": 100.0 }, "medium": { - "accuracy": 54.5, + "accuracy": 63.6, "cost": 0.0003, "robustness": 100.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0002, + "accuracy": 50.0, + "cost": 0.0003, "robustness": 0 }, "all": { - "accuracy": 89.0, - "cost": 0.0002, - "robustness": 80.0 + "accuracy": 90.4, + "cost": 0.0003, + "robustness": 100.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 94.0, + "accuracy": 96.9, "cost": 0.0004, - "robustness": 11.8 + "robustness": 97.1 }, "medium": { - "accuracy": 49.6, + "accuracy": 66.5, "cost": 0.0004, - "robustness": 14.3 + "robustness": 100.0 }, "hard": { - "accuracy": 5.3, + "accuracy": 21.2, "cost": 0.0004, - "robustness": 33.3 + "robustness": 100.0 }, "all": { - "accuracy": 73.1, + "accuracy": 80.8, "cost": 0.0004, - "robustness": 14.9 + "robustness": 97.9 } } } @@ -21387,123 +25115,123 @@ } } }, - "lynkr": { + "asiainfo": { "metrics": { "easy": { - "accuracy": 93.5, - "cost": 0.0001, - "robustness": 96.0 + "accuracy": 95.9, + "cost": 0.0048, + "robustness": 65.2 }, "medium": { - "accuracy": 60.9, - "cost": 0.0003, - "robustness": 89.4 + "accuracy": 73.3, + "cost": 0.0085, + "robustness": 66.4 }, "hard": { - "accuracy": 26.7, - "cost": 0.0005, - "robustness": 89.0 + "accuracy": 35.5, + "cost": 0.0162, + "robustness": 80.7 }, "all": { - "accuracy": 68.4, - "cost": 0.0003, - "robustness": 92.4 + "accuracy": 75.2, + "cost": 0.0085, + "robustness": 69.5 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 90.6, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 95.0, + "cost": 0.0037, + "robustness": 87.5 }, "medium": { - "accuracy": 42.4, - "cost": 0.0005, + "accuracy": 52.5, + "cost": 0.0076, "robustness": 90.9 }, "hard": { - "accuracy": 3.4, - "cost": 0.0004, + "accuracy": 12.9, + "cost": 0.0032, "robustness": 80.0 }, "all": { - "accuracy": 54.7, - "cost": 0.0004, - "robustness": 91.7 + "accuracy": 62.3, + "cost": 0.005, + "robustness": 87.5 } }, "subcategories": { "Arts": { "metrics": { "easy": { - "accuracy": 96.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 100.0, + "cost": 0.0043, + "robustness": 0.0 }, "medium": { - "accuracy": 41.0, - "cost": 0.0003, + "accuracy": 64.1, + "cost": 0.0023, "robustness": 100.0 }, "hard": { - "accuracy": 1.8, - "cost": 0.0004, + "accuracy": 5.4, + "cost": 0.0008, "robustness": 100.0 }, "all": { - "accuracy": 48.1, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 56.5, + "cost": 0.0025, + "robustness": 80.0 } } }, "Music": { "metrics": { "easy": { - "accuracy": 90.9, - "cost": 0.0001, + "accuracy": 97.0, + "cost": 0.0027, "robustness": 100.0 }, "medium": { - "accuracy": 38.8, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 39.7, + "cost": 0.0135, + "robustness": 80.0 }, "hard": { - "accuracy": 8.0, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 24.0, + "cost": 0.0159, + "robustness": 0.0 }, "all": { - "accuracy": 57.1, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 61.7, + "cost": 0.0093, + "robustness": 81.8 } } }, "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 87.9, - "cost": 0.0004, + "accuracy": 91.4, + "cost": 0.0041, "robustness": 100.0 }, "medium": { - "accuracy": 47.0, - "cost": 0.0006, - "robustness": 75.0 + "accuracy": 63.0, + "cost": 0.0028, + "robustness": 100.0 }, "hard": { - "accuracy": 3.0, - "cost": 0.0004, - "robustness": 50.0 + "accuracy": 15.2, + "cost": 0.0005, + "robustness": 100.0 }, "all": { - "accuracy": 56.2, - "cost": 0.0005, - "robustness": 75.0 + "accuracy": 65.7, + "cost": 0.0029, + "robustness": 100.0 } } } @@ -21512,72 +25240,72 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 95.5, - "cost": 0.0001, - "robustness": 96.2 + "accuracy": 97.4, + "cost": 0.0072, + "robustness": 34.6 }, "medium": { - "accuracy": 77.7, - "cost": 0.0006, - "robustness": 85.7 + "accuracy": 83.1, + "cost": 0.0138, + "robustness": 52.4 }, "hard": { - "accuracy": 51.8, - "cost": 0.0019, - "robustness": 53.3 + "accuracy": 51.4, + "cost": 0.0581, + "robustness": 80.0 }, "all": { - "accuracy": 82.0, - "cost": 0.0006, - "robustness": 82.3 + "accuracy": 84.6, + "cost": 0.0183, + "robustness": 51.6 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 96.2, - "cost": 0.0002, - "robustness": 92.3 + "accuracy": 98.1, + "cost": 0.0072, + "robustness": 61.5 }, "medium": { - "accuracy": 83.8, - "cost": 0.0006, - "robustness": 88.2 + "accuracy": 84.6, + "cost": 0.0151, + "robustness": 52.9 }, "hard": { - "accuracy": 59.3, - "cost": 0.0022, - "robustness": 46.2 + "accuracy": 56.5, + "cost": 0.0668, + "robustness": 92.3 }, "all": { - "accuracy": 83.9, - "cost": 0.0008, - "robustness": 76.7 + "accuracy": 84.4, + "cost": 0.0226, + "robustness": 67.4 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 94.5, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.4, + "cost": 0.0072, + "robustness": 7.7 }, "medium": { - "accuracy": 50.6, - "cost": 0.0001, - "robustness": 75.0 + "accuracy": 76.5, + "cost": 0.0079, + "robustness": 50.0 }, "hard": { - "accuracy": 8.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 22.2, + "cost": 0.0072, + "robustness": 0.0 }, "all": { - "accuracy": 77.3, - "cost": 0.0001, - "robustness": 94.7 + "accuracy": 85.4, + "cost": 0.0074, + "robustness": 15.8 } } } @@ -21586,96 +25314,96 @@ "History": { "metrics": { "easy": { - "accuracy": 93.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.0, + "cost": 0.0058, + "robustness": 78.3 }, "medium": { - "accuracy": 46.3, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 76.8, + "cost": 0.0064, + "robustness": 66.7 }, "hard": { - "accuracy": 4.0, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 22.8, + "cost": 0.0066, + "robustness": 90.0 }, "all": { - "accuracy": 62.4, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 75.6, + "cost": 0.0061, + "robustness": 79.5 } }, "subcategories": { "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 95.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.6, + "cost": 0.0048, + "robustness": 25.0 }, "medium": { - "accuracy": 33.3, - "cost": 0.0001, + "accuracy": 100.0, + "cost": 0.0055, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, + "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 82.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.2, + "cost": 0.0049, + "robustness": 25.0 } } }, "Geography": { "metrics": { "easy": { - "accuracy": 95.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.2, + "cost": 0.0032, + "robustness": 87.5 }, "medium": { - "accuracy": 44.8, - "cost": 0.0002, + "accuracy": 62.1, + "cost": 0.0015, "robustness": 100.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0004, + "accuracy": 25.0, + "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 78.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 84.7, + "cost": 0.0026, + "robustness": 88.9 } } }, "History": { "metrics": { "easy": { - "accuracy": 91.9, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 95.1, + "cost": 0.0073, + "robustness": 90.9 }, "medium": { - "accuracy": 47.5, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 78.4, + "cost": 0.0074, + "robustness": 60.0 }, "hard": { - "accuracy": 4.4, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 22.8, + "cost": 0.0073, + "robustness": 90.0 }, "all": { - "accuracy": 55.6, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 70.7, + "cost": 0.0073, + "robustness": 84.6 } } } @@ -21684,48 +25412,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 77.0, - "cost": 0.0001, + "accuracy": 90.4, + "cost": 0.0012, "robustness": 90.9 }, "medium": { - "accuracy": 35.3, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 51.5, + "cost": 0.0038, + "robustness": 88.9 }, "hard": { - "accuracy": 49.4, - "cost": 0.0002, - "robustness": 96.0 + "accuracy": 60.6, + "cost": 0.0026, + "robustness": 76.0 }, "all": { - "accuracy": 54.3, - "cost": 0.0002, - "robustness": 95.6 + "accuracy": 67.3, + "cost": 0.0025, + "robustness": 82.2 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 77.0, - "cost": 0.0001, + "accuracy": 90.4, + "cost": 0.0012, "robustness": 90.9 }, "medium": { - "accuracy": 35.3, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 51.5, + "cost": 0.0038, + "robustness": 88.9 }, "hard": { - "accuracy": 49.4, - "cost": 0.0002, - "robustness": 96.0 + "accuracy": 60.6, + "cost": 0.0026, + "robustness": 76.0 }, "all": { - "accuracy": 54.3, - "cost": 0.0002, - "robustness": 95.6 + "accuracy": 67.3, + "cost": 0.0025, + "robustness": 82.2 } } } @@ -21734,23 +25462,23 @@ "Literature": { "metrics": { "easy": { - "accuracy": 94.2, - "cost": 0.0001, + "accuracy": 100.0, + "cost": 0.0077, "robustness": 100.0 }, "medium": { - "accuracy": 28.2, - "cost": 0.0004, + "accuracy": 70.4, + "cost": 0.0067, "robustness": 100.0 }, "hard": { - "accuracy": 24.8, - "cost": 0.0003, + "accuracy": 31.5, + "cost": 0.0198, "robustness": 95.8 }, "all": { - "accuracy": 36.3, - "cost": 0.0003, + "accuracy": 46.4, + "cost": 0.0166, "robustness": 96.6 } }, @@ -21758,23 +25486,23 @@ "Literature, rhetoric and criticism": { "metrics": { "easy": { - "accuracy": 94.2, - "cost": 0.0001, + "accuracy": 100.0, + "cost": 0.0077, "robustness": 100.0 }, "medium": { - "accuracy": 28.2, - "cost": 0.0004, + "accuracy": 70.4, + "cost": 0.0067, "robustness": 100.0 }, "hard": { - "accuracy": 24.8, - "cost": 0.0003, + "accuracy": 31.5, + "cost": 0.0198, "robustness": 95.8 }, "all": { - "accuracy": 36.3, - "cost": 0.0003, + "accuracy": 46.4, + "cost": 0.0166, "robustness": 96.6 } } @@ -21784,72 +25512,72 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 94.6, - "cost": 0.0001, - "robustness": 96.3 + "accuracy": 94.1, + "cost": 0.0029, + "robustness": 70.4 }, "medium": { - "accuracy": 58.3, - "cost": 0.0002, + "accuracy": 69.2, + "cost": 0.0038, "robustness": 80.0 }, "hard": { - "accuracy": 7.4, - "cost": 0.0003, - "robustness": 85.7 + "accuracy": 23.5, + "cost": 0.0025, + "robustness": 71.4 }, "all": { - "accuracy": 73.6, - "cost": 0.0002, - "robustness": 90.9 + "accuracy": 78.4, + "cost": 0.0031, + "robustness": 72.7 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 93.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 90.3, + "cost": 0.0025, + "robustness": 71.4 }, "medium": { - "accuracy": 31.8, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 42.4, + "cost": 0.0051, + "robustness": 80.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 7.7, + "cost": 0.0059, + "robustness": 0.0 }, "all": { - "accuracy": 70.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 72.1, + "cost": 0.0034, + "robustness": 66.7 } } }, "Philosophical logic": { "metrics": { "easy": { - "accuracy": 98.5, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 100.0, + "cost": 0.0049, + "robustness": 75.0 }, "medium": { "accuracy": 92.1, - "cost": 0.0003, + "cost": 0.004, "robustness": 66.7 }, "hard": { - "accuracy": 80.0, - "cost": 0.0006, + "accuracy": 40.0, + "cost": 0.0087, "robustness": 0 }, "all": { - "accuracy": 95.4, - "cost": 0.0002, - "robustness": 85.7 + "accuracy": 94.4, + "cost": 0.0047, + "robustness": 71.4 } } }, @@ -21857,47 +25585,47 @@ "metrics": { "easy": { "accuracy": 94.6, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.0022, + "robustness": 33.3 }, "medium": { - "accuracy": 57.5, - "cost": 0.0003, - "robustness": 50.0 + "accuracy": 82.5, + "cost": 0.0024, + "robustness": 100.0 }, "hard": { - "accuracy": 4.4, - "cost": 0.0004, + "accuracy": 28.9, + "cost": 0.0011, "robustness": 100.0 }, "all": { - "accuracy": 49.2, - "cost": 0.0003, - "robustness": 87.5 + "accuracy": 66.4, + "cost": 0.0019, + "robustness": 75.0 } } }, "Psychology": { "metrics": { "easy": { - "accuracy": 95.0, - "cost": 0.0001, + "accuracy": 96.7, + "cost": 0.0026, "robustness": 83.3 }, "medium": { - "accuracy": 79.2, - "cost": 0.0002, + "accuracy": 87.5, + "cost": 0.0027, "robustness": 0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 16.7, + "cost": 0.0015, + "robustness": 100.0 }, "all": { - "accuracy": 81.8, - "cost": 0.0002, - "robustness": 75.0 + "accuracy": 86.6, + "cost": 0.0026, + "robustness": 87.5 } } } @@ -21906,37 +25634,37 @@ "Science": { "metrics": { "easy": { - "accuracy": 97.0, - "cost": 0.0001, - "robustness": 94.9 + "accuracy": 98.8, + "cost": 0.0045, + "robustness": 64.1 }, "medium": { - "accuracy": 72.5, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 82.4, + "cost": 0.0083, + "robustness": 66.7 }, "hard": { - "accuracy": 25.3, - "cost": 0.0006, - "robustness": 100.0 + "accuracy": 32.0, + "cost": 0.0104, + "robustness": 44.4 }, "all": { - "accuracy": 79.5, - "cost": 0.0002, - "robustness": 97.1 + "accuracy": 84.5, + "cost": 0.0065, + "robustness": 62.3 } }, "subcategories": { "Animals (Zoology)": { "metrics": { "easy": { - "accuracy": 95.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 100.0, + "cost": 0.0057, + "robustness": 0.0 }, "medium": { - "accuracy": 60.0, - "cost": 0.0001, + "accuracy": 80.0, + "cost": 0.009, "robustness": 0 }, "hard": { @@ -21945,9 +25673,9 @@ "robustness": 0 }, "all": { - "accuracy": 88.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.0, + "cost": 0.0063, + "robustness": 0.0 } } }, @@ -21955,119 +25683,119 @@ "metrics": { "easy": { "accuracy": 99.1, - "cost": 0.0001, - "robustness": 71.4 + "cost": 0.003, + "robustness": 85.7 }, "medium": { - "accuracy": 70.6, - "cost": 0.0002, + "accuracy": 82.4, + "cost": 0.0059, "robustness": 0 }, "hard": { - "accuracy": 22.2, + "accuracy": 33.3, "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 90.9, - "cost": 0.0002, - "robustness": 71.4 + "accuracy": 93.0, + "cost": 0.0032, + "robustness": 85.7 } } }, "Chemistry": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 97.7, + "cost": 0.0037, + "robustness": 66.7 }, "medium": { - "accuracy": 76.5, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 91.2, + "cost": 0.0082, + "robustness": 50.0 }, "hard": { - "accuracy": 23.1, - "cost": 0.0003, + "accuracy": 30.8, + "cost": 0.0078, "robustness": 0 }, "all": { - "accuracy": 80.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 85.6, + "cost": 0.006, + "robustness": 60.0 } } }, "Earth sciences and geology": { - "metrics": { - "easy": { - "accuracy": 96.5, - "cost": 0.0001, - "robustness": 100.0 + "metrics": { + "easy": { + "accuracy": 98.8, + "cost": 0.0037, + "robustness": 50.0 }, "medium": { - "accuracy": 59.3, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 74.6, + "cost": 0.003, + "robustness": 83.3 }, "hard": { - "accuracy": 7.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 15.4, + "cost": 0.0038, + "robustness": 0.0 }, "all": { - "accuracy": 86.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 91.2, + "cost": 0.0036, + "robustness": 60.0 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 96.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 98.9, + "cost": 0.0064, + "robustness": 63.6 }, "medium": { - "accuracy": 78.8, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 86.0, + "cost": 0.0106, + "robustness": 63.6 }, "hard": { - "accuracy": 43.8, - "cost": 0.0011, - "robustness": 100.0 + "accuracy": 47.2, + "cost": 0.0172, + "robustness": 60.0 }, "all": { - "accuracy": 81.2, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 85.9, + "cost": 0.0097, + "robustness": 63.0 } } }, "Physics": { "metrics": { "easy": { - "accuracy": 95.5, - "cost": 0.0001, + "accuracy": 97.7, + "cost": 0.0021, "robustness": 100.0 }, "medium": { - "accuracy": 69.0, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 81.0, + "cost": 0.0053, + "robustness": 50.0 }, "hard": { - "accuracy": 18.2, - "cost": 0.0002, + "accuracy": 63.6, + "cost": 0.0073, "robustness": 0 }, "all": { - "accuracy": 75.3, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 86.6, + "cost": 0.0041, + "robustness": 80.0 } } }, @@ -22075,23 +25803,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 100.0 + "cost": 0.0043, + "robustness": 60.0 }, "medium": { - "accuracy": 44.0, - "cost": 0.0002, + "accuracy": 56.0, + "cost": 0.0044, "robustness": 0 }, "hard": { - "accuracy": 3.4, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 6.8, + "cost": 0.0043, + "robustness": 33.3 }, "all": { - "accuracy": 37.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 42.1, + "cost": 0.0043, + "robustness": 50.0 } } } @@ -22100,24 +25828,24 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 94.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 94.6, + "cost": 0.0052, + "robustness": 72.2 }, "medium": { - "accuracy": 64.8, - "cost": 0.0003, - "robustness": 66.7 + "accuracy": 75.9, + "cost": 0.0096, + "robustness": 50.0 }, "hard": { - "accuracy": 7.4, - "cost": 0.0003, + "accuracy": 22.1, + "cost": 0.0112, "robustness": 85.7 }, "all": { - "accuracy": 65.0, - "cost": 0.0002, - "robustness": 86.5 + "accuracy": 72.4, + "cost": 0.0081, + "robustness": 67.6 } }, "subcategories": { @@ -22125,95 +25853,95 @@ "metrics": { "easy": { "accuracy": 96.2, - "cost": 0.0002, - "robustness": 100.0 + "cost": 0.0053, + "robustness": 75.0 }, "medium": { - "accuracy": 72.4, - "cost": 0.0003, - "robustness": 66.7 + "accuracy": 81.0, + "cost": 0.0107, + "robustness": 33.3 }, "hard": { - "accuracy": 10.0, - "cost": 0.0003, + "accuracy": 23.8, + "cost": 0.0167, "robustness": 66.7 }, "all": { - "accuracy": 70.9, - "cost": 0.0002, - "robustness": 85.7 + "accuracy": 76.8, + "cost": 0.0095, + "robustness": 61.9 } } }, "Law": { "metrics": { "easy": { - "accuracy": 86.2, - "cost": 0.0002, + "accuracy": 90.8, + "cost": 0.0072, "robustness": 100.0 }, "medium": { - "accuracy": 52.4, - "cost": 0.0003, - "robustness": 50.0 + "accuracy": 73.0, + "cost": 0.0101, + "robustness": 75.0 }, "hard": { - "accuracy": 6.7, - "cost": 0.0003, + "accuracy": 40.0, + "cost": 0.0095, "robustness": 100.0 }, "all": { - "accuracy": 57.6, - "cost": 0.0002, - "robustness": 77.8 + "accuracy": 74.1, + "cost": 0.0088, + "robustness": 88.9 } } }, "Social problems": { "metrics": { "easy": { - "accuracy": 97.9, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 93.6, + "cost": 0.0023, + "robustness": 33.3 }, "medium": { - "accuracy": 66.7, - "cost": 0.0001, + "accuracy": 22.2, + "cost": 0.0031, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, + "cost": 0.0033, "robustness": 100.0 }, "all": { - "accuracy": 85.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 75.4, + "cost": 0.0025, + "robustness": 50.0 } } }, "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 66.7, - "cost": 0.0001, + "accuracy": 100.0, + "cost": 0.0034, "robustness": 0 }, "medium": { - "accuracy": 38.9, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 66.7, + "cost": 0.0012, + "robustness": 50.0 }, "hard": { - "accuracy": 2.9, - "cost": 0.0003, + "accuracy": 5.9, + "cost": 0.0009, "robustness": 100.0 }, "all": { - "accuracy": 18.2, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 30.9, + "cost": 0.0011, + "robustness": 66.7 } } } @@ -22222,24 +25950,24 @@ "Technology": { "metrics": { "easy": { - "accuracy": 93.2, - "cost": 0.0001, - "robustness": 93.0 + "accuracy": 94.2, + "cost": 0.0043, + "robustness": 58.1 }, "medium": { - "accuracy": 62.6, - "cost": 0.0003, - "robustness": 90.5 + "accuracy": 74.0, + "cost": 0.0082, + "robustness": 57.1 }, "hard": { - "accuracy": 19.7, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 28.7, + "cost": 0.0069, + "robustness": 85.7 }, "all": { - "accuracy": 75.8, - "cost": 0.0002, - "robustness": 93.0 + "accuracy": 80.8, + "cost": 0.0058, + "robustness": 60.6 } }, "subcategories": { @@ -22247,71 +25975,71 @@ "metrics": { "easy": { "accuracy": 96.0, - "cost": 0.0002, - "robustness": 100.0 + "cost": 0.0077, + "robustness": 40.0 }, "medium": { - "accuracy": 79.0, - "cost": 0.0005, - "robustness": 84.6 + "accuracy": 83.2, + "cost": 0.0129, + "robustness": 53.8 }, "hard": { - "accuracy": 40.5, - "cost": 0.0007, + "accuracy": 52.4, + "cost": 0.0163, "robustness": 100.0 }, "all": { - "accuracy": 80.6, - "cost": 0.0004, - "robustness": 89.5 + "accuracy": 84.2, + "cost": 0.0114, + "robustness": 52.6 } } }, "Management and public relations": { "metrics": { "easy": { - "accuracy": 96.7, - "cost": 0.0001, - "robustness": 75.0 + "accuracy": 100.0, + "cost": 0.0044, + "robustness": 25.0 }, "medium": { "accuracy": 36.4, - "cost": 0.0001, + "cost": 0.0049, "robustness": 100.0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, + "cost": 0.0097, "robustness": 0 }, "all": { - "accuracy": 84.9, - "cost": 0.0001, - "robustness": 80.0 + "accuracy": 87.7, + "cost": 0.0047, + "robustness": 40.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 92.3, - "cost": 0.0001, - "robustness": 94.1 + "accuracy": 93.2, + "cost": 0.0036, + "robustness": 64.7 }, "medium": { - "accuracy": 52.5, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 69.4, + "cost": 0.0051, + "robustness": 57.1 }, "hard": { - "accuracy": 12.4, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 20.4, + "cost": 0.0034, + "robustness": 83.3 }, "all": { - "accuracy": 73.5, - "cost": 0.0001, - "robustness": 95.7 + "accuracy": 79.1, + "cost": 0.004, + "robustness": 66.0 } } } diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index 46dd371..48d200a 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -1,47 +1,80 @@ [ + { + "Router Name": "Paix2", + "Arena Score": 77.63, + "Optimal Selection Score": 89.67, + "Optimal Cost Score": 62.92, + "Optimal Acc. Score": 100.0, + "Robustness Score": 77.86, + "Latency Score": null, + "Accuracy": 79.69, + "Cost per 1k": 0.27 + }, + { + "Router Name": "Sqwish Router", + "Arena Score": 76.21, + "Optimal Selection Score": 9.04, + "Optimal Cost Score": 23.49, + "Optimal Acc. Score": 94.07, + "Robustness Score": 51.67, + "Latency Score": null, + "Accuracy": 79.76, + "Cost per 1k": 0.7 + }, + { + "Router Name": "Divyam", + "Arena Score": 75.85, + "Optimal Selection Score": 4.93, + "Optimal Cost Score": 16.03, + "Optimal Acc. Score": 93.43, + "Robustness Score": 98.33, + "Latency Score": null, + "Accuracy": 78.59, + "Cost per 1k": 0.48 + }, { "Router Name": "Cross-Router", - "Arena Score": 76.12, + "Arena Score": 75.75, "Optimal Selection Score": 17.66, "Optimal Cost Score": 45.49, "Optimal Acc. Score": 90.31, "Robustness Score": 67.14, "Latency Score": null, "Accuracy": 78.14, - "Cost per 1k": 0.3 + "Cost per 1k": 0.4 + }, + { + "Router Name": "Nadir-Tumbler", + "Arena Score": 75.17, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 66.43, + "Latency Score": null, + "Accuracy": 75.34, + "Cost per 1k": 0.08 }, { "Router Name": "vllm", - "Arena Score": 75.3, + "Arena Score": 74.86, "Optimal Selection Score": 16.81, "Optimal Cost Score": 25.1, "Optimal Acc. Score": 89.37, "Robustness Score": 67.62, "Latency Score": null, "Accuracy": 77.18, - "Cost per 1k": 0.3 + "Cost per 1k": 0.42 }, { - "Router Name": "Sqwish Router", - "Arena Score": 75.27, - "Optimal Selection Score": 7.41, - "Optimal Cost Score": 25.1, - "Optimal Acc. Score": 90.47, - "Robustness Score": 100.0, - "Latency Score": null, - "Accuracy": 76.4, - "Cost per 1k": 0.18 - }, - { - "Router Name": "Nadir-Tumbler", - "Arena Score": 75.17, + "Router Name": "nadir-caliper", + "Arena Score": 74.55, "Optimal Selection Score": null, "Optimal Cost Score": null, "Optimal Acc. Score": null, - "Robustness Score": 66.43, + "Robustness Score": 79.76, "Latency Score": null, - "Accuracy": 75.34, - "Cost per 1k": 0.08 + "Accuracy": 75.84, + "Cost per 1k": 0.22 }, { "Router Name": "AgentForge Router", @@ -54,6 +87,17 @@ "Accuracy": 74.72, "Cost per 1k": 0.13 }, + { + "Router Name": "BARouter", + "Arena Score": 73.79, + "Optimal Selection Score": 64.41, + "Optimal Cost Score": 67.13, + "Optimal Acc. Score": 93.84, + "Robustness Score": 68.81, + "Latency Score": null, + "Accuracy": 75.72, + "Cost per 1k": 0.36 + }, { "Router Name": "Weave Router", "Arena Score": 72.82, @@ -164,17 +208,6 @@ "Accuracy": 68.41, "Cost per 1k": 0.29 }, - { - "Router Name": "BARouter", - "Arena Score": 67.09, - "Optimal Selection Score": 58.56, - "Optimal Cost Score": 59.03, - "Optimal Acc. Score": 87.91, - "Robustness Score": 52.38, - "Latency Score": null, - "Accuracy": 68.8, - "Cost per 1k": 0.63 - }, { "Router Name": "mirt_bert", "Arena Score": 66.89, @@ -197,6 +230,17 @@ "Accuracy": 66.34, "Cost per 1k": 0.21 }, + { + "Router Name": "AsiaInfo-Router", + "Arena Score": 65.87, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 69.52, + "Latency Score": null, + "Accuracy": 75.2, + "Cost per 1k": 8.54 + }, { "Router Name": "gpt5", "Arena Score": 64.32, diff --git a/src/data/routers.json b/src/data/routers.json index f2d9d2b..b1aa659 100644 --- a/src/data/routers.json +++ b/src/data/routers.json @@ -486,5 +486,55 @@ "qwen/qwen3-235b-a22b-2507", "z-ai/glm-4.7" ] + }, + "nadir-caliper": { + "name": "nadir-caliper", + "affiliation": "@doramirdor", + "githubUrl": "https://github.com/doramirdor", + "type": "open-source", + "description": "Submitted by @doramirdor.", + "modelPool": [ + "qwen/qwen3-235b-a22b-2507", + "google/gemini-3.1-flash-lite", + "grok-4.3", + "deepseek/deepseek-v3.2" + ] + }, + "Paix2": { + "name": "Paix2", + "affiliation": "@xufan866", + "githubUrl": "https://github.com/xufan866", + "type": "open-source", + "description": "Submitted by @xufan866.", + "modelPool": [ + "MiniMax-M3", + "agnes-2.0-flash", + "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B", + "THUDM/GLM-4-9B-0414" + ] + }, + "Divyam": { + "name": "Divyam", + "affiliation": "@samikd", + "githubUrl": "https://github.com/samikd", + "type": "open-source", + "description": "Submitted by @samikd.", + "modelPool": [ + "qwen/qwen3-235b-a22b-2507", + "Qwen/Qwen3-Coder-Next", + "gemini-3-flash-preview", + "google/gemini-3.1-flash-lite" + ] + }, + "AsiaInfo-Router": { + "name": "AsiaInfo-Router", + "affiliation": "@Uncle-LL", + "githubUrl": "https://github.com/Uncle-LL", + "type": "open-source", + "description": "Submitted by @Uncle-LL.", + "modelPool": [ + "claude-opus-4-7", + "deepseek/deepseek-v4-flash" + ] } }