| {"./data_dir/eval_vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.664, "ndcg_at_3": 0.71728, "ndcg_at_5": 0.73063, "ndcg_at_10": 0.75071, "ndcg_at_20": 0.76029, "ndcg_at_50": 0.77029, "ndcg_at_100": 0.77647, "map_at_1": 0.664, "map_at_3": 0.70333, "map_at_5": 0.71083, "map_at_10": 0.71913, "map_at_20": 0.72174, "map_at_50": 0.72338, "map_at_100": 0.72393, "recall_at_1": 0.664, "recall_at_3": 0.758, "recall_at_5": 0.79, "recall_at_10": 0.852, "recall_at_20": 0.89, "recall_at_50": 0.94, "recall_at_100": 0.978, "precision_at_1": 0.664, "precision_at_3": 0.25267, "precision_at_5": 0.158, "precision_at_10": 0.0852, "precision_at_20": 0.0445, "precision_at_50": 0.0188, "precision_at_100": 0.00978, "mrr_at_1": 0.666, "mrr_at_3": 0.7036666666666667, "mrr_at_5": 0.7110666666666667, "mrr_at_10": 0.719484126984127, "mrr_at_20": 0.722312611975228, "mrr_at_50": 0.7237972836422734, "mrr_at_100": 0.7243297450156544, "naucs_at_1_max": 0.5889213023056643, "naucs_at_1_std": -0.03738843019443627, "naucs_at_1_diff1": 0.9044815265827187, "naucs_at_3_max": 0.6597876116843661, "naucs_at_3_std": 0.041596651295516755, "naucs_at_3_diff1": 0.8645218684327024, "naucs_at_5_max": 0.6839060283261598, "naucs_at_5_std": 0.11626710270036247, "naucs_at_5_diff1": 0.8423112992040783, "naucs_at_10_max": 0.7812541175850534, "naucs_at_10_std": 0.21647297546578018, "naucs_at_10_diff1": 0.8277202413893048, "naucs_at_20_max": 0.7649600549780963, "naucs_at_20_std": 0.12761790224207298, "naucs_at_20_diff1": 0.7988574864702351, "naucs_at_50_max": 0.7467787114845937, "naucs_at_50_std": 0.2255835667600402, "naucs_at_50_diff1": 0.7687519452225302, "naucs_at_100_max": 0.8461505814447046, "naucs_at_100_std": 0.3166539343009966, "naucs_at_100_diff1": 0.8011628893981787}, "./data_dir/eval_vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.4745, "ndcg_at_3": 0.54173, "ndcg_at_5": 0.56769, "ndcg_at_10": 0.5861, "ndcg_at_20": 0.6048, "ndcg_at_50": 0.61661, "ndcg_at_100": 0.62385, "map_at_1": 0.4745, "map_at_3": 0.52513, "map_at_5": 0.53976, "map_at_10": 0.54722, "map_at_20": 0.55245, "map_at_50": 0.55432, "map_at_100": 0.55497, "recall_at_1": 0.4745, "recall_at_3": 0.5898, "recall_at_5": 0.65188, "recall_at_10": 0.70953, "recall_at_20": 0.78271, "recall_at_50": 0.84257, "recall_at_100": 0.88692, "precision_at_1": 0.4745, "precision_at_3": 0.1966, "precision_at_5": 0.13038, "precision_at_10": 0.07095, "precision_at_20": 0.03914, "precision_at_50": 0.01685, "precision_at_100": 0.00887, "mrr_at_1": 0.4722838137472284, "mrr_at_3": 0.5240206947524021, "mrr_at_5": 0.5384331116038433, "mrr_at_10": 0.545998310632457, "mrr_at_20": 0.5506564713842315, "mrr_at_50": 0.5529029054852187, "mrr_at_100": 0.5535124417758647, "naucs_at_1_max": 0.24308198716486704, "naucs_at_1_std": 0.4263216056628872, "naucs_at_1_diff1": 0.8579772747628749, "naucs_at_3_max": 0.16805600057176534, "naucs_at_3_std": 0.3924645370055808, "naucs_at_3_diff1": 0.7770233728520576, "naucs_at_5_max": 0.1218197178734784, "naucs_at_5_std": 0.4817802245485417, "naucs_at_5_diff1": 0.7532614461045579, "naucs_at_10_max": 0.03428092836449581, "naucs_at_10_std": 0.5151585952399194, "naucs_at_10_diff1": 0.7143986620927637, "naucs_at_20_max": -0.13960806153123748, "naucs_at_20_std": 0.6261345359875006, "naucs_at_20_diff1": 0.6826006385134176, "naucs_at_50_max": -0.2400541568650652, "naucs_at_50_std": 0.7530843562467058, "naucs_at_50_diff1": 0.6593166140032869, "naucs_at_100_max": -0.08884413440903777, "naucs_at_100_std": 0.7993547173030003, "naucs_at_100_diff1": 0.6944870252918233}, "./data_dir/eval_vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.91, "ndcg_at_3": 0.93262, "ndcg_at_5": 0.93693, "ndcg_at_10": 0.93994, "ndcg_at_20": 0.94494, "ndcg_at_50": 0.94702, "ndcg_at_100": 0.94702, "map_at_1": 0.91, "map_at_3": 0.92667, "map_at_5": 0.92917, "map_at_10": 0.93028, "map_at_20": 0.93161, "map_at_50": 0.93198, "map_at_100": 0.93198, "recall_at_1": 0.91, "recall_at_3": 0.95, "recall_at_5": 0.96, "recall_at_10": 0.97, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.91, "precision_at_3": 0.31667, "precision_at_5": 0.192, "precision_at_10": 0.097, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.91, "mrr_at_3": 0.9283333333333332, "mrr_at_5": 0.9308333333333333, "mrr_at_10": 0.9319444444444445, "mrr_at_20": 0.9332777777777779, "mrr_at_50": 0.9336944444444444, "mrr_at_100": 0.9336944444444444, "naucs_at_1_max": 0.3011723207801644, "naucs_at_1_std": -0.4870837223778408, "naucs_at_1_diff1": 0.8742089428363928, "naucs_at_3_max": 0.730158730158726, "naucs_at_3_std": -0.21027077497665203, "naucs_at_3_diff1": 0.9183006535947692, "naucs_at_5_max": 0.6953781512605006, "naucs_at_5_std": -0.48015873015872135, "naucs_at_5_diff1": 0.8978758169934612, "naucs_at_10_max": 0.807812013694364, "naucs_at_10_std": -0.06022408963585601, "naucs_at_10_diff1": 0.8638344226579531, "naucs_at_20_max": 0.5541549953314738, "naucs_at_20_std": -0.1713352007469681, "naucs_at_20_diff1": 0.8692810457516413, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tatdqa_test": {"ndcg_at_1": 0.64095, "ndcg_at_3": 0.736, "ndcg_at_5": 0.7601, "ndcg_at_10": 0.7803, "ndcg_at_20": 0.78827, "ndcg_at_50": 0.79223, "ndcg_at_100": 0.7941, "map_at_1": 0.64095, "map_at_3": 0.71203, "map_at_5": 0.72546, "map_at_10": 0.734, "map_at_20": 0.73618, "map_at_50": 0.73687, "map_at_100": 0.73703, "recall_at_1": 0.64095, "recall_at_3": 0.80559, "recall_at_5": 0.86391, "recall_at_10": 0.92527, "recall_at_20": 0.95687, "recall_at_50": 0.97631, "recall_at_100": 0.98785, "precision_at_1": 0.64095, "precision_at_3": 0.26853, "precision_at_5": 0.17278, "precision_at_10": 0.09253, "precision_at_20": 0.04784, "precision_at_50": 0.01953, "precision_at_100": 0.00988, "mrr_at_1": 0.6354799513973268, "mrr_at_3": 0.7106115836371001, "mrr_at_5": 0.7234305386796274, "mrr_at_10": 0.731879833747999, "mrr_at_20": 0.7340755638044026, "mrr_at_50": 0.7348432926756366, "mrr_at_100": 0.7350248567706289, "naucs_at_1_max": 0.19811506816235372, "naucs_at_1_std": -0.2554693200419591, "naucs_at_1_diff1": 0.7675648256745395, "naucs_at_3_max": 0.2318301363088243, "naucs_at_3_std": -0.27759660815315446, "naucs_at_3_diff1": 0.6712729518538324, "naucs_at_5_max": 0.24793757976069947, "naucs_at_5_std": -0.2400954177562814, "naucs_at_5_diff1": 0.6522915581849575, "naucs_at_10_max": 0.33999905443604334, "naucs_at_10_std": -0.09243929665705147, "naucs_at_10_diff1": 0.6186302038158574, "naucs_at_20_max": 0.33414218317326, "naucs_at_20_std": 0.10107766265773643, "naucs_at_20_diff1": 0.5589735061846622, "naucs_at_50_max": 0.39336569274053196, "naucs_at_50_std": 0.3146553201604068, "naucs_at_50_diff1": 0.5632664235073339, "naucs_at_100_max": 0.6769613453644149, "naucs_at_100_std": 0.7790878478446186, "naucs_at_100_diff1": 0.6726227816477444}, "./data_dir/eval_vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.7834, "ndcg_at_3": 0.83083, "ndcg_at_5": 0.84232, "ndcg_at_10": 0.85157, "ndcg_at_20": 0.85985, "ndcg_at_50": 0.86443, "ndcg_at_100": 0.8657, "map_at_1": 0.7834, "map_at_3": 0.81984, "map_at_5": 0.82611, "map_at_10": 0.82998, "map_at_20": 0.8323, "map_at_50": 0.83311, "map_at_100": 0.83321, "recall_at_1": 0.7834, "recall_at_3": 0.86235, "recall_at_5": 0.89069, "recall_at_10": 0.91903, "recall_at_20": 0.95142, "recall_at_50": 0.97368, "recall_at_100": 0.98178, "precision_at_1": 0.7834, "precision_at_3": 0.28745, "precision_at_5": 0.17814, "precision_at_10": 0.0919, "precision_at_20": 0.04757, "precision_at_50": 0.01947, "precision_at_100": 0.00982, "mrr_at_1": 0.7813765182186235, "mrr_at_3": 0.8188259109311741, "mrr_at_5": 0.8248987854251012, "mrr_at_10": 0.829565098644046, "mrr_at_20": 0.8313956960718094, "mrr_at_50": 0.8322900405575263, "mrr_at_100": 0.832391579382424, "naucs_at_1_max": 0.518447362653061, "naucs_at_1_std": 0.025991835344041193, "naucs_at_1_diff1": 0.9010494190422811, "naucs_at_3_max": 0.5691970896074877, "naucs_at_3_std": 0.020323756354481724, "naucs_at_3_diff1": 0.8408073942635276, "naucs_at_5_max": 0.5665846312895253, "naucs_at_5_std": 0.08497408554034125, "naucs_at_5_diff1": 0.82223001004187, "naucs_at_10_max": 0.6452921091747841, "naucs_at_10_std": 0.19809424135208908, "naucs_at_10_diff1": 0.8431957937117052, "naucs_at_20_max": 0.7447325753492595, "naucs_at_20_std": 0.4528568090604771, "naucs_at_20_diff1": 0.8030103604465574, "naucs_at_50_max": 0.8343240898820317, "naucs_at_50_std": 0.6770625322907705, "naucs_at_50_diff1": 0.8472448651285527, "naucs_at_100_max": 0.8102131093810145, "naucs_at_100_std": 0.7381222519798937, "naucs_at_100_diff1": 0.8742471393840519}, "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.96786, "ndcg_at_5": 0.96786, "ndcg_at_10": 0.96786, "ndcg_at_20": 0.96786, "ndcg_at_50": 0.96786, "ndcg_at_100": 0.96786, "map_at_1": 0.92, "map_at_3": 0.95667, "map_at_5": 0.95667, "map_at_10": 0.95667, "map_at_20": 0.95667, "map_at_50": 0.95667, "map_at_100": 0.95667, "recall_at_1": 0.92, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.9666666666666667, "mrr_at_5": 0.9666666666666667, "mrr_at_10": 0.9666666666666667, "mrr_at_20": 0.9666666666666667, "mrr_at_50": 0.9666666666666667, "mrr_at_100": 0.9666666666666667, "naucs_at_1_max": 0.7619047619047616, "naucs_at_1_std": 0.24060457516339795, "naucs_at_1_diff1": 0.9162581699346404, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.56786, "ndcg_at_3": 0.62806, "ndcg_at_5": 0.65635, "ndcg_at_10": 0.67295, "ndcg_at_20": 0.68721, "ndcg_at_50": 0.70267, "ndcg_at_100": 0.71485, "map_at_1": 0.56786, "map_at_3": 0.6131, "map_at_5": 0.62899, "map_at_10": 0.63542, "map_at_20": 0.63924, "map_at_50": 0.64167, "map_at_100": 0.64292, "recall_at_1": 0.56786, "recall_at_3": 0.67143, "recall_at_5": 0.73929, "recall_at_10": 0.79286, "recall_at_20": 0.85, "recall_at_50": 0.92857, "recall_at_100": 1.0, "precision_at_1": 0.56786, "precision_at_3": 0.22381, "precision_at_5": 0.14786, "precision_at_10": 0.07929, "precision_at_20": 0.0425, "precision_at_50": 0.01857, "precision_at_100": 0.01, "mrr_at_1": 0.5678571428571428, "mrr_at_3": 0.6113095238095237, "mrr_at_5": 0.6277380952380953, "mrr_at_10": 0.6339356575963718, "mrr_at_20": 0.6379618770431741, "mrr_at_50": 0.6403947405923297, "mrr_at_100": 0.6416450055922677, "naucs_at_1_max": 0.15868730409527165, "naucs_at_1_std": 0.005476384052524141, "naucs_at_1_diff1": 0.6898556504931816, "naucs_at_3_max": 0.2265285828624092, "naucs_at_3_std": 0.07120249326962727, "naucs_at_3_diff1": 0.6460005820862944, "naucs_at_5_max": 0.18104774909508717, "naucs_at_5_std": 0.049974351749500545, "naucs_at_5_diff1": 0.599728345290476, "naucs_at_10_max": 0.08937034529451629, "naucs_at_10_std": 0.021830460219087747, "naucs_at_10_diff1": 0.5156241862403002, "naucs_at_20_max": -0.015480451861837658, "naucs_at_20_std": -0.04558133048207932, "naucs_at_20_diff1": 0.5020338431500163, "naucs_at_50_max": 0.02670401493930801, "naucs_at_50_std": -0.1887955182072852, "naucs_at_50_diff1": 0.31573295985060545, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.88, "ndcg_at_3": 0.92786, "ndcg_at_5": 0.94421, "ndcg_at_10": 0.94421, "ndcg_at_20": 0.94421, "ndcg_at_50": 0.94421, "ndcg_at_100": 0.94421, "map_at_1": 0.88, "map_at_3": 0.91667, "map_at_5": 0.92567, "map_at_10": 0.92567, "map_at_20": 0.92567, "map_at_50": 0.92567, "map_at_100": 0.92567, "recall_at_1": 0.88, "recall_at_3": 0.96, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.88, "precision_at_3": 0.32, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.91, "mrr_at_3": 0.9333333333333332, "mrr_at_5": 0.9423333333333334, "mrr_at_10": 0.9423333333333334, "mrr_at_20": 0.9423333333333334, "mrr_at_50": 0.9423333333333334, "mrr_at_100": 0.9423333333333334, "naucs_at_1_max": 0.45863824371619505, "naucs_at_1_std": 0.23079064587973264, "naucs_at_1_diff1": 0.8033725739739099, "naucs_at_3_max": 0.8068394024276336, "naucs_at_3_std": 0.5087535014005626, "naucs_at_3_diff1": 0.6038748832866443, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/shiftproject_test": {"ndcg_at_1": 0.45, "ndcg_at_3": 0.59464, "ndcg_at_5": 0.63982, "ndcg_at_10": 0.66889, "ndcg_at_20": 0.67709, "ndcg_at_50": 0.68517, "ndcg_at_100": 0.68853, "map_at_1": 0.45, "map_at_3": 0.55833, "map_at_5": 0.58333, "map_at_10": 0.59531, "map_at_20": 0.59789, "map_at_50": 0.59924, "map_at_100": 0.59957, "recall_at_1": 0.45, "recall_at_3": 0.7, "recall_at_5": 0.81, "recall_at_10": 0.9, "recall_at_20": 0.93, "recall_at_50": 0.97, "recall_at_100": 0.99, "precision_at_1": 0.45, "precision_at_3": 0.23333, "precision_at_5": 0.162, "precision_at_10": 0.09, "precision_at_20": 0.0465, "precision_at_50": 0.0194, "precision_at_100": 0.0099, "mrr_at_1": 0.47, "mrr_at_3": 0.5833333333333334, "mrr_at_5": 0.6018333333333333, "mrr_at_10": 0.615718253968254, "mrr_at_20": 0.6165515873015873, "mrr_at_50": 0.6179859646889059, "mrr_at_100": 0.6183226650256062, "naucs_at_1_max": 0.105925489425784, "naucs_at_1_std": -0.060066784521705045, "naucs_at_1_diff1": 0.49797682184246717, "naucs_at_3_max": 0.11120518825436876, "naucs_at_3_std": -0.10580075662042858, "naucs_at_3_diff1": 0.46674473067915695, "naucs_at_5_max": 0.03777044371103874, "naucs_at_5_std": -0.21048771543820866, "naucs_at_5_diff1": 0.38344072502488297, "naucs_at_10_max": 0.09103641456582774, "naucs_at_10_std": -0.14047619047618726, "naucs_at_10_diff1": 0.32413632119514585, "naucs_at_20_max": -0.0814992663732126, "naucs_at_20_std": -0.3702147525676927, "naucs_at_20_diff1": 0.34020274776577397, "naucs_at_50_max": -0.20401493930905526, "naucs_at_50_std": -0.5308123249299683, "naucs_at_50_diff1": 0.7860255213196357, "naucs_at_100_max": -0.5634920634920583, "naucs_at_100_std": -0.5634920634920583, "naucs_at_100_diff1": 0.35807656395892007}, "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95786, "ndcg_at_5": 0.96172, "ndcg_at_10": 0.96172, "ndcg_at_20": 0.96172, "ndcg_at_50": 0.96172, "ndcg_at_100": 0.96332, "map_at_1": 0.92, "map_at_3": 0.95, "map_at_5": 0.952, "map_at_10": 0.952, "map_at_20": 0.952, "map_at_50": 0.952, "map_at_100": 0.95213, "recall_at_1": 0.92, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.92, "mrr_at_3": 0.95, "mrr_at_5": 0.9525, "mrr_at_10": 0.9525, "mrr_at_20": 0.9525, "mrr_at_50": 0.9525, "mrr_at_100": 0.9526315789473684, "naucs_at_1_max": 0.749649859943977, "naucs_at_1_std": 0.25116713352007414, "naucs_at_1_diff1": 0.9279295051353874, "naucs_at_3_max": 0.8611111111111119, "naucs_at_3_std": 0.6790382819794457, "naucs_at_3_diff1": 0.7117180205415458, "naucs_at_5_max": 0.7222222222222276, "naucs_at_5_std": 0.35807656395891135, "naucs_at_5_diff1": 0.8692810457516413, "naucs_at_10_max": 0.7222222222222276, "naucs_at_10_std": 0.35807656395891135, "naucs_at_10_diff1": 0.8692810457516413, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 0.8692810457516413, "naucs_at_50_max": 0.7222222222222041, "naucs_at_50_std": 0.35807656395892007, "naucs_at_50_diff1": 0.8692810457516374, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}} |