--- base_model: - Alibaba-NLP/gte-Qwen2-7B-instruct language: - en - zh model-index: - name: infly/inf-retriever-v1 results: - dataset: config: default name: MTEB CmedqaRetrieval (default) revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301 split: dev type: C-MTEB/CmedqaRetrieval metrics: - type: ndcg_at_1 value: 38.21 - type: ndcg_at_3 value: 38.438 - type: ndcg_at_5 value: 40.443 - type: ndcg_at_10 value: 43.306 - type: ndcg_at_20 value: 46.183 - type: ndcg_at_100 value: 50.646 - type: ndcg_at_1000 value: 52.82 - type: map_at_1 value: 25.153 - type: map_at_3 value: 33.038000000000004 - type: map_at_5 value: 35.228 - type: map_at_10 value: 36.924 - type: map_at_20 value: 37.96 - type: map_at_100 value: 38.792 - type: map_at_1000 value: 38.914 - type: recall_at_1 value: 25.153 - type: recall_at_3 value: 38.253 - type: recall_at_5 value: 44.507999999999996 - type: recall_at_10 value: 53.012 - type: recall_at_20 value: 62.915 - type: recall_at_100 value: 83.487 - type: recall_at_1000 value: 98.059 - type: precision_at_1 value: 38.21 - type: precision_at_3 value: 21.605 - type: precision_at_5 value: 15.664 - type: precision_at_10 value: 9.585 - type: precision_at_20 value: 5.731 - type: precision_at_100 value: 1.545 - type: precision_at_1000 value: 0.183 - type: mrr_at_1 value: 38.2096 - type: mrr_at_3 value: 43.6776 - type: mrr_at_5 value: 44.992900000000006 - type: mrr_at_10 value: 46.0602 - type: mrr_at_20 value: 46.6425 - type: mrr_at_100 value: 47.0609 - type: mrr_at_1000 value: 47.1081 - type: nauc_ndcg_at_1_max value: 47.2941 - type: nauc_ndcg_at_1_std value: 1.1506 - type: nauc_ndcg_at_1_diff1 value: 65.07300000000001 - type: nauc_ndcg_at_3_max value: 40.0306 - type: nauc_ndcg_at_3_std value: -2.1843999999999997 - type: nauc_ndcg_at_3_diff1 value: 55.63550000000001 - type: nauc_ndcg_at_5_max value: 38.4437 - type: nauc_ndcg_at_5_std value: -2.8594 - type: nauc_ndcg_at_5_diff1 value: 54.3432 - type: nauc_ndcg_at_10_max value: 37.9058 - type: nauc_ndcg_at_10_std value: -2.2464 - type: nauc_ndcg_at_10_diff1 value: 53.2127 - type: nauc_ndcg_at_20_max value: 38.6325 - type: nauc_ndcg_at_20_std value: -1.4661 - type: nauc_ndcg_at_20_diff1 value: 53.0613 - type: nauc_ndcg_at_100_max value: 40.2745 - type: nauc_ndcg_at_100_std value: 1.2013 - type: nauc_ndcg_at_100_diff1 value: 53.5367 - type: nauc_ndcg_at_1000_max value: 40.773199999999996 - type: nauc_ndcg_at_1000_std value: 0.4336 - type: nauc_ndcg_at_1000_diff1 value: 54.4837 - type: nauc_map_at_1_max value: 29.7667 - type: nauc_map_at_1_std value: -6.5036 - type: nauc_map_at_1_diff1 value: 55.916 - type: nauc_map_at_3_max value: 34.8091 - type: nauc_map_at_3_std value: -5.9474 - type: nauc_map_at_3_diff1 value: 54.0836 - type: nauc_map_at_5_max value: 35.9765 - type: nauc_map_at_5_std value: -5.1715 - type: nauc_map_at_5_diff1 value: 53.89960000000001 - type: nauc_map_at_10_max value: 36.7189 - type: nauc_map_at_10_std value: -4.2277000000000005 - type: nauc_map_at_10_diff1 value: 53.570499999999996 - type: nauc_map_at_20_max value: 37.327799999999996 - type: nauc_map_at_20_std value: -3.7319 - type: nauc_map_at_20_diff1 value: 53.662299999999995 - type: nauc_map_at_100_max value: 37.8037 - type: nauc_map_at_100_std value: -3.0759000000000003 - type: nauc_map_at_100_diff1 value: 53.8127 - type: nauc_map_at_1000_max value: 37.851800000000004 - type: nauc_map_at_1000_std value: -3.0761 - type: nauc_map_at_1000_diff1 value: 53.8489 - type: nauc_recall_at_1_max value: 29.7667 - type: nauc_recall_at_1_std value: -6.5036 - type: nauc_recall_at_1_diff1 value: 55.916 - type: nauc_recall_at_3_max value: 30.2726 - type: nauc_recall_at_3_std value: -6.4455 - type: nauc_recall_at_3_diff1 value: 47.646 - type: nauc_recall_at_5_max value: 29.5304 - type: nauc_recall_at_5_std value: -5.3595 - type: nauc_recall_at_5_diff1 value: 44.6824 - type: nauc_recall_at_10_max value: 28.063399999999998 - type: nauc_recall_at_10_std value: -2.122 - type: nauc_recall_at_10_diff1 value: 40.477999999999994 - type: nauc_recall_at_20_max value: 27.8269 - type: nauc_recall_at_20_std value: -0.0429 - type: nauc_recall_at_20_diff1 value: 37.2018 - type: nauc_recall_at_100_max value: 30.131000000000004 - type: nauc_recall_at_100_std value: 19.334899999999998 - type: nauc_recall_at_100_diff1 value: 29.274499999999996 - type: nauc_recall_at_1000_max value: 33.964 - type: nauc_recall_at_1000_std value: 53.2129 - type: nauc_recall_at_1000_diff1 value: 19.7514 - type: nauc_precision_at_1_max value: 47.2941 - type: nauc_precision_at_1_std value: 1.1506 - type: nauc_precision_at_1_diff1 value: 65.07300000000001 - type: nauc_precision_at_3_max value: 45.4966 - type: nauc_precision_at_3_std value: 5.432 - type: nauc_precision_at_3_diff1 value: 44.0588 - type: nauc_precision_at_5_max value: 43.9257 - type: nauc_precision_at_5_std value: 8.1048 - type: nauc_precision_at_5_diff1 value: 37.5711 - type: nauc_precision_at_10_max value: 41.274499999999996 - type: nauc_precision_at_10_std value: 12.023399999999999 - type: nauc_precision_at_10_diff1 value: 29.940299999999997 - type: nauc_precision_at_20_max value: 39.8018 - type: nauc_precision_at_20_std value: 15.2937 - type: nauc_precision_at_20_diff1 value: 24.4872 - type: nauc_precision_at_100_max value: 35.012 - type: nauc_precision_at_100_std value: 24.002499999999998 - type: nauc_precision_at_100_diff1 value: 13.2948 - type: nauc_precision_at_1000_max value: 29.172700000000003 - type: nauc_precision_at_1000_std value: 20.634900000000002 - type: nauc_precision_at_1000_diff1 value: 6.460000000000001 - type: nauc_mrr_at_1_max value: 47.2941 - type: nauc_mrr_at_1_std value: 1.1506 - type: nauc_mrr_at_1_diff1 value: 65.07300000000001 - type: nauc_mrr_at_3_max value: 45.6503 - type: nauc_mrr_at_3_std value: 0.5387000000000001 - type: nauc_mrr_at_3_diff1 value: 61.84889999999999 - type: nauc_mrr_at_5_max value: 45.3089 - type: nauc_mrr_at_5_std value: 0.7656 - type: nauc_mrr_at_5_diff1 value: 61.148 - type: nauc_mrr_at_10_max value: 45.2298 - type: nauc_mrr_at_10_std value: 1.15 - type: nauc_mrr_at_10_diff1 value: 60.775800000000004 - type: nauc_mrr_at_20_max value: 45.2277 - type: nauc_mrr_at_20_std value: 1.1927999999999999 - type: nauc_mrr_at_20_diff1 value: 60.6927 - type: nauc_mrr_at_100_max value: 45.3238 - type: nauc_mrr_at_100_std value: 1.3096999999999999 - type: nauc_mrr_at_100_diff1 value: 60.7616 - type: nauc_mrr_at_1000_max value: 45.3298 - type: nauc_mrr_at_1000_std value: 1.2893 - type: nauc_mrr_at_1000_diff1 value: 60.7901 - type: main_score value: 43.306 task: type: Retrieval - dataset: config: default name: MTEB CovidRetrieval (default) revision: 1271c7809071a13532e05f25fb53511ffce77117 split: dev type: C-MTEB/CovidRetrieval metrics: - type: ndcg_at_1 value: 77.97699999999999 - type: ndcg_at_3 value: 84.489 - type: ndcg_at_5 value: 86.18299999999999 - type: ndcg_at_10 value: 87.15899999999999 - type: ndcg_at_20 value: 87.638 - type: ndcg_at_100 value: 87.976 - type: ndcg_at_1000 value: 88.02600000000001 - type: map_at_1 value: 77.713 - type: map_at_3 value: 82.947 - type: map_at_5 value: 83.909 - type: map_at_10 value: 84.332 - type: map_at_20 value: 84.47200000000001 - type: map_at_100 value: 84.52 - type: map_at_1000 value: 84.52199999999999 - type: recall_at_1 value: 77.713 - type: recall_at_3 value: 88.857 - type: recall_at_5 value: 92.914 - type: recall_at_10 value: 95.838 - type: recall_at_20 value: 97.682 - type: recall_at_100 value: 99.473 - type: recall_at_1000 value: 99.895 - type: precision_at_1 value: 77.97699999999999 - type: precision_at_3 value: 29.855999999999998 - type: precision_at_5 value: 18.756999999999998 - type: precision_at_10 value: 9.684 - type: precision_at_20 value: 4.936999999999999 - type: precision_at_100 value: 1.005 - type: precision_at_1000 value: 0.101 - type: mrr_at_1 value: 77.9768 - type: mrr_at_3 value: 82.947 - type: mrr_at_5 value: 83.9322 - type: mrr_at_10 value: 84.3297 - type: mrr_at_20 value: 84.4602 - type: mrr_at_100 value: 84.5085 - type: mrr_at_1000 value: 84.5098 - type: nauc_ndcg_at_1_max value: 10.6257 - type: nauc_ndcg_at_1_std value: -37.586999999999996 - type: nauc_ndcg_at_1_diff1 value: 87.6739 - type: nauc_ndcg_at_3_max value: 11.4731 - type: nauc_ndcg_at_3_std value: -39.4782 - type: nauc_ndcg_at_3_diff1 value: 85.1518 - type: nauc_ndcg_at_5_max value: 13.3553 - type: nauc_ndcg_at_5_std value: -38.0635 - type: nauc_ndcg_at_5_diff1 value: 84.7753 - type: nauc_ndcg_at_10_max value: 13.5939 - type: nauc_ndcg_at_10_std value: -35.3862 - type: nauc_ndcg_at_10_diff1 value: 85.2025 - type: nauc_ndcg_at_20_max value: 12.3043 - type: nauc_ndcg_at_20_std value: -34.4532 - type: nauc_ndcg_at_20_diff1 value: 85.4869 - type: nauc_ndcg_at_100_max value: 12.359 - type: nauc_ndcg_at_100_std value: -36.2481 - type: nauc_ndcg_at_100_diff1 value: 85.6223 - type: nauc_ndcg_at_1000_max value: 12.554000000000002 - type: nauc_ndcg_at_1000_std value: -36.4419 - type: nauc_ndcg_at_1000_diff1 value: 85.65469999999999 - type: nauc_map_at_1_max value: 10.7918 - type: nauc_map_at_1_std value: -38.0017 - type: nauc_map_at_1_diff1 value: 88.115 - type: nauc_map_at_3_max value: 11.2948 - type: nauc_map_at_3_std value: -38.7444 - type: nauc_map_at_3_diff1 value: 85.90090000000001 - type: nauc_map_at_5_max value: 12.3632 - type: nauc_map_at_5_std value: -38.1401 - type: nauc_map_at_5_diff1 value: 85.7334 - type: nauc_map_at_10_max value: 12.567900000000002 - type: nauc_map_at_10_std value: -37.196 - type: nauc_map_at_10_diff1 value: 85.8887 - type: nauc_map_at_20_max value: 12.2576 - type: nauc_map_at_20_std value: -36.9194 - type: nauc_map_at_20_diff1 value: 85.9492 - type: nauc_map_at_100_max value: 12.313699999999999 - type: nauc_map_at_100_std value: -37.1156 - type: nauc_map_at_100_diff1 value: 85.96589999999999 - type: nauc_map_at_1000_max value: 12.3179 - type: nauc_map_at_1000_std value: -37.122899999999994 - type: nauc_map_at_1000_diff1 value: 85.9666 - type: nauc_recall_at_1_max value: 10.7918 - type: nauc_recall_at_1_std value: -38.0017 - type: nauc_recall_at_1_diff1 value: 88.115 - type: nauc_recall_at_3_max value: 11.5712 - type: nauc_recall_at_3_std value: -42.0403 - type: nauc_recall_at_3_diff1 value: 82.2085 - type: nauc_recall_at_5_max value: 19.3703 - type: nauc_recall_at_5_std value: -36.4651 - type: nauc_recall_at_5_diff1 value: 79.02289999999999 - type: nauc_recall_at_10_max value: 24.702099999999998 - type: nauc_recall_at_10_std value: -13.0473 - type: nauc_recall_at_10_diff1 value: 79.0931 - type: nauc_recall_at_20_max value: 8.377600000000001 - type: nauc_recall_at_20_std value: 19.5544 - type: nauc_recall_at_20_diff1 value: 80.15440000000001 - type: nauc_recall_at_100_max value: -16.4048 - type: nauc_recall_at_100_std value: -4.217 - type: nauc_recall_at_100_diff1 value: 77.3525 - type: nauc_recall_at_1000_max value: 35.780899999999995 - type: nauc_recall_at_1000_std value: -17.1046 - type: nauc_recall_at_1000_diff1 value: 72.2168 - type: nauc_precision_at_1_max value: 10.6257 - type: nauc_precision_at_1_std value: -37.586999999999996 - type: nauc_precision_at_1_diff1 value: 87.6739 - type: nauc_precision_at_3_max value: 12.3466 - type: nauc_precision_at_3_std value: -35.2022 - type: nauc_precision_at_3_diff1 value: 69.3658 - type: nauc_precision_at_5_max value: 21.8156 - type: nauc_precision_at_5_std value: -26.7725 - type: nauc_precision_at_5_diff1 value: 55.6505 - type: nauc_precision_at_10_max value: 28.0424 - type: nauc_precision_at_10_std value: -3.9004999999999996 - type: nauc_precision_at_10_diff1 value: 40.0286 - type: nauc_precision_at_20_max value: 17.8965 - type: nauc_precision_at_20_std value: 20.0597 - type: nauc_precision_at_20_diff1 value: 19.717299999999998 - type: nauc_precision_at_100_max value: 19.5903 - type: nauc_precision_at_100_std value: 15.1328 - type: nauc_precision_at_100_diff1 value: -26.816699999999997 - type: nauc_precision_at_1000_max value: 30.2629 - type: nauc_precision_at_1000_std value: 18.6426 - type: nauc_precision_at_1000_diff1 value: -49.766 - type: nauc_mrr_at_1_max value: 11.444799999999999 - type: nauc_mrr_at_1_std value: -38.4039 - type: nauc_mrr_at_1_diff1 value: 87.6739 - type: nauc_mrr_at_3_max value: 12.0501 - type: nauc_mrr_at_3_std value: -39.6139 - type: nauc_mrr_at_3_diff1 value: 85.8329 - type: nauc_mrr_at_5_max value: 12.549199999999999 - type: nauc_mrr_at_5_std value: -38.6156 - type: nauc_mrr_at_5_diff1 value: 85.6892 - type: nauc_mrr_at_10_max value: 12.647400000000001 - type: nauc_mrr_at_10_std value: -37.7216 - type: nauc_mrr_at_10_diff1 value: 85.8803 - type: nauc_mrr_at_20_max value: 12.3774 - type: nauc_mrr_at_20_std value: -37.5013 - type: nauc_mrr_at_20_diff1 value: 85.95 - type: nauc_mrr_at_100_max value: 12.4338 - type: nauc_mrr_at_100_std value: -37.6993 - type: nauc_mrr_at_100_diff1 value: 85.9667 - type: nauc_mrr_at_1000_max value: 12.4381 - type: nauc_mrr_at_1000_std value: -37.7067 - type: nauc_mrr_at_1000_diff1 value: 85.9674 - type: main_score value: 87.15899999999999 task: type: Retrieval - dataset: config: default name: MTEB DuRetrieval (default) revision: a1a333e290fe30b10f3f56498e3a0d911a693ced split: dev type: C-MTEB/DuRetrieval metrics: - type: ndcg_at_1 value: 91.95 - type: ndcg_at_3 value: 89.564 - type: ndcg_at_5 value: 88.74 - type: ndcg_at_10 value: 90.816 - type: ndcg_at_20 value: 92.049 - type: ndcg_at_100 value: 92.944 - type: ndcg_at_1000 value: 93.101 - type: map_at_1 value: 27.468999999999998 - type: map_at_3 value: 58.833 - type: map_at_5 value: 74.146 - type: map_at_10 value: 84.49 - type: map_at_20 value: 86.583 - type: map_at_100 value: 87.15 - type: map_at_1000 value: 87.167 - type: recall_at_1 value: 27.468999999999998 - type: recall_at_3 value: 61.253 - type: recall_at_5 value: 78.605 - type: recall_at_10 value: 91.984 - type: recall_at_20 value: 96.15700000000001 - type: recall_at_100 value: 99.012 - type: recall_at_1000 value: 99.85000000000001 - type: precision_at_1 value: 91.95 - type: precision_at_3 value: 79.93299999999999 - type: precision_at_5 value: 67.52 - type: precision_at_10 value: 43.125 - type: precision_at_20 value: 23.365 - type: precision_at_100 value: 4.873 - type: precision_at_1000 value: 0.49100000000000005 - type: mrr_at_1 value: 91.95 - type: mrr_at_3 value: 94.6833 - type: mrr_at_5 value: 94.80330000000001 - type: mrr_at_10 value: 94.8327 - type: mrr_at_20 value: 94.86449999999999 - type: mrr_at_100 value: 94.8776 - type: mrr_at_1000 value: 94.87910000000001 - type: nauc_ndcg_at_1_max value: 38.3147 - type: nauc_ndcg_at_1_std value: 25.4415 - type: nauc_ndcg_at_1_diff1 value: 20.072400000000002 - type: nauc_ndcg_at_3_max value: 28.4644 - type: nauc_ndcg_at_3_std value: 16.3111 - type: nauc_ndcg_at_3_diff1 value: 4.136 - type: nauc_ndcg_at_5_max value: 20.8656 - type: nauc_ndcg_at_5_std value: 7.4921 - type: nauc_ndcg_at_5_diff1 value: 6.9999 - type: nauc_ndcg_at_10_max value: 31.9021 - type: nauc_ndcg_at_10_std value: 9.895199999999999 - type: nauc_ndcg_at_10_diff1 value: 7.4908 - type: nauc_ndcg_at_20_max value: 35.1138 - type: nauc_ndcg_at_20_std value: 15.992799999999999 - type: nauc_ndcg_at_20_diff1 value: 5.8599 - type: nauc_ndcg_at_100_max value: 35.4467 - type: nauc_ndcg_at_100_std value: 20.2691 - type: nauc_ndcg_at_100_diff1 value: 4.6742 - type: nauc_ndcg_at_1000_max value: 34.8608 - type: nauc_ndcg_at_1000_std value: 19.7889 - type: nauc_ndcg_at_1000_diff1 value: 4.455 - type: nauc_map_at_1_max value: -14.544799999999999 - type: nauc_map_at_1_std value: -36.809 - type: nauc_map_at_1_diff1 value: 48.945 - type: nauc_map_at_3_max value: -4.3561000000000005 - type: nauc_map_at_3_std value: -36.5114 - type: nauc_map_at_3_diff1 value: 34.5076 - type: nauc_map_at_5_max value: 4.7926 - type: nauc_map_at_5_std value: -29.224 - type: nauc_map_at_5_diff1 value: 23.6996 - type: nauc_map_at_10_max value: 23.3542 - type: nauc_map_at_10_std value: -8.1883 - type: nauc_map_at_10_diff1 value: 10.3199 - type: nauc_map_at_20_max value: 26.1761 - type: nauc_map_at_20_std value: 4.3185 - type: nauc_map_at_20_diff1 value: 5.7029000000000005 - type: nauc_map_at_100_max value: 26.251 - type: nauc_map_at_100_std value: 7.3792 - type: nauc_map_at_100_diff1 value: 4.4363 - type: nauc_map_at_1000_max value: 26.215 - type: nauc_map_at_1000_std value: 7.3778999999999995 - type: nauc_map_at_1000_diff1 value: 4.3981 - type: nauc_recall_at_1_max value: -14.544799999999999 - type: nauc_recall_at_1_std value: -36.809 - type: nauc_recall_at_1_diff1 value: 48.945 - type: nauc_recall_at_3_max value: -8.1063 - type: nauc_recall_at_3_std value: -39.984700000000004 - type: nauc_recall_at_3_diff1 value: 36.1148 - type: nauc_recall_at_5_max value: -1.8721 - type: nauc_recall_at_5_std value: -37.5321 - type: nauc_recall_at_5_diff1 value: 27.4684 - type: nauc_recall_at_10_max value: 27.440900000000003 - type: nauc_recall_at_10_std value: -17.471500000000002 - type: nauc_recall_at_10_diff1 value: 18.1816 - type: nauc_recall_at_20_max value: 42.033500000000004 - type: nauc_recall_at_20_std value: 13.2844 - type: nauc_recall_at_20_diff1 value: 11.135100000000001 - type: nauc_recall_at_100_max value: 63.4582 - type: nauc_recall_at_100_std value: 54.1563 - type: nauc_recall_at_100_diff1 value: 8.7789 - type: nauc_recall_at_1000_max value: 73.73150000000001 - type: nauc_recall_at_1000_std value: 78.3703 - type: nauc_recall_at_1000_diff1 value: 15.1756 - type: nauc_precision_at_1_max value: 38.3147 - type: nauc_precision_at_1_std value: 25.4415 - type: nauc_precision_at_1_diff1 value: 20.072400000000002 - type: nauc_precision_at_3_max value: 33.7041 - type: nauc_precision_at_3_std value: 40.7562 - type: nauc_precision_at_3_diff1 value: -41.5615 - type: nauc_precision_at_5_max value: 26.691100000000002 - type: nauc_precision_at_5_std value: 44.0977 - type: nauc_precision_at_5_diff1 value: -42.8939 - type: nauc_precision_at_10_max value: 23.5844 - type: nauc_precision_at_10_std value: 48.653400000000005 - type: nauc_precision_at_10_diff1 value: -39.3653 - type: nauc_precision_at_20_max value: 18.6268 - type: nauc_precision_at_20_std value: 52.2362 - type: nauc_precision_at_20_diff1 value: -38.1004 - type: nauc_precision_at_100_max value: 15.3183 - type: nauc_precision_at_100_std value: 52.4707 - type: nauc_precision_at_100_diff1 value: -37.5629 - type: nauc_precision_at_1000_max value: 14.562700000000001 - type: nauc_precision_at_1000_std value: 51.9881 - type: nauc_precision_at_1000_diff1 value: -37.6074 - type: nauc_mrr_at_1_max value: 38.3147 - type: nauc_mrr_at_1_std value: 25.4415 - type: nauc_mrr_at_1_diff1 value: 20.072400000000002 - type: nauc_mrr_at_3_max value: 48.4265 - type: nauc_mrr_at_3_std value: 32.605000000000004 - type: nauc_mrr_at_3_diff1 value: 20.5285 - type: nauc_mrr_at_5_max value: 47.6027 - type: nauc_mrr_at_5_std value: 32.5635 - type: nauc_mrr_at_5_diff1 value: 19.7022 - type: nauc_mrr_at_10_max value: 47.4584 - type: nauc_mrr_at_10_std value: 32.5057 - type: nauc_mrr_at_10_diff1 value: 19.8632 - type: nauc_mrr_at_20_max value: 47.1959 - type: nauc_mrr_at_20_std value: 32.3991 - type: nauc_mrr_at_20_diff1 value: 19.741 - type: nauc_mrr_at_100_max value: 47.1371 - type: nauc_mrr_at_100_std value: 32.263999999999996 - type: nauc_mrr_at_100_diff1 value: 19.6933 - type: nauc_mrr_at_1000_max value: 47.1235 - type: nauc_mrr_at_1000_std value: 32.2476 - type: nauc_mrr_at_1000_diff1 value: 19.6877 - type: main_score value: 90.816 task: type: Retrieval - dataset: config: default name: MTEB EcomRetrieval (default) revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9 split: dev type: C-MTEB/EcomRetrieval metrics: - type: ndcg_at_1 value: 55.900000000000006 - type: ndcg_at_3 value: 66.019 - type: ndcg_at_5 value: 68.47999999999999 - type: ndcg_at_10 value: 70.678 - type: ndcg_at_20 value: 72.024 - type: ndcg_at_100 value: 72.933 - type: ndcg_at_1000 value: 73.20400000000001 - type: map_at_1 value: 55.900000000000006 - type: map_at_3 value: 63.56699999999999 - type: map_at_5 value: 64.927 - type: map_at_10 value: 65.83200000000001 - type: map_at_20 value: 66.205 - type: map_at_100 value: 66.33099999999999 - type: map_at_1000 value: 66.34 - type: recall_at_1 value: 55.900000000000006 - type: recall_at_3 value: 73.1 - type: recall_at_5 value: 79.10000000000001 - type: recall_at_10 value: 85.9 - type: recall_at_20 value: 91.2 - type: recall_at_100 value: 96.1 - type: recall_at_1000 value: 98.3 - type: precision_at_1 value: 55.900000000000006 - type: precision_at_3 value: 24.367 - type: precision_at_5 value: 15.82 - type: precision_at_10 value: 8.59 - type: precision_at_20 value: 4.5600000000000005 - type: precision_at_100 value: 0.9610000000000001 - type: precision_at_1000 value: 0.098 - type: mrr_at_1 value: 55.900000000000006 - type: mrr_at_3 value: 63.5667 - type: mrr_at_5 value: 64.92670000000001 - type: mrr_at_10 value: 65.8319 - type: mrr_at_20 value: 66.2046 - type: mrr_at_100 value: 66.3313 - type: mrr_at_1000 value: 66.3405 - type: nauc_ndcg_at_1_max value: 7.968999999999999 - type: nauc_ndcg_at_1_std value: -13.8408 - type: nauc_ndcg_at_1_diff1 value: 74.252 - type: nauc_ndcg_at_3_max value: 9.5736 - type: nauc_ndcg_at_3_std value: -11.1447 - type: nauc_ndcg_at_3_diff1 value: 69.1241 - type: nauc_ndcg_at_5_max value: 11.5841 - type: nauc_ndcg_at_5_std value: -10.9319 - type: nauc_ndcg_at_5_diff1 value: 68.50110000000001 - type: nauc_ndcg_at_10_max value: 12.167 - type: nauc_ndcg_at_10_std value: -10.105699999999999 - type: nauc_ndcg_at_10_diff1 value: 69.02199999999999 - type: nauc_ndcg_at_20_max value: 13.3886 - type: nauc_ndcg_at_20_std value: -8.485199999999999 - type: nauc_ndcg_at_20_diff1 value: 69.5569 - type: nauc_ndcg_at_100_max value: 12.834100000000001 - type: nauc_ndcg_at_100_std value: -7.8253 - type: nauc_ndcg_at_100_diff1 value: 69.7176 - type: nauc_ndcg_at_1000_max value: 12.510499999999999 - type: nauc_ndcg_at_1000_std value: -8.5495 - type: nauc_ndcg_at_1000_diff1 value: 69.9608 - type: nauc_map_at_1_max value: 7.968999999999999 - type: nauc_map_at_1_std value: -13.8408 - type: nauc_map_at_1_diff1 value: 74.252 - type: nauc_map_at_3_max value: 9.1602 - type: nauc_map_at_3_std value: -11.9092 - type: nauc_map_at_3_diff1 value: 70.4804 - type: nauc_map_at_5_max value: 10.0994 - type: nauc_map_at_5_std value: -11.874600000000001 - type: nauc_map_at_5_diff1 value: 70.2168 - type: nauc_map_at_10_max value: 10.319799999999999 - type: nauc_map_at_10_std value: -11.574 - type: nauc_map_at_10_diff1 value: 70.4827 - type: nauc_map_at_20_max value: 10.5658 - type: nauc_map_at_20_std value: -11.232000000000001 - type: nauc_map_at_20_diff1 value: 70.6199 - type: nauc_map_at_100_max value: 10.4852 - type: nauc_map_at_100_std value: -11.1232 - type: nauc_map_at_100_diff1 value: 70.64859999999999 - type: nauc_map_at_1000_max value: 10.4838 - type: nauc_map_at_1000_std value: -11.1379 - type: nauc_map_at_1000_diff1 value: 70.6609 - type: nauc_recall_at_1_max value: 7.968999999999999 - type: nauc_recall_at_1_std value: -13.8408 - type: nauc_recall_at_1_diff1 value: 74.252 - type: nauc_recall_at_3_max value: 11.016399999999999 - type: nauc_recall_at_3_std value: -8.4394 - type: nauc_recall_at_3_diff1 value: 64.3724 - type: nauc_recall_at_5_max value: 18.305 - type: nauc_recall_at_5_std value: -6.8603 - type: nauc_recall_at_5_diff1 value: 61.24380000000001 - type: nauc_recall_at_10_max value: 23.8141 - type: nauc_recall_at_10_std value: -0.9058999999999999 - type: nauc_recall_at_10_diff1 value: 60.687999999999995 - type: nauc_recall_at_20_max value: 42.9479 - type: nauc_recall_at_20_std value: 20.9999 - type: nauc_recall_at_20_diff1 value: 61.755700000000004 - type: nauc_recall_at_100_max value: 63.88589999999999 - type: nauc_recall_at_100_std value: 73.6108 - type: nauc_recall_at_100_diff1 value: 56.4067 - type: nauc_recall_at_1000_max value: 89.56450000000001 - type: nauc_recall_at_1000_std value: 99.23110000000001 - type: nauc_recall_at_1000_diff1 value: 59.913799999999995 - type: nauc_precision_at_1_max value: 7.968999999999999 - type: nauc_precision_at_1_std value: -13.8408 - type: nauc_precision_at_1_diff1 value: 74.252 - type: nauc_precision_at_3_max value: 11.016399999999999 - type: nauc_precision_at_3_std value: -8.4394 - type: nauc_precision_at_3_diff1 value: 64.3724 - type: nauc_precision_at_5_max value: 18.305 - type: nauc_precision_at_5_std value: -6.8603 - type: nauc_precision_at_5_diff1 value: 61.24380000000001 - type: nauc_precision_at_10_max value: 23.8141 - type: nauc_precision_at_10_std value: -0.9058999999999999 - type: nauc_precision_at_10_diff1 value: 60.687999999999995 - type: nauc_precision_at_20_max value: 42.9479 - type: nauc_precision_at_20_std value: 20.9999 - type: nauc_precision_at_20_diff1 value: 61.755700000000004 - type: nauc_precision_at_100_max value: 63.88589999999999 - type: nauc_precision_at_100_std value: 73.6108 - type: nauc_precision_at_100_diff1 value: 56.4067 - type: nauc_precision_at_1000_max value: 89.56450000000001 - type: nauc_precision_at_1000_std value: 99.23110000000001 - type: nauc_precision_at_1000_diff1 value: 59.913799999999995 - type: nauc_mrr_at_1_max value: 7.968999999999999 - type: nauc_mrr_at_1_std value: -13.8408 - type: nauc_mrr_at_1_diff1 value: 74.252 - type: nauc_mrr_at_3_max value: 9.1602 - type: nauc_mrr_at_3_std value: -11.9092 - type: nauc_mrr_at_3_diff1 value: 70.4804 - type: nauc_mrr_at_5_max value: 10.0994 - type: nauc_mrr_at_5_std value: -11.874600000000001 - type: nauc_mrr_at_5_diff1 value: 70.2168 - type: nauc_mrr_at_10_max value: 10.319799999999999 - type: nauc_mrr_at_10_std value: -11.574 - type: nauc_mrr_at_10_diff1 value: 70.4827 - type: nauc_mrr_at_20_max value: 10.5658 - type: nauc_mrr_at_20_std value: -11.232000000000001 - type: nauc_mrr_at_20_diff1 value: 70.6199 - type: nauc_mrr_at_100_max value: 10.4852 - type: nauc_mrr_at_100_std value: -11.1232 - type: nauc_mrr_at_100_diff1 value: 70.64859999999999 - type: nauc_mrr_at_1000_max value: 10.4838 - type: nauc_mrr_at_1000_std value: -11.1379 - type: nauc_mrr_at_1000_diff1 value: 70.6609 - type: main_score value: 70.678 task: type: Retrieval - dataset: config: default name: MTEB MMarcoRetrieval (default) revision: 539bbde593d947e2a124ba72651aafc09eb33fc2 split: dev type: C-MTEB/MMarcoRetrieval metrics: - type: ndcg_at_1 value: 75.74499999999999 - type: ndcg_at_3 value: 82.188 - type: ndcg_at_5 value: 83.869 - type: ndcg_at_10 value: 85.11800000000001 - type: ndcg_at_20 value: 85.624 - type: ndcg_at_100 value: 86.05000000000001 - type: ndcg_at_1000 value: 86.177 - type: map_at_1 value: 73.33 - type: map_at_3 value: 80.333 - type: map_at_5 value: 81.349 - type: map_at_10 value: 81.906 - type: map_at_20 value: 82.059 - type: map_at_100 value: 82.127 - type: map_at_1000 value: 82.132 - type: recall_at_1 value: 73.33 - type: recall_at_3 value: 86.823 - type: recall_at_5 value: 90.814 - type: recall_at_10 value: 94.509 - type: recall_at_20 value: 96.422 - type: recall_at_100 value: 98.6 - type: recall_at_1000 value: 99.599 - type: precision_at_1 value: 75.74499999999999 - type: precision_at_3 value: 30.625999999999998 - type: precision_at_5 value: 19.281000000000002 - type: precision_at_10 value: 10.05 - type: precision_at_20 value: 5.133 - type: precision_at_100 value: 1.051 - type: precision_at_1000 value: 0.106 - type: mrr_at_1 value: 75.74499999999999 - type: mrr_at_3 value: 81.0148 - type: mrr_at_5 value: 81.8658 - type: mrr_at_10 value: 82.3666 - type: mrr_at_20 value: 82.4972 - type: mrr_at_100 value: 82.55449999999999 - type: mrr_at_1000 value: 82.5588 - type: nauc_ndcg_at_1_max value: 13.195799999999998 - type: nauc_ndcg_at_1_std value: -27.2173 - type: nauc_ndcg_at_1_diff1 value: 83.5531 - type: nauc_ndcg_at_3_max value: 17.1078 - type: nauc_ndcg_at_3_std value: -24.1926 - type: nauc_ndcg_at_3_diff1 value: 80.4983 - type: nauc_ndcg_at_5_max value: 19.3079 - type: nauc_ndcg_at_5_std value: -21.5239 - type: nauc_ndcg_at_5_diff1 value: 80.6484 - type: nauc_ndcg_at_10_max value: 19.5153 - type: nauc_ndcg_at_10_std value: -19.997799999999998 - type: nauc_ndcg_at_10_diff1 value: 80.9953 - type: nauc_ndcg_at_20_max value: 19.5474 - type: nauc_ndcg_at_20_std value: -19.3015 - type: nauc_ndcg_at_20_diff1 value: 81.25670000000001 - type: nauc_ndcg_at_100_max value: 18.569399999999998 - type: nauc_ndcg_at_100_std value: -20.4957 - type: nauc_ndcg_at_100_diff1 value: 81.3873 - type: nauc_ndcg_at_1000_max value: 18.1522 - type: nauc_ndcg_at_1000_std value: -21.3527 - type: nauc_ndcg_at_1000_diff1 value: 81.44590000000001 - type: nauc_map_at_1_max value: 9.1671 - type: nauc_map_at_1_std value: -27.6298 - type: nauc_map_at_1_diff1 value: 82.5107 - type: nauc_map_at_3_max value: 15.1633 - type: nauc_map_at_3_std value: -25.835200000000004 - type: nauc_map_at_3_diff1 value: 81.2211 - type: nauc_map_at_5_max value: 16.4272 - type: nauc_map_at_5_std value: -24.3902 - type: nauc_map_at_5_diff1 value: 81.3282 - type: nauc_map_at_10_max value: 16.4997 - type: nauc_map_at_10_std value: -23.860200000000003 - type: nauc_map_at_10_diff1 value: 81.46759999999999 - type: nauc_map_at_20_max value: 16.4924 - type: nauc_map_at_20_std value: -23.6935 - type: nauc_map_at_20_diff1 value: 81.54299999999999 - type: nauc_map_at_100_max value: 16.381 - type: nauc_map_at_100_std value: -23.832 - type: nauc_map_at_100_diff1 value: 81.56219999999999 - type: nauc_map_at_1000_max value: 16.370199999999997 - type: nauc_map_at_1000_std value: -23.8562 - type: nauc_map_at_1000_diff1 value: 81.5641 - type: nauc_recall_at_1_max value: 9.1671 - type: nauc_recall_at_1_std value: -27.6298 - type: nauc_recall_at_1_diff1 value: 82.5107 - type: nauc_recall_at_3_max value: 22.362000000000002 - type: nauc_recall_at_3_std value: -19.3576 - type: nauc_recall_at_3_diff1 value: 76.78540000000001 - type: nauc_recall_at_5_max value: 32.5794 - type: nauc_recall_at_5_std value: -7.4792 - type: nauc_recall_at_5_diff1 value: 75.7277 - type: nauc_recall_at_10_max value: 42.6548 - type: nauc_recall_at_10_std value: 12.512300000000002 - type: nauc_recall_at_10_diff1 value: 75.7877 - type: nauc_recall_at_20_max value: 55.737 - type: nauc_recall_at_20_std value: 39.0218 - type: nauc_recall_at_20_diff1 value: 77.1814 - type: nauc_recall_at_100_max value: 64.1065 - type: nauc_recall_at_100_std value: 70.1918 - type: nauc_recall_at_100_diff1 value: 78.1114 - type: nauc_recall_at_1000_max value: 69.8721 - type: nauc_recall_at_1000_std value: 77.2501 - type: nauc_recall_at_1000_diff1 value: 84.894 - type: nauc_precision_at_1_max value: 13.195799999999998 - type: nauc_precision_at_1_std value: -27.2173 - type: nauc_precision_at_1_diff1 value: 83.5531 - type: nauc_precision_at_3_max value: 22.4513 - type: nauc_precision_at_3_std value: -6.339 - type: nauc_precision_at_3_diff1 value: 36.5773 - type: nauc_precision_at_5_max value: 27.0464 - type: nauc_precision_at_5_std value: 5.2645 - type: nauc_precision_at_5_diff1 value: 22.6914 - type: nauc_precision_at_10_max value: 28.171000000000003 - type: nauc_precision_at_10_std value: 17.0803 - type: nauc_precision_at_10_diff1 value: 6.1321 - type: nauc_precision_at_20_max value: 27.9335 - type: nauc_precision_at_20_std value: 24.931800000000003 - type: nauc_precision_at_20_diff1 value: -4.4056 - type: nauc_precision_at_100_max value: 21.8066 - type: nauc_precision_at_100_std value: 24.659 - type: nauc_precision_at_100_diff1 value: -19.358900000000002 - type: nauc_precision_at_1000_max value: 17.9584 - type: nauc_precision_at_1000_std value: 20.4178 - type: nauc_precision_at_1000_diff1 value: -27.5299 - type: nauc_mrr_at_1_max value: 13.195799999999998 - type: nauc_mrr_at_1_std value: -27.2173 - type: nauc_mrr_at_1_diff1 value: 83.5531 - type: nauc_mrr_at_3_max value: 16.801099999999998 - type: nauc_mrr_at_3_std value: -24.4238 - type: nauc_mrr_at_3_diff1 value: 81.5653 - type: nauc_mrr_at_5_max value: 17.6255 - type: nauc_mrr_at_5_std value: -23.345399999999998 - type: nauc_mrr_at_5_diff1 value: 81.6503 - type: nauc_mrr_at_10_max value: 17.5126 - type: nauc_mrr_at_10_std value: -22.9439 - type: nauc_mrr_at_10_diff1 value: 81.8044 - type: nauc_mrr_at_20_max value: 17.468700000000002 - type: nauc_mrr_at_20_std value: -22.8673 - type: nauc_mrr_at_20_diff1 value: 81.851 - type: nauc_mrr_at_100_max value: 17.3429 - type: nauc_mrr_at_100_std value: -23.0229 - type: nauc_mrr_at_100_diff1 value: 81.8634 - type: nauc_mrr_at_1000_max value: 17.3315 - type: nauc_mrr_at_1000_std value: -23.045099999999998 - type: nauc_mrr_at_1000_diff1 value: 81.8656 - type: main_score value: 85.11800000000001 task: type: Retrieval - dataset: config: default name: MTEB MedicalRetrieval (default) revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6 split: dev type: C-MTEB/MedicalRetrieval metrics: - type: ndcg_at_1 value: 55.00000000000001 - type: ndcg_at_3 value: 61.334 - type: ndcg_at_5 value: 62.590999999999994 - type: ndcg_at_10 value: 63.913 - type: ndcg_at_20 value: 64.748 - type: ndcg_at_100 value: 66.675 - type: ndcg_at_1000 value: 67.894 - type: map_at_1 value: 55.00000000000001 - type: map_at_3 value: 59.85 - type: map_at_5 value: 60.56 - type: map_at_10 value: 61.102999999999994 - type: map_at_20 value: 61.332 - type: map_at_100 value: 61.583 - type: map_at_1000 value: 61.626999999999995 - type: recall_at_1 value: 55.00000000000001 - type: recall_at_3 value: 65.60000000000001 - type: recall_at_5 value: 68.60000000000001 - type: recall_at_10 value: 72.7 - type: recall_at_20 value: 76 - type: recall_at_100 value: 86.6 - type: recall_at_1000 value: 96.3 - type: precision_at_1 value: 55.00000000000001 - type: precision_at_3 value: 21.867 - type: precision_at_5 value: 13.719999999999999 - type: precision_at_10 value: 7.2700000000000005 - type: precision_at_20 value: 3.8 - type: precision_at_100 value: 0.8659999999999999 - type: precision_at_1000 value: 0.096 - type: mrr_at_1 value: 55.00000000000001 - type: mrr_at_3 value: 59.85 - type: mrr_at_5 value: 60.56 - type: mrr_at_10 value: 61.1027 - type: mrr_at_20 value: 61.3323 - type: mrr_at_100 value: 61.583 - type: mrr_at_1000 value: 61.6268 - type: nauc_ndcg_at_1_max value: 45.7614 - type: nauc_ndcg_at_1_std value: 17.589299999999998 - type: nauc_ndcg_at_1_diff1 value: 84.6716 - type: nauc_ndcg_at_3_max value: 43.3722 - type: nauc_ndcg_at_3_std value: 21.6068 - type: nauc_ndcg_at_3_diff1 value: 79.70779999999999 - type: nauc_ndcg_at_5_max value: 43.6396 - type: nauc_ndcg_at_5_std value: 21.9309 - type: nauc_ndcg_at_5_diff1 value: 79.0538 - type: nauc_ndcg_at_10_max value: 43.5044 - type: nauc_ndcg_at_10_std value: 22.351 - type: nauc_ndcg_at_10_diff1 value: 78.7137 - type: nauc_ndcg_at_20_max value: 43.616899999999994 - type: nauc_ndcg_at_20_std value: 22.1586 - type: nauc_ndcg_at_20_diff1 value: 78.6764 - type: nauc_ndcg_at_100_max value: 43.135400000000004 - type: nauc_ndcg_at_100_std value: 22.4145 - type: nauc_ndcg_at_100_diff1 value: 78.35 - type: nauc_ndcg_at_1000_max value: 43.669799999999995 - type: nauc_ndcg_at_1000_std value: 22.17 - type: nauc_ndcg_at_1000_diff1 value: 79.0599 - type: nauc_map_at_1_max value: 45.7614 - type: nauc_map_at_1_std value: 17.589299999999998 - type: nauc_map_at_1_diff1 value: 84.6716 - type: nauc_map_at_3_max value: 44.0428 - type: nauc_map_at_3_std value: 20.6219 - type: nauc_map_at_3_diff1 value: 80.8918 - type: nauc_map_at_5_max value: 44.2069 - type: nauc_map_at_5_std value: 20.7613 - type: nauc_map_at_5_diff1 value: 80.545 - type: nauc_map_at_10_max value: 44.1289 - type: nauc_map_at_10_std value: 20.9091 - type: nauc_map_at_10_diff1 value: 80.417 - type: nauc_map_at_20_max value: 44.1582 - type: nauc_map_at_20_std value: 20.832700000000003 - type: nauc_map_at_20_diff1 value: 80.4097 - type: nauc_map_at_100_max value: 44.105199999999996 - type: nauc_map_at_100_std value: 20.8381 - type: nauc_map_at_100_diff1 value: 80.3634 - type: nauc_map_at_1000_max value: 44.124 - type: nauc_map_at_1000_std value: 20.846 - type: nauc_map_at_1000_diff1 value: 80.3866 - type: nauc_recall_at_1_max value: 45.7614 - type: nauc_recall_at_1_std value: 17.589299999999998 - type: nauc_recall_at_1_diff1 value: 84.6716 - type: nauc_recall_at_3_max value: 41.2286 - type: nauc_recall_at_3_std value: 24.6954 - type: nauc_recall_at_3_diff1 value: 76.0107 - type: nauc_recall_at_5_max value: 41.7596 - type: nauc_recall_at_5_std value: 25.866899999999998 - type: nauc_recall_at_5_diff1 value: 74.06779999999999 - type: nauc_recall_at_10_max value: 41.261900000000004 - type: nauc_recall_at_10_std value: 27.895999999999997 - type: nauc_recall_at_10_diff1 value: 72.3198 - type: nauc_recall_at_20_max value: 41.6291 - type: nauc_recall_at_20_std value: 27.720899999999997 - type: nauc_recall_at_20_diff1 value: 71.4933 - type: nauc_recall_at_100_max value: 34.897 - type: nauc_recall_at_100_std value: 34.8631 - type: nauc_recall_at_100_diff1 value: 63.698 - type: nauc_recall_at_1000_max value: 43.4905 - type: nauc_recall_at_1000_std value: 44.3006 - type: nauc_recall_at_1000_diff1 value: 66.7197 - type: nauc_precision_at_1_max value: 45.7614 - type: nauc_precision_at_1_std value: 17.589299999999998 - type: nauc_precision_at_1_diff1 value: 84.6716 - type: nauc_precision_at_3_max value: 41.2286 - type: nauc_precision_at_3_std value: 24.6954 - type: nauc_precision_at_3_diff1 value: 76.0107 - type: nauc_precision_at_5_max value: 41.7596 - type: nauc_precision_at_5_std value: 25.866899999999998 - type: nauc_precision_at_5_diff1 value: 74.06779999999999 - type: nauc_precision_at_10_max value: 41.261900000000004 - type: nauc_precision_at_10_std value: 27.895999999999997 - type: nauc_precision_at_10_diff1 value: 72.3198 - type: nauc_precision_at_20_max value: 41.6291 - type: nauc_precision_at_20_std value: 27.720899999999997 - type: nauc_precision_at_20_diff1 value: 71.4933 - type: nauc_precision_at_100_max value: 34.897 - type: nauc_precision_at_100_std value: 34.8631 - type: nauc_precision_at_100_diff1 value: 63.698 - type: nauc_precision_at_1000_max value: 43.4905 - type: nauc_precision_at_1000_std value: 44.3006 - type: nauc_precision_at_1000_diff1 value: 66.7197 - type: nauc_mrr_at_1_max value: 45.7614 - type: nauc_mrr_at_1_std value: 17.589299999999998 - type: nauc_mrr_at_1_diff1 value: 84.6716 - type: nauc_mrr_at_3_max value: 44.0428 - type: nauc_mrr_at_3_std value: 20.6219 - type: nauc_mrr_at_3_diff1 value: 80.8918 - type: nauc_mrr_at_5_max value: 44.2069 - type: nauc_mrr_at_5_std value: 20.7613 - type: nauc_mrr_at_5_diff1 value: 80.545 - type: nauc_mrr_at_10_max value: 44.1289 - type: nauc_mrr_at_10_std value: 20.9091 - type: nauc_mrr_at_10_diff1 value: 80.417 - type: nauc_mrr_at_20_max value: 44.1582 - type: nauc_mrr_at_20_std value: 20.832700000000003 - type: nauc_mrr_at_20_diff1 value: 80.4097 - type: nauc_mrr_at_100_max value: 44.1051 - type: nauc_mrr_at_100_std value: 20.8382 - type: nauc_mrr_at_100_diff1 value: 80.3634 - type: nauc_mrr_at_1000_max value: 44.1239 - type: nauc_mrr_at_1000_std value: 20.8461 - type: nauc_mrr_at_1000_diff1 value: 80.3866 - type: main_score value: 63.913 task: type: Retrieval - dataset: config: default name: MTEB T2Retrieval (default) revision: 8731a845f1bf500a4f111cf1070785c793d10e64 split: dev type: C-MTEB/T2Retrieval metrics: - type: ndcg_at_1 value: 91.22 - type: ndcg_at_3 value: 88.018 - type: ndcg_at_5 value: 87.023 - type: ndcg_at_10 value: 87.256 - type: ndcg_at_20 value: 88.897 - type: ndcg_at_100 value: 90.386 - type: ndcg_at_1000 value: 90.81099999999999 - type: map_at_1 value: 28.203 - type: map_at_3 value: 56.16 - type: map_at_5 value: 69.28699999999999 - type: map_at_10 value: 80.361 - type: map_at_20 value: 83.152 - type: map_at_100 value: 83.894 - type: map_at_1000 value: 83.94 - type: recall_at_1 value: 28.203 - type: recall_at_3 value: 57.821 - type: recall_at_5 value: 72.616 - type: recall_at_10 value: 86.56 - type: recall_at_20 value: 92.145 - type: recall_at_100 value: 96.827 - type: recall_at_1000 value: 98.967 - type: precision_at_1 value: 91.22 - type: precision_at_3 value: 76.946 - type: precision_at_5 value: 64.822 - type: precision_at_10 value: 43.370999999999995 - type: precision_at_20 value: 23.977999999999998 - type: precision_at_100 value: 5.08 - type: precision_at_1000 value: 0.518 - type: mrr_at_1 value: 91.2195 - type: mrr_at_3 value: 93.2214 - type: mrr_at_5 value: 93.48570000000001 - type: mrr_at_10 value: 93.6156 - type: mrr_at_20 value: 93.6549 - type: mrr_at_100 value: 93.6802 - type: mrr_at_1000 value: 93.6823 - type: nauc_ndcg_at_1_max value: 57.4777 - type: nauc_ndcg_at_1_std value: 31.1 - type: nauc_ndcg_at_1_diff1 value: 47.8114 - type: nauc_ndcg_at_3_max value: 47.8308 - type: nauc_ndcg_at_3_std value: 27.538899999999998 - type: nauc_ndcg_at_3_diff1 value: 13.552900000000001 - type: nauc_ndcg_at_5_max value: 40.672799999999995 - type: nauc_ndcg_at_5_std value: 22.5295 - type: nauc_ndcg_at_5_diff1 value: 13.9263 - type: nauc_ndcg_at_10_max value: 33.6021 - type: nauc_ndcg_at_10_std value: 15.9671 - type: nauc_ndcg_at_10_diff1 value: 17.678 - type: nauc_ndcg_at_20_max value: 39.0053 - type: nauc_ndcg_at_20_std value: 21.218 - type: nauc_ndcg_at_20_diff1 value: 17.5961 - type: nauc_ndcg_at_100_max value: 44.208 - type: nauc_ndcg_at_100_std value: 27.437 - type: nauc_ndcg_at_100_diff1 value: 17.7034 - type: nauc_ndcg_at_1000_max value: 44.8086 - type: nauc_ndcg_at_1000_std value: 27.460600000000003 - type: nauc_ndcg_at_1000_diff1 value: 18.2625 - type: nauc_map_at_1_max value: -24.823500000000003 - type: nauc_map_at_1_std value: -39.5167 - type: nauc_map_at_1_diff1 value: 55.3064 - type: nauc_map_at_3_max value: -17.5097 - type: nauc_map_at_3_std value: -39.6402 - type: nauc_map_at_3_diff1 value: 41.255199999999995 - type: nauc_map_at_5_max value: -5.4253 - type: nauc_map_at_5_std value: -30.6847 - type: nauc_map_at_5_diff1 value: 33.1347 - type: nauc_map_at_10_max value: 20.598 - type: nauc_map_at_10_std value: -1.9429999999999998 - type: nauc_map_at_10_diff1 value: 19.081799999999998 - type: nauc_map_at_20_max value: 30.55 - type: nauc_map_at_20_std value: 12.5299 - type: nauc_map_at_20_diff1 value: 14.2657 - type: nauc_map_at_100_max value: 32.8313 - type: nauc_map_at_100_std value: 15.956000000000001 - type: nauc_map_at_100_diff1 value: 13.5621 - type: nauc_map_at_1000_max value: 32.8957 - type: nauc_map_at_1000_std value: 16.003 - type: nauc_map_at_1000_diff1 value: 13.5638 - type: nauc_recall_at_1_max value: -24.823500000000003 - type: nauc_recall_at_1_std value: -39.5167 - type: nauc_recall_at_1_diff1 value: 55.3064 - type: nauc_recall_at_3_max value: -21.037 - type: nauc_recall_at_3_std value: -42.2523 - type: nauc_recall_at_3_diff1 value: 39.705200000000005 - type: nauc_recall_at_5_max value: -12.8722 - type: nauc_recall_at_5_std value: -37.0029 - type: nauc_recall_at_5_diff1 value: 32.3874 - type: nauc_recall_at_10_max value: 11.2835 - type: nauc_recall_at_10_std value: -10.2755 - type: nauc_recall_at_10_diff1 value: 18.333099999999998 - type: nauc_recall_at_20_max value: 28.602800000000002 - type: nauc_recall_at_20_std value: 15.407499999999999 - type: nauc_recall_at_20_diff1 value: 10.1417 - type: nauc_recall_at_100_max value: 46.5586 - type: nauc_recall_at_100_std value: 44.6235 - type: nauc_recall_at_100_diff1 value: 5.0230999999999995 - type: nauc_recall_at_1000_max value: 64.8937 - type: nauc_recall_at_1000_std value: 64.5098 - type: nauc_recall_at_1000_diff1 value: 8.7287 - type: nauc_precision_at_1_max value: 57.4777 - type: nauc_precision_at_1_std value: 31.1 - type: nauc_precision_at_1_diff1 value: 47.8114 - type: nauc_precision_at_3_max value: 51.668499999999995 - type: nauc_precision_at_3_std value: 48.100500000000004 - type: nauc_precision_at_3_diff1 value: -36.261500000000005 - type: nauc_precision_at_5_max value: 48.2878 - type: nauc_precision_at_5_std value: 52.525 - type: nauc_precision_at_5_diff1 value: -39.636300000000006 - type: nauc_precision_at_10_max value: 44.6478 - type: nauc_precision_at_10_std value: 56.92549999999999 - type: nauc_precision_at_10_diff1 value: -38.2201 - type: nauc_precision_at_20_max value: 43.105 - type: nauc_precision_at_20_std value: 60.181700000000006 - type: nauc_precision_at_20_diff1 value: -37.841 - type: nauc_precision_at_100_max value: 41.861 - type: nauc_precision_at_100_std value: 61.2084 - type: nauc_precision_at_100_diff1 value: -37.7674 - type: nauc_precision_at_1000_max value: 40.9026 - type: nauc_precision_at_1000_std value: 60.2503 - type: nauc_precision_at_1000_diff1 value: -37.656 - type: nauc_mrr_at_1_max value: 57.4777 - type: nauc_mrr_at_1_std value: 31.1 - type: nauc_mrr_at_1_diff1 value: 47.8114 - type: nauc_mrr_at_3_max value: 61.8122 - type: nauc_mrr_at_3_std value: 35.3495 - type: nauc_mrr_at_3_diff1 value: 47.9786 - type: nauc_mrr_at_5_max value: 62.030300000000004 - type: nauc_mrr_at_5_std value: 35.8677 - type: nauc_mrr_at_5_diff1 value: 48.0773 - type: nauc_mrr_at_10_max value: 61.9735 - type: nauc_mrr_at_10_std value: 35.8413 - type: nauc_mrr_at_10_diff1 value: 48.0359 - type: nauc_mrr_at_20_max value: 61.917100000000005 - type: nauc_mrr_at_20_std value: 35.8176 - type: nauc_mrr_at_20_diff1 value: 48.024699999999996 - type: nauc_mrr_at_100_max value: 61.8535 - type: nauc_mrr_at_100_std value: 35.7606 - type: nauc_mrr_at_100_diff1 value: 48.0105 - type: nauc_mrr_at_1000_max value: 61.8448 - type: nauc_mrr_at_1000_std value: 35.747299999999996 - type: nauc_mrr_at_1000_diff1 value: 48.0124 - type: main_score value: 87.256 task: type: Retrieval - dataset: config: default name: MTEB VideoRetrieval (default) revision: 58c2597a5943a2ba48f4668c3b90d796283c5639 split: dev type: C-MTEB/VideoRetrieval metrics: - type: ndcg_at_1 value: 65.9 - type: ndcg_at_3 value: 75.395 - type: ndcg_at_5 value: 76.97 - type: ndcg_at_10 value: 78.549 - type: ndcg_at_20 value: 79.489 - type: ndcg_at_100 value: 80.269 - type: ndcg_at_1000 value: 80.459 - type: map_at_1 value: 65.9 - type: map_at_3 value: 73.167 - type: map_at_5 value: 74.047 - type: map_at_10 value: 74.713 - type: map_at_20 value: 74.97399999999999 - type: map_at_100 value: 75.089 - type: map_at_1000 value: 75.09599999999999 - type: recall_at_1 value: 65.9 - type: recall_at_3 value: 81.8 - type: recall_at_5 value: 85.6 - type: recall_at_10 value: 90.4 - type: recall_at_20 value: 94.1 - type: recall_at_100 value: 98.2 - type: recall_at_1000 value: 99.7 - type: precision_at_1 value: 65.9 - type: precision_at_3 value: 27.267000000000003 - type: precision_at_5 value: 17.119999999999997 - type: precision_at_10 value: 9.04 - type: precision_at_20 value: 4.705 - type: precision_at_100 value: 0.9820000000000001 - type: precision_at_1000 value: 0.1 - type: mrr_at_1 value: 65.8 - type: mrr_at_3 value: 73.1167 - type: mrr_at_5 value: 73.9967 - type: mrr_at_10 value: 74.66329999999999 - type: mrr_at_20 value: 74.9235 - type: mrr_at_100 value: 75.0391 - type: mrr_at_1000 value: 75.0463 - type: nauc_ndcg_at_1_max value: -7.102899999999999 - type: nauc_ndcg_at_1_std value: -40.454 - type: nauc_ndcg_at_1_diff1 value: 73.7435 - type: nauc_ndcg_at_3_max value: -10.2162 - type: nauc_ndcg_at_3_std value: -51.03 - type: nauc_ndcg_at_3_diff1 value: 70.7969 - type: nauc_ndcg_at_5_max value: -8.1372 - type: nauc_ndcg_at_5_std value: -50.727999999999994 - type: nauc_ndcg_at_5_diff1 value: 70.4751 - type: nauc_ndcg_at_10_max value: -8.305 - type: nauc_ndcg_at_10_std value: -48.972500000000004 - type: nauc_ndcg_at_10_diff1 value: 70.9306 - type: nauc_ndcg_at_20_max value: -7.61 - type: nauc_ndcg_at_20_std value: -48.0327 - type: nauc_ndcg_at_20_diff1 value: 71.02159999999999 - type: nauc_ndcg_at_100_max value: -7.6188 - type: nauc_ndcg_at_100_std value: -46.3162 - type: nauc_ndcg_at_100_diff1 value: 71.1421 - type: nauc_ndcg_at_1000_max value: -7.8627 - type: nauc_ndcg_at_1000_std value: -46.7545 - type: nauc_ndcg_at_1000_diff1 value: 71.352 - type: nauc_map_at_1_max value: -7.102899999999999 - type: nauc_map_at_1_std value: -40.454 - type: nauc_map_at_1_diff1 value: 73.7435 - type: nauc_map_at_3_max value: -9.6192 - type: nauc_map_at_3_std value: -48.0776 - type: nauc_map_at_3_diff1 value: 71.545 - type: nauc_map_at_5_max value: -8.5417 - type: nauc_map_at_5_std value: -47.8473 - type: nauc_map_at_5_diff1 value: 71.42389999999999 - type: nauc_map_at_10_max value: -8.6365 - type: nauc_map_at_10_std value: -47.1493 - type: nauc_map_at_10_diff1 value: 71.5916 - type: nauc_map_at_20_max value: -8.516 - type: nauc_map_at_20_std value: -46.958800000000004 - type: nauc_map_at_20_diff1 value: 71.6177 - type: nauc_map_at_100_max value: -8.498999999999999 - type: nauc_map_at_100_std value: -46.7142 - type: nauc_map_at_100_diff1 value: 71.638 - type: nauc_map_at_1000_max value: -8.4997 - type: nauc_map_at_1000_std value: -46.7198 - type: nauc_map_at_1000_diff1 value: 71.6453 - type: nauc_recall_at_1_max value: -7.102899999999999 - type: nauc_recall_at_1_std value: -40.454 - type: nauc_recall_at_1_diff1 value: 73.7435 - type: nauc_recall_at_3_max value: -12.3728 - type: nauc_recall_at_3_std value: -62.4854 - type: nauc_recall_at_3_diff1 value: 67.9381 - type: nauc_recall_at_5_max value: -5.5908 - type: nauc_recall_at_5_std value: -64.0017 - type: nauc_recall_at_5_diff1 value: 66.01129999999999 - type: nauc_recall_at_10_max value: -5.2142 - type: nauc_recall_at_10_std value: -59.1241 - type: nauc_recall_at_10_diff1 value: 67.0484 - type: nauc_recall_at_20_max value: 6.0224 - type: nauc_recall_at_20_std value: -52.987899999999996 - type: nauc_recall_at_20_diff1 value: 65.76859999999999 - type: nauc_recall_at_100_max value: 33.408500000000004 - type: nauc_recall_at_100_std value: 15.325800000000001 - type: nauc_recall_at_100_diff1 value: 59.588699999999996 - type: nauc_recall_at_1000_max value: 100 - type: nauc_recall_at_1000_std value: 95.6427 - type: nauc_recall_at_1000_diff1 value: 91.28540000000001 - type: nauc_precision_at_1_max value: -7.102899999999999 - type: nauc_precision_at_1_std value: -40.454 - type: nauc_precision_at_1_diff1 value: 73.7435 - type: nauc_precision_at_3_max value: -12.3728 - type: nauc_precision_at_3_std value: -62.4854 - type: nauc_precision_at_3_diff1 value: 67.9381 - type: nauc_precision_at_5_max value: -5.5908 - type: nauc_precision_at_5_std value: -64.0017 - type: nauc_precision_at_5_diff1 value: 66.01129999999999 - type: nauc_precision_at_10_max value: -5.2142 - type: nauc_precision_at_10_std value: -59.1241 - type: nauc_precision_at_10_diff1 value: 67.0484 - type: nauc_precision_at_20_max value: 6.0224 - type: nauc_precision_at_20_std value: -52.987899999999996 - type: nauc_precision_at_20_diff1 value: 65.76859999999999 - type: nauc_precision_at_100_max value: 33.408500000000004 - type: nauc_precision_at_100_std value: 15.325800000000001 - type: nauc_precision_at_100_diff1 value: 59.588699999999996 - type: nauc_precision_at_1000_max value: 100 - type: nauc_precision_at_1000_std value: 95.6427 - type: nauc_precision_at_1000_diff1 value: 91.28540000000001 - type: nauc_mrr_at_1_max value: -7.704999999999999 - type: nauc_mrr_at_1_std value: -40.435300000000005 - type: nauc_mrr_at_1_diff1 value: 73.9529 - type: nauc_mrr_at_3_max value: -9.9824 - type: nauc_mrr_at_3_std value: -48.0572 - type: nauc_mrr_at_3_diff1 value: 71.6749 - type: nauc_mrr_at_5_max value: -8.9185 - type: nauc_mrr_at_5_std value: -47.8265 - type: nauc_mrr_at_5_diff1 value: 71.5584 - type: nauc_mrr_at_10_max value: -9.023299999999999 - type: nauc_mrr_at_10_std value: -47.1288 - type: nauc_mrr_at_10_diff1 value: 71.7295 - type: nauc_mrr_at_20_max value: -8.9076 - type: nauc_mrr_at_20_std value: -46.9382 - type: nauc_mrr_at_20_diff1 value: 71.7573 - type: nauc_mrr_at_100_max value: -8.892700000000001 - type: nauc_mrr_at_100_std value: -46.6939 - type: nauc_mrr_at_100_diff1 value: 71.7782 - type: nauc_mrr_at_1000_max value: -8.8935 - type: nauc_mrr_at_1000_std value: -46.699400000000004 - type: nauc_mrr_at_1000_diff1 value: 71.7855 - type: main_score value: 78.549 task: type: Retrieval tags: - sentence-transformers - transformers - sentence-similarity - mteb license: apache-2.0 --- # INF-Retriever-v1 ## Model Overview - **INF-Retriever-v1** is an LLM-based dense retrieval model developed by [INF TECH](https://www.infly.cn/en). It is built upon the [gte-Qwen2-7B-instruct](https://huggingface.co./Alibaba-NLP/gte-Qwen2-7B-instruct) model and specifically fine-tuned to excel in retrieval tasks, particularly for Chinese and English data. - As of January 23, 2025, **INF-Retriever-v1** ranks both **No.1** on the Automated Heterogeneous Information Retrieval Benchmark of version 24.04 & 24.05([AIR-Bench](https://huggingface.co./spaces/AIR-Bench/leaderboard)), showcasing its cutting-edge performance in heterogeneous information retrieval tasks. ## Key Features - **Optimized for Chinese and English retrieval**: The model has been specifically fine-tuned with retrieval-focused datasets in both languages, significantly improving its accuracy and efficiency for a variety of retrieval scenarios. - **Top-tier performance**: **INF-Retriever-v1** has achieved outstanding results on the AIR-Bench leaderboard, making it a top choice for heterogeneous information retrieval tasks across various domains. ## Usage ### Sentence Transformers ```python from sentence_transformers import SentenceTransformer model = SentenceTransformer("infly/inf-retriever-v1", trust_remote_code=True) # In case you want to reduce the maximum length: model.max_seq_length = 8192 queries = [ "how much protein should a female eat", "summit define", ] documents = [ "As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", "Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments.", ] query_embeddings = model.encode(queries, prompt_name="query") document_embeddings = model.encode(documents) scores = (query_embeddings @ document_embeddings.T) * 100 print(scores.tolist()) # [[86.8702392578125, 67.82364654541016], [59.51014709472656, 82.33668518066406]] ``` ### Transformers ```python import torch import torch.nn.functional as F from torch import Tensor from transformers import AutoTokenizer, AutoModel def last_token_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor: left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0]) if left_padding: return last_hidden_states[:, -1] else: sequence_lengths = attention_mask.sum(dim=1) - 1 batch_size = last_hidden_states.shape[0] return last_hidden_states[torch.arange(batch_size, device=last_hidden_states.device), sequence_lengths] def get_detailed_instruct(task_description: str, query: str) -> str: return f'Instruct: {task_description}\nQuery: {query}' # Each query must come with a one-sentence instruction that describes the task task = 'Given a web search query, retrieve relevant passages that answer the query' queries = [ get_detailed_instruct(task, 'how much protein should a female eat'), get_detailed_instruct(task, 'summit define') ] # No need to add instruction for retrieval documents documents = [ "As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", "Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments." ] input_texts = queries + documents tokenizer = AutoTokenizer.from_pretrained('infly/inf-retriever-v1', trust_remote_code=True) model = AutoModel.from_pretrained('infly/inf-retriever-v1', trust_remote_code=True) max_length = 8192 # Tokenize the input texts batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt') outputs = model(**batch_dict) embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask']) # normalize embeddings embeddings = F.normalize(embeddings, p=2, dim=1) scores = (embeddings[:2] @ embeddings[2:].T) * 100 print(scores.tolist()) # [[86.87025451660156, 67.82366180419922], [59.510135650634766, 82.33667755126953]] ``` ## Evaluation ### AIR-Bench **INF-Retriever-v1** has demonstrated superior retrieval capabilities across multiple domains and languages. The results from the Automated Heterogeneous Information Retrieval Benchmark ([AIR-Bench](https://huggingface.co./spaces/AIR-Bench/leaderboard)) as of January 23, 2025, are as follows: #### AIR-Bench_24.04 (Bilingual, EN & ZH) | Model Name | Average⬆️ | wiki_en | wiki_zh | web_en | web_zh | healthcare_en | healthcare_zh | law_en | arxiv_en | news_en | news_zh | finance_en | finance_zh | msmarco_en | |-----------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|----------|---------------|---------------|-----------|-----------|-----------|-----------|------------|------------|------------| | [E5-mistral-7b-instruct](https://huggingface.co./intfloat/e5-mistral-7b-instruct) | 45.26 | 61.67 | 55.97 | 44.41 | 45.96 | 56.32 | 35.79 | 19.32 | 44.78 | 48.18 | 35.99 | 54.79 | 26.11 | 59.03 | | [BGE-M3](https://huggingface.co./BAAI/bge-m3) | 46.65 | 60.49 | 62.36 | 47.35 | 50.38 | 49.1 | **42.38** | 26.68 | 40.76 | 48.04 | 40.75 | 51.52 | 32.18 | 54.4 | | [BGE-Multilingual-Gemma2](https://huggingface.co./BAAI/bge-multilingual-gemma2) | 46.83 | 63.71 | 67.3 | 50.38 | 53.24 | 47.24 | 42.13 | 22.58 | 23.28 | 50.91 | 44.02 | 49.3 | 31.6 | **63.14** | | [GTE-Qwen2-7B-instruct](https://huggingface.co./Alibaba-NLP/gte-Qwen2-7B-instruct) | 48.38 | 63.46 | 66.44 | 51.2 | 51.98 | 54.2 | 38.82 | 22.31 | 40.27 | **54.07** | 43.03 | 58.2 | 26.63 | 58.39 | | **INF-Retriever-v1** | **52.56** | **65.25** | **68.44** | **52.13** | **56.6** | **56.96** | 42.03 | **34.51** | **50.62** | 53.32 | **50.02** | **58.34** | **35.42** | 59.64 | #### AIR-Bench_24.05 (Multilingual, 13 languages) Although INF-Retriever-v1 has been fine-tuned exclusively on English and Chinese, it continues to perform exceptionally well across other languages, securing the No. 1 position on this multilingual benchmark. | Model Name | Average⬆️ | wiki_en | wiki_zh | wiki_ar | wiki_bn | wiki_de | wiki_es | wiki_fa | wiki_fr | wiki_hi | wiki_id | wiki_ja | wiki_ko | wiki_ru | web_en | web_zh | web_ar | web_bn | web_de | web_es | web_fa | web_fr | web_hi | web_id | web_ja | web_ko | web_ru | healthcare_en | healthcare_zh | healthcare_de | healthcare_es | healthcare_fr | law_en | law_de | law_fr | arxiv_en | science_ru | news_en | news_zh | news_ar | news_bn | news_de | news_es | news_fa | news_fr | news_hi | news_id | news_ja | news_ko | news_ru | finance_en | finance_zh | finance_ar | finance_fr | |--------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|----------|-----------|-----------|----------|--------|-----------|-----------|-----------|---------------|---------------|---------------|---------------|---------------|-----------|-----------|-----------|-----------|------------|-----------|-----------|-----------|-----------|-----------|----------|-----------|----------|-----------|-----------|-----------|-----------|-----------|------------|------------|------------|------------| | [GTE-Qwen2-7B-instruct](https://huggingface.co./Alibaba-NLP/gte-Qwen2-7B-instruct) | 50.05 | **73.59** | 67.5 | 59.44 | 58.17 | 63.96 | 67.62 | 57.05 | 70.32 | 60.54 | 61.81 | 62.88 | 59.17 | 62.95 | **58.99** | 51.66 | 55.56 | 51.45 | 48.62 | 54.11 | 49.54 | 55.16 | 53.06 | 55.51 | 57.27 | 57.54 | 55.88 | 54.46 | 38.66 | 53.92 | 53.78 | 30.29 | 22.75 | 13.18 | 13.15 | 41.32 | 45.21 | **52.74** | 43.17 | 37.63 | **61.31** | 44.89 | 45.21 | 30.1 | 49.76 | 30.28 | 46.44 | 44.13 | 47.19 | 46.55 | 59.23 | 34.61 | 43.56 | 39.57 | | [Multilingual-E5-large-instruct](https://huggingface.co./intfloat/multilingual-e5-large-instruct) | 51.11 | 68.62 | 62.82 | 63.21 | 64.45 | 65.81 | 68.1 | 64.2 | 69.72 | 71.81 | 66.36 | 64.12 | 64.79 | 62.57 | 41.58 | 47.06 | 56.4 | 56.17 | 50.87 | 52.24 | 58.68 | 50.2 | 56.32 | 54.49 | 54.89 | 55.81 | 54.97 | 54.02 | 39.76 | 52.06 | 51.74 | 36.64 | 16.9 | 15.59 | 15.12 | 39.52 | 56.86 | 44.28 | 35.46 | 48.2 | 49.31 | 47.84 | 45.99 | **45.59** | 50.58 | 39.66 | 48.59 | 47.6 | 50.52 | 48.81 | 52.79 | 37.72 | 48.95 | 42.74 | | [BGE-M3](https://huggingface.co./BAAI/bge-m3) | 51.31 | 69.7 | 63.52 | 59.65 | 64.33 | 64.68 | 65.4 | 61.14 | 66.04 | 69.02 | 66.3 | 60.86 | 62.36 | 60.18 | 53.88 | 50.2 | 52.53 | 55.53 | 51.89 | 51.78 | 55.81 | 51.46 | 57.06 | 53.14 | 54.75 | 55.28 | 54.53 | 49.05 | 42.31 | 49 | 53.05 | 39.29 | 26.95 | 20.11 | 20.2 | 41.64 | 55.18 | 47.34 | 41 | 44.93 | 59.03 | 47.87 | 44.7 | 43.81 | 49.52 | 42.12 | 47.45 | 47.09 | 48.14 | 48.31 | 52.92 | 40.23 | 45.76 | 41.44 | | [BGE-Multilingual-Gemma2](https://huggingface.co./BAAI/bge-multilingual-gemma2) | 54.46 | 72.8 | 68.64 | **63.42** | **69.48** | **67.91** | **71.79** | **67.57** | **71.28** | **75.39** | **68.91** | **68.29** | **66.78** | **64.15** | 56.48 | 53.04 | **59.97** | **59.68** | **57.72** | **58.2** | **62.43** | **59.54** | **64.5** | **60** | **60.26** | 59.64 | **60.12** | 47.48 | **42.35** | 55.4 | **63.13** | **45.13** | 22.6 | 15.75 | 14.29 | 24 | 44.13 | 50.29 | 43.42 | 48.41 | 58.77 | **52.05** | **49.9** | 43.4 | **56.8** | **44.89** | 50.65 | **51.51** | 51.64 | 51.48 | 50.08 | 39.23 | 50.25 | **51.1** | | **INF-Retriever-v1** | **54.47** | 73.52 | **69.45** | 63.13 | 61.58 | 66.8 | 69.29 | 63.03 | 69.74 | 69.02 | 68.63 | 63.45 | 64.44 | 62.74 | 57.6 | **56.46** | 58.48 | 53.7 | 55.2 | 57.08 | 53.27 | 57.35 | 55.64 | 58.85 | 59.52 | **60.01** | 58.79 | **57.03** | 41.82 | **55.46** | 57.6 | 43.25 | **34.76** | **21.75** | **21.87** | **51.38** | **59.72** | 52.7 | **49.78** | **49.11** | 43.62 | 51.47 | 49.52 | 40.43 | 54.54 | 38.57 | **51.06** | 51.12 | **53.15** | **51.88** | **59.44** | **44.13** | **50.71** | 44.2 | ## Contributors ### Supervisors Wei Chu • Yinghui Xu • Yuan Qi ### INF memory team Junhan Yang (junhanyang@inftech.ai) • Jiahe Wan • Yichen Yao (eason.yyc@inftech.ai)