| --- |
| license: bigscience-bloom-rail-1.0 |
| language: |
| - ak |
| - ar |
| - as |
| - bm |
| - bn |
| - ca |
| - code |
| - en |
| - es |
| - eu |
| - fon |
| - fr |
| - gu |
| - hi |
| - id |
| - ig |
| - ki |
| - kn |
| - lg |
| - ln |
| - ml |
| - mr |
| - ne |
| - nso |
| - ny |
| - or |
| - pa |
| - pt |
| - rn |
| - rw |
| - sn |
| - st |
| - sw |
| - ta |
| - te |
| - tn |
| - ts |
| - tum |
| - tw |
| - ur |
| - vi |
| - wo |
| - xh |
| - yo |
| - zh |
| - zhs |
| - zht |
| - zu |
| tags: |
| - mteb |
| model-index: |
| - name: udever-bloom-3b |
| results: |
| - task: |
| type: STS |
| dataset: |
| type: C-MTEB/AFQMC |
| name: MTEB AFQMC |
| config: default |
| split: validation |
| revision: None |
| metrics: |
| - type: cos_sim_pearson |
| value: 30.0892025910701 |
| - type: cos_sim_spearman |
| value: 30.549960550731782 |
| - type: euclidean_pearson |
| value: 29.68940732194022 |
| - type: euclidean_spearman |
| value: 30.254869740623715 |
| - type: manhattan_pearson |
| value: 29.693089299297732 |
| - type: manhattan_spearman |
| value: 30.21293218369479 |
| - task: |
| type: STS |
| dataset: |
| type: C-MTEB/ATEC |
| name: MTEB ATEC |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: cos_sim_pearson |
| value: 36.469490571108054 |
| - type: cos_sim_spearman |
| value: 37.34843946308442 |
| - type: euclidean_pearson |
| value: 39.697664194640886 |
| - type: euclidean_spearman |
| value: 37.623976566242334 |
| - type: manhattan_pearson |
| value: 39.8389981955552 |
| - type: manhattan_spearman |
| value: 37.689111419556 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/amazon_counterfactual |
| name: MTEB AmazonCounterfactualClassification (en) |
| config: en |
| split: test |
| revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
| metrics: |
| - type: accuracy |
| value: 78.8955223880597 |
| - type: ap |
| value: 43.270679598956285 |
| - type: f1 |
| value: 73.10740489387823 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/amazon_polarity |
| name: MTEB AmazonPolarityClassification |
| config: default |
| split: test |
| revision: e2d317d38cd51312af73b3d32a06d1a08b442046 |
| metrics: |
| - type: accuracy |
| value: 87.981225 |
| - type: ap |
| value: 83.55047186016726 |
| - type: f1 |
| value: 87.95185650917034 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/amazon_reviews_multi |
| name: MTEB AmazonReviewsClassification (en) |
| config: en |
| split: test |
| revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| metrics: |
| - type: accuracy |
| value: 42.58 |
| - type: f1 |
| value: 42.011158109228425 |
| - task: |
| type: Retrieval |
| dataset: |
| type: arguana |
| name: MTEB ArguAna |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 22.688 |
| - type: map_at_10 |
| value: 38.855000000000004 |
| - type: map_at_100 |
| value: 39.859 |
| - type: map_at_1000 |
| value: 39.871 |
| - type: map_at_3 |
| value: 33.428000000000004 |
| - type: map_at_5 |
| value: 36.571999999999996 |
| - type: mrr_at_1 |
| value: 23.044 |
| - type: mrr_at_10 |
| value: 39.022 |
| - type: mrr_at_100 |
| value: 40.019 |
| - type: mrr_at_1000 |
| value: 40.03 |
| - type: mrr_at_3 |
| value: 33.642 |
| - type: mrr_at_5 |
| value: 36.707 |
| - type: ndcg_at_1 |
| value: 22.688 |
| - type: ndcg_at_10 |
| value: 48.33 |
| - type: ndcg_at_100 |
| value: 52.616 |
| - type: ndcg_at_1000 |
| value: 52.891999999999996 |
| - type: ndcg_at_3 |
| value: 37.104 |
| - type: ndcg_at_5 |
| value: 42.764 |
| - type: precision_at_1 |
| value: 22.688 |
| - type: precision_at_10 |
| value: 7.881 |
| - type: precision_at_100 |
| value: 0.975 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: precision_at_3 |
| value: 15.931999999999999 |
| - type: precision_at_5 |
| value: 12.304 |
| - type: recall_at_1 |
| value: 22.688 |
| - type: recall_at_10 |
| value: 78.805 |
| - type: recall_at_100 |
| value: 97.51100000000001 |
| - type: recall_at_1000 |
| value: 99.644 |
| - type: recall_at_3 |
| value: 47.795 |
| - type: recall_at_5 |
| value: 61.522 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/arxiv-clustering-p2p |
| name: MTEB ArxivClusteringP2P |
| config: default |
| split: test |
| revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d |
| metrics: |
| - type: v_measure |
| value: 45.37384003345981 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/arxiv-clustering-s2s |
| name: MTEB ArxivClusteringS2S |
| config: default |
| split: test |
| revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 |
| metrics: |
| - type: v_measure |
| value: 36.52143615051018 |
| - task: |
| type: Reranking |
| dataset: |
| type: mteb/askubuntudupquestions-reranking |
| name: MTEB AskUbuntuDupQuestions |
| config: default |
| split: test |
| revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
| metrics: |
| - type: map |
| value: 59.91826882625199 |
| - type: mrr |
| value: 73.30530273051049 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/biosses-sts |
| name: MTEB BIOSSES |
| config: default |
| split: test |
| revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
| metrics: |
| - type: cos_sim_pearson |
| value: 86.80556032491437 |
| - type: cos_sim_spearman |
| value: 84.81639043031876 |
| - type: euclidean_pearson |
| value: 84.20426417923026 |
| - type: euclidean_spearman |
| value: 83.53503593258247 |
| - type: manhattan_pearson |
| value: 84.25387997667964 |
| - type: manhattan_spearman |
| value: 83.11394200032217 |
| - task: |
| type: STS |
| dataset: |
| type: C-MTEB/BQ |
| name: MTEB BQ |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: cos_sim_pearson |
| value: 47.017986848644625 |
| - type: cos_sim_spearman |
| value: 47.16708658456057 |
| - type: euclidean_pearson |
| value: 47.81098065168003 |
| - type: euclidean_spearman |
| value: 48.01014499886206 |
| - type: manhattan_pearson |
| value: 48.013333352251244 |
| - type: manhattan_spearman |
| value: 48.252964666749016 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/bucc-bitext-mining |
| name: MTEB BUCC (de-en) |
| config: de-en |
| split: test |
| revision: d51519689f32196a32af33b075a01d0e7c51e252 |
| metrics: |
| - type: accuracy |
| value: 71.78496868475992 |
| - type: f1 |
| value: 71.05715215634456 |
| - type: precision |
| value: 70.7532208520454 |
| - type: recall |
| value: 71.78496868475992 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/bucc-bitext-mining |
| name: MTEB BUCC (fr-en) |
| config: fr-en |
| split: test |
| revision: d51519689f32196a32af33b075a01d0e7c51e252 |
| metrics: |
| - type: accuracy |
| value: 98.34910851860005 |
| - type: f1 |
| value: 98.16751045564604 |
| - type: precision |
| value: 98.07762858610317 |
| - type: recall |
| value: 98.34910851860005 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/bucc-bitext-mining |
| name: MTEB BUCC (ru-en) |
| config: ru-en |
| split: test |
| revision: d51519689f32196a32af33b075a01d0e7c51e252 |
| metrics: |
| - type: accuracy |
| value: 59.965361967440245 |
| - type: f1 |
| value: 58.44898687503467 |
| - type: precision |
| value: 57.83301194437321 |
| - type: recall |
| value: 59.965361967440245 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/bucc-bitext-mining |
| name: MTEB BUCC (zh-en) |
| config: zh-en |
| split: test |
| revision: d51519689f32196a32af33b075a01d0e7c51e252 |
| metrics: |
| - type: accuracy |
| value: 98.63085834649816 |
| - type: f1 |
| value: 98.59575215025451 |
| - type: precision |
| value: 98.5781990521327 |
| - type: recall |
| value: 98.63085834649816 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/banking77 |
| name: MTEB Banking77Classification |
| config: default |
| split: test |
| revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
| metrics: |
| - type: accuracy |
| value: 84.15584415584416 |
| - type: f1 |
| value: 84.1389435939967 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/biorxiv-clustering-p2p |
| name: MTEB BiorxivClusteringP2P |
| config: default |
| split: test |
| revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 |
| metrics: |
| - type: v_measure |
| value: 36.52184607783334 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/biorxiv-clustering-s2s |
| name: MTEB BiorxivClusteringS2S |
| config: default |
| split: test |
| revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 |
| metrics: |
| - type: v_measure |
| value: 31.976191171733653 |
| - task: |
| type: Clustering |
| dataset: |
| type: C-MTEB/CLSClusteringP2P |
| name: MTEB CLSClusteringP2P |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: v_measure |
| value: 36.733774048381484 |
| - task: |
| type: Clustering |
| dataset: |
| type: C-MTEB/CLSClusteringS2S |
| name: MTEB CLSClusteringS2S |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: v_measure |
| value: 36.451952183379056 |
| - task: |
| type: Reranking |
| dataset: |
| type: C-MTEB/CMedQAv1-reranking |
| name: MTEB CMedQAv1 |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map |
| value: 68.9131612041328 |
| - type: mrr |
| value: 73.47626984126985 |
| - task: |
| type: Reranking |
| dataset: |
| type: C-MTEB/CMedQAv2-reranking |
| name: MTEB CMedQAv2 |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map |
| value: 69.42233467142258 |
| - type: mrr |
| value: 74.22722222222221 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackAndroidRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 32.943 |
| - type: map_at_10 |
| value: 42.796 |
| - type: map_at_100 |
| value: 44.141999999999996 |
| - type: map_at_1000 |
| value: 44.277 |
| - type: map_at_3 |
| value: 39.201 |
| - type: map_at_5 |
| value: 41.262 |
| - type: mrr_at_1 |
| value: 41.488 |
| - type: mrr_at_10 |
| value: 49.214999999999996 |
| - type: mrr_at_100 |
| value: 50.02799999999999 |
| - type: mrr_at_1000 |
| value: 50.075 |
| - type: mrr_at_3 |
| value: 46.733000000000004 |
| - type: mrr_at_5 |
| value: 48.171 |
| - type: ndcg_at_1 |
| value: 41.488 |
| - type: ndcg_at_10 |
| value: 48.619 |
| - type: ndcg_at_100 |
| value: 53.868 |
| - type: ndcg_at_1000 |
| value: 56.027 |
| - type: ndcg_at_3 |
| value: 43.765 |
| - type: ndcg_at_5 |
| value: 45.974 |
| - type: precision_at_1 |
| value: 41.488 |
| - type: precision_at_10 |
| value: 9.07 |
| - type: precision_at_100 |
| value: 1.4460000000000002 |
| - type: precision_at_1000 |
| value: 0.19499999999999998 |
| - type: precision_at_3 |
| value: 20.649 |
| - type: precision_at_5 |
| value: 14.878 |
| - type: recall_at_1 |
| value: 32.943 |
| - type: recall_at_10 |
| value: 59.217 |
| - type: recall_at_100 |
| value: 81.337 |
| - type: recall_at_1000 |
| value: 95.185 |
| - type: recall_at_3 |
| value: 44.377 |
| - type: recall_at_5 |
| value: 51.088 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackEnglishRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 26.412999999999997 |
| - type: map_at_10 |
| value: 34.766999999999996 |
| - type: map_at_100 |
| value: 35.774 |
| - type: map_at_1000 |
| value: 35.894999999999996 |
| - type: map_at_3 |
| value: 31.935000000000002 |
| - type: map_at_5 |
| value: 33.661 |
| - type: mrr_at_1 |
| value: 33.248 |
| - type: mrr_at_10 |
| value: 40.274 |
| - type: mrr_at_100 |
| value: 40.92 |
| - type: mrr_at_1000 |
| value: 40.977000000000004 |
| - type: mrr_at_3 |
| value: 38.004 |
| - type: mrr_at_5 |
| value: 39.425 |
| - type: ndcg_at_1 |
| value: 33.248 |
| - type: ndcg_at_10 |
| value: 39.828 |
| - type: ndcg_at_100 |
| value: 43.863 |
| - type: ndcg_at_1000 |
| value: 46.228 |
| - type: ndcg_at_3 |
| value: 35.643 |
| - type: ndcg_at_5 |
| value: 37.851 |
| - type: precision_at_1 |
| value: 33.248 |
| - type: precision_at_10 |
| value: 7.4079999999999995 |
| - type: precision_at_100 |
| value: 1.162 |
| - type: precision_at_1000 |
| value: 0.168 |
| - type: precision_at_3 |
| value: 16.964000000000002 |
| - type: precision_at_5 |
| value: 12.267999999999999 |
| - type: recall_at_1 |
| value: 26.412999999999997 |
| - type: recall_at_10 |
| value: 48.93 |
| - type: recall_at_100 |
| value: 66.437 |
| - type: recall_at_1000 |
| value: 81.68900000000001 |
| - type: recall_at_3 |
| value: 36.822 |
| - type: recall_at_5 |
| value: 42.925000000000004 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackGamingRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 37.07 |
| - type: map_at_10 |
| value: 49.051 |
| - type: map_at_100 |
| value: 50.13999999999999 |
| - type: map_at_1000 |
| value: 50.2 |
| - type: map_at_3 |
| value: 46.01 |
| - type: map_at_5 |
| value: 47.711 |
| - type: mrr_at_1 |
| value: 42.32 |
| - type: mrr_at_10 |
| value: 52.32 |
| - type: mrr_at_100 |
| value: 53.068000000000005 |
| - type: mrr_at_1000 |
| value: 53.09700000000001 |
| - type: mrr_at_3 |
| value: 49.864000000000004 |
| - type: mrr_at_5 |
| value: 51.312000000000005 |
| - type: ndcg_at_1 |
| value: 42.32 |
| - type: ndcg_at_10 |
| value: 54.727000000000004 |
| - type: ndcg_at_100 |
| value: 59.153 |
| - type: ndcg_at_1000 |
| value: 60.373 |
| - type: ndcg_at_3 |
| value: 49.478 |
| - type: ndcg_at_5 |
| value: 51.998999999999995 |
| - type: precision_at_1 |
| value: 42.32 |
| - type: precision_at_10 |
| value: 8.802999999999999 |
| - type: precision_at_100 |
| value: 1.196 |
| - type: precision_at_1000 |
| value: 0.135 |
| - type: precision_at_3 |
| value: 22.006 |
| - type: precision_at_5 |
| value: 15.072 |
| - type: recall_at_1 |
| value: 37.07 |
| - type: recall_at_10 |
| value: 68.221 |
| - type: recall_at_100 |
| value: 87.22999999999999 |
| - type: recall_at_1000 |
| value: 95.929 |
| - type: recall_at_3 |
| value: 54.321 |
| - type: recall_at_5 |
| value: 60.358000000000004 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackGisRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 23.055 |
| - type: map_at_10 |
| value: 31.163999999999998 |
| - type: map_at_100 |
| value: 32.213 |
| - type: map_at_1000 |
| value: 32.303 |
| - type: map_at_3 |
| value: 28.610000000000003 |
| - type: map_at_5 |
| value: 30.091 |
| - type: mrr_at_1 |
| value: 24.972 |
| - type: mrr_at_10 |
| value: 32.981 |
| - type: mrr_at_100 |
| value: 33.948 |
| - type: mrr_at_1000 |
| value: 34.015 |
| - type: mrr_at_3 |
| value: 30.546 |
| - type: mrr_at_5 |
| value: 31.959 |
| - type: ndcg_at_1 |
| value: 24.972 |
| - type: ndcg_at_10 |
| value: 35.806 |
| - type: ndcg_at_100 |
| value: 40.991 |
| - type: ndcg_at_1000 |
| value: 43.296 |
| - type: ndcg_at_3 |
| value: 30.849 |
| - type: ndcg_at_5 |
| value: 33.334 |
| - type: precision_at_1 |
| value: 24.972 |
| - type: precision_at_10 |
| value: 5.571000000000001 |
| - type: precision_at_100 |
| value: 0.853 |
| - type: precision_at_1000 |
| value: 0.109 |
| - type: precision_at_3 |
| value: 12.956999999999999 |
| - type: precision_at_5 |
| value: 9.333 |
| - type: recall_at_1 |
| value: 23.055 |
| - type: recall_at_10 |
| value: 48.301 |
| - type: recall_at_100 |
| value: 72.051 |
| - type: recall_at_1000 |
| value: 89.408 |
| - type: recall_at_3 |
| value: 35.315000000000005 |
| - type: recall_at_5 |
| value: 41.031 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackMathematicaRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 14.782 |
| - type: map_at_10 |
| value: 21.94 |
| - type: map_at_100 |
| value: 23.172 |
| - type: map_at_1000 |
| value: 23.302999999999997 |
| - type: map_at_3 |
| value: 19.911 |
| - type: map_at_5 |
| value: 20.998 |
| - type: mrr_at_1 |
| value: 18.407999999999998 |
| - type: mrr_at_10 |
| value: 25.936999999999998 |
| - type: mrr_at_100 |
| value: 27.035999999999998 |
| - type: mrr_at_1000 |
| value: 27.118 |
| - type: mrr_at_3 |
| value: 23.983999999999998 |
| - type: mrr_at_5 |
| value: 25.141000000000002 |
| - type: ndcg_at_1 |
| value: 18.407999999999998 |
| - type: ndcg_at_10 |
| value: 26.387 |
| - type: ndcg_at_100 |
| value: 32.606 |
| - type: ndcg_at_1000 |
| value: 35.744 |
| - type: ndcg_at_3 |
| value: 22.686999999999998 |
| - type: ndcg_at_5 |
| value: 24.375 |
| - type: precision_at_1 |
| value: 18.407999999999998 |
| - type: precision_at_10 |
| value: 4.801 |
| - type: precision_at_100 |
| value: 0.9299999999999999 |
| - type: precision_at_1000 |
| value: 0.134 |
| - type: precision_at_3 |
| value: 10.945 |
| - type: precision_at_5 |
| value: 7.811 |
| - type: recall_at_1 |
| value: 14.782 |
| - type: recall_at_10 |
| value: 36.018 |
| - type: recall_at_100 |
| value: 63.552 |
| - type: recall_at_1000 |
| value: 85.857 |
| - type: recall_at_3 |
| value: 25.898 |
| - type: recall_at_5 |
| value: 30.081999999999997 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackPhysicsRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 26.369 |
| - type: map_at_10 |
| value: 37.704 |
| - type: map_at_100 |
| value: 39.018 |
| - type: map_at_1000 |
| value: 39.134 |
| - type: map_at_3 |
| value: 34.243 |
| - type: map_at_5 |
| value: 36.083 |
| - type: mrr_at_1 |
| value: 32.916000000000004 |
| - type: mrr_at_10 |
| value: 43.488 |
| - type: mrr_at_100 |
| value: 44.29 |
| - type: mrr_at_1000 |
| value: 44.336999999999996 |
| - type: mrr_at_3 |
| value: 40.696 |
| - type: mrr_at_5 |
| value: 42.289 |
| - type: ndcg_at_1 |
| value: 32.916000000000004 |
| - type: ndcg_at_10 |
| value: 44.362 |
| - type: ndcg_at_100 |
| value: 49.730999999999995 |
| - type: ndcg_at_1000 |
| value: 51.857 |
| - type: ndcg_at_3 |
| value: 38.683 |
| - type: ndcg_at_5 |
| value: 41.249 |
| - type: precision_at_1 |
| value: 32.916000000000004 |
| - type: precision_at_10 |
| value: 8.412 |
| - type: precision_at_100 |
| value: 1.2970000000000002 |
| - type: precision_at_1000 |
| value: 0.166 |
| - type: precision_at_3 |
| value: 18.895999999999997 |
| - type: precision_at_5 |
| value: 13.550999999999998 |
| - type: recall_at_1 |
| value: 26.369 |
| - type: recall_at_10 |
| value: 58.464000000000006 |
| - type: recall_at_100 |
| value: 80.884 |
| - type: recall_at_1000 |
| value: 94.676 |
| - type: recall_at_3 |
| value: 42.485 |
| - type: recall_at_5 |
| value: 49.262 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackProgrammersRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 23.896 |
| - type: map_at_10 |
| value: 33.384 |
| - type: map_at_100 |
| value: 34.683 |
| - type: map_at_1000 |
| value: 34.807 |
| - type: map_at_3 |
| value: 30.724 |
| - type: map_at_5 |
| value: 32.339 |
| - type: mrr_at_1 |
| value: 29.909000000000002 |
| - type: mrr_at_10 |
| value: 38.395 |
| - type: mrr_at_100 |
| value: 39.339 |
| - type: mrr_at_1000 |
| value: 39.404 |
| - type: mrr_at_3 |
| value: 36.339 |
| - type: mrr_at_5 |
| value: 37.618 |
| - type: ndcg_at_1 |
| value: 29.909000000000002 |
| - type: ndcg_at_10 |
| value: 38.688 |
| - type: ndcg_at_100 |
| value: 44.399 |
| - type: ndcg_at_1000 |
| value: 46.942 |
| - type: ndcg_at_3 |
| value: 34.548 |
| - type: ndcg_at_5 |
| value: 36.605 |
| - type: precision_at_1 |
| value: 29.909000000000002 |
| - type: precision_at_10 |
| value: 7.066 |
| - type: precision_at_100 |
| value: 1.174 |
| - type: precision_at_1000 |
| value: 0.155 |
| - type: precision_at_3 |
| value: 16.819 |
| - type: precision_at_5 |
| value: 11.872 |
| - type: recall_at_1 |
| value: 23.896 |
| - type: recall_at_10 |
| value: 49.531 |
| - type: recall_at_100 |
| value: 73.977 |
| - type: recall_at_1000 |
| value: 91.393 |
| - type: recall_at_3 |
| value: 37.53 |
| - type: recall_at_5 |
| value: 43.373 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 24.153166666666667 |
| - type: map_at_10 |
| value: 32.7705 |
| - type: map_at_100 |
| value: 33.93133333333334 |
| - type: map_at_1000 |
| value: 34.052499999999995 |
| - type: map_at_3 |
| value: 30.158500000000004 |
| - type: map_at_5 |
| value: 31.595916666666664 |
| - type: mrr_at_1 |
| value: 28.87725 |
| - type: mrr_at_10 |
| value: 36.86358333333333 |
| - type: mrr_at_100 |
| value: 37.74550000000001 |
| - type: mrr_at_1000 |
| value: 37.80916666666666 |
| - type: mrr_at_3 |
| value: 34.634499999999996 |
| - type: mrr_at_5 |
| value: 35.926750000000006 |
| - type: ndcg_at_1 |
| value: 28.87725 |
| - type: ndcg_at_10 |
| value: 37.82341666666667 |
| - type: ndcg_at_100 |
| value: 42.98408333333333 |
| - type: ndcg_at_1000 |
| value: 45.44883333333333 |
| - type: ndcg_at_3 |
| value: 33.41875000000001 |
| - type: ndcg_at_5 |
| value: 35.45158333333333 |
| - type: precision_at_1 |
| value: 28.87725 |
| - type: precision_at_10 |
| value: 6.638249999999999 |
| - type: precision_at_100 |
| value: 1.0863333333333334 |
| - type: precision_at_1000 |
| value: 0.14858333333333335 |
| - type: precision_at_3 |
| value: 15.481 |
| - type: precision_at_5 |
| value: 10.953916666666668 |
| - type: recall_at_1 |
| value: 24.153166666666667 |
| - type: recall_at_10 |
| value: 48.796499999999995 |
| - type: recall_at_100 |
| value: 71.53716666666666 |
| - type: recall_at_1000 |
| value: 88.72158333333333 |
| - type: recall_at_3 |
| value: 36.419583333333335 |
| - type: recall_at_5 |
| value: 41.735833333333325 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackStatsRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 21.523 |
| - type: map_at_10 |
| value: 28.915000000000003 |
| - type: map_at_100 |
| value: 29.808 |
| - type: map_at_1000 |
| value: 29.910999999999998 |
| - type: map_at_3 |
| value: 26.863999999999997 |
| - type: map_at_5 |
| value: 27.801 |
| - type: mrr_at_1 |
| value: 24.387 |
| - type: mrr_at_10 |
| value: 31.703 |
| - type: mrr_at_100 |
| value: 32.481 |
| - type: mrr_at_1000 |
| value: 32.559 |
| - type: mrr_at_3 |
| value: 29.805999999999997 |
| - type: mrr_at_5 |
| value: 30.688 |
| - type: ndcg_at_1 |
| value: 24.387 |
| - type: ndcg_at_10 |
| value: 33.272 |
| - type: ndcg_at_100 |
| value: 37.79 |
| - type: ndcg_at_1000 |
| value: 40.428 |
| - type: ndcg_at_3 |
| value: 29.409000000000002 |
| - type: ndcg_at_5 |
| value: 30.813000000000002 |
| - type: precision_at_1 |
| value: 24.387 |
| - type: precision_at_10 |
| value: 5.337 |
| - type: precision_at_100 |
| value: 0.8240000000000001 |
| - type: precision_at_1000 |
| value: 0.11299999999999999 |
| - type: precision_at_3 |
| value: 13.19 |
| - type: precision_at_5 |
| value: 8.926 |
| - type: recall_at_1 |
| value: 21.523 |
| - type: recall_at_10 |
| value: 44.054 |
| - type: recall_at_100 |
| value: 64.80900000000001 |
| - type: recall_at_1000 |
| value: 84.265 |
| - type: recall_at_3 |
| value: 33.019999999999996 |
| - type: recall_at_5 |
| value: 36.561 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackTexRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 15.461 |
| - type: map_at_10 |
| value: 21.802 |
| - type: map_at_100 |
| value: 22.825 |
| - type: map_at_1000 |
| value: 22.95 |
| - type: map_at_3 |
| value: 19.79 |
| - type: map_at_5 |
| value: 20.828 |
| - type: mrr_at_1 |
| value: 18.789 |
| - type: mrr_at_10 |
| value: 25.373 |
| - type: mrr_at_100 |
| value: 26.269 |
| - type: mrr_at_1000 |
| value: 26.355 |
| - type: mrr_at_3 |
| value: 23.394000000000002 |
| - type: mrr_at_5 |
| value: 24.451999999999998 |
| - type: ndcg_at_1 |
| value: 18.789 |
| - type: ndcg_at_10 |
| value: 25.948 |
| - type: ndcg_at_100 |
| value: 30.926 |
| - type: ndcg_at_1000 |
| value: 33.938 |
| - type: ndcg_at_3 |
| value: 22.281000000000002 |
| - type: ndcg_at_5 |
| value: 23.818 |
| - type: precision_at_1 |
| value: 18.789 |
| - type: precision_at_10 |
| value: 4.766 |
| - type: precision_at_100 |
| value: 0.848 |
| - type: precision_at_1000 |
| value: 0.127 |
| - type: precision_at_3 |
| value: 10.633 |
| - type: precision_at_5 |
| value: 7.6259999999999994 |
| - type: recall_at_1 |
| value: 15.461 |
| - type: recall_at_10 |
| value: 34.967999999999996 |
| - type: recall_at_100 |
| value: 57.25900000000001 |
| - type: recall_at_1000 |
| value: 78.738 |
| - type: recall_at_3 |
| value: 24.495 |
| - type: recall_at_5 |
| value: 28.510999999999996 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackUnixRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 25.165 |
| - type: map_at_10 |
| value: 32.66 |
| - type: map_at_100 |
| value: 33.842 |
| - type: map_at_1000 |
| value: 33.952 |
| - type: map_at_3 |
| value: 30.503999999999998 |
| - type: map_at_5 |
| value: 31.546000000000003 |
| - type: mrr_at_1 |
| value: 29.851 |
| - type: mrr_at_10 |
| value: 37.112 |
| - type: mrr_at_100 |
| value: 38.057 |
| - type: mrr_at_1000 |
| value: 38.119 |
| - type: mrr_at_3 |
| value: 35.106 |
| - type: mrr_at_5 |
| value: 36.22 |
| - type: ndcg_at_1 |
| value: 29.851 |
| - type: ndcg_at_10 |
| value: 37.395 |
| - type: ndcg_at_100 |
| value: 42.906 |
| - type: ndcg_at_1000 |
| value: 45.427 |
| - type: ndcg_at_3 |
| value: 33.465 |
| - type: ndcg_at_5 |
| value: 35.02 |
| - type: precision_at_1 |
| value: 29.851 |
| - type: precision_at_10 |
| value: 6.166 |
| - type: precision_at_100 |
| value: 1.005 |
| - type: precision_at_1000 |
| value: 0.132 |
| - type: precision_at_3 |
| value: 15.235999999999999 |
| - type: precision_at_5 |
| value: 10.354 |
| - type: recall_at_1 |
| value: 25.165 |
| - type: recall_at_10 |
| value: 47.439 |
| - type: recall_at_100 |
| value: 71.56099999999999 |
| - type: recall_at_1000 |
| value: 89.435 |
| - type: recall_at_3 |
| value: 36.275 |
| - type: recall_at_5 |
| value: 40.435 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackWebmastersRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 25.589000000000002 |
| - type: map_at_10 |
| value: 33.729 |
| - type: map_at_100 |
| value: 35.306 |
| - type: map_at_1000 |
| value: 35.552 |
| - type: map_at_3 |
| value: 30.988 |
| - type: map_at_5 |
| value: 32.406 |
| - type: mrr_at_1 |
| value: 30.830000000000002 |
| - type: mrr_at_10 |
| value: 38.446999999999996 |
| - type: mrr_at_100 |
| value: 39.478 |
| - type: mrr_at_1000 |
| value: 39.544000000000004 |
| - type: mrr_at_3 |
| value: 36.034 |
| - type: mrr_at_5 |
| value: 37.546 |
| - type: ndcg_at_1 |
| value: 30.830000000000002 |
| - type: ndcg_at_10 |
| value: 39.22 |
| - type: ndcg_at_100 |
| value: 45.004 |
| - type: ndcg_at_1000 |
| value: 47.837 |
| - type: ndcg_at_3 |
| value: 34.811 |
| - type: ndcg_at_5 |
| value: 36.831 |
| - type: precision_at_1 |
| value: 30.830000000000002 |
| - type: precision_at_10 |
| value: 7.489999999999999 |
| - type: precision_at_100 |
| value: 1.534 |
| - type: precision_at_1000 |
| value: 0.241 |
| - type: precision_at_3 |
| value: 16.14 |
| - type: precision_at_5 |
| value: 11.66 |
| - type: recall_at_1 |
| value: 25.589000000000002 |
| - type: recall_at_10 |
| value: 49.238 |
| - type: recall_at_100 |
| value: 74.893 |
| - type: recall_at_1000 |
| value: 92.902 |
| - type: recall_at_3 |
| value: 36.75 |
| - type: recall_at_5 |
| value: 42.256 |
| - task: |
| type: Retrieval |
| dataset: |
| type: BeIR/cqadupstack |
| name: MTEB CQADupstackWordpressRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 17.572 |
| - type: map_at_10 |
| value: 25.334 |
| - type: map_at_100 |
| value: 26.253 |
| - type: map_at_1000 |
| value: 26.346000000000004 |
| - type: map_at_3 |
| value: 23.122 |
| - type: map_at_5 |
| value: 24.425 |
| - type: mrr_at_1 |
| value: 19.409000000000002 |
| - type: mrr_at_10 |
| value: 27.118 |
| - type: mrr_at_100 |
| value: 28.032 |
| - type: mrr_at_1000 |
| value: 28.110000000000003 |
| - type: mrr_at_3 |
| value: 25.108000000000004 |
| - type: mrr_at_5 |
| value: 26.3 |
| - type: ndcg_at_1 |
| value: 19.409000000000002 |
| - type: ndcg_at_10 |
| value: 29.629 |
| - type: ndcg_at_100 |
| value: 34.572 |
| - type: ndcg_at_1000 |
| value: 37.289 |
| - type: ndcg_at_3 |
| value: 25.406000000000002 |
| - type: ndcg_at_5 |
| value: 27.55 |
| - type: precision_at_1 |
| value: 19.409000000000002 |
| - type: precision_at_10 |
| value: 4.769 |
| - type: precision_at_100 |
| value: 0.767 |
| - type: precision_at_1000 |
| value: 0.108 |
| - type: precision_at_3 |
| value: 11.337 |
| - type: precision_at_5 |
| value: 8.096 |
| - type: recall_at_1 |
| value: 17.572 |
| - type: recall_at_10 |
| value: 41.177 |
| - type: recall_at_100 |
| value: 64.456 |
| - type: recall_at_1000 |
| value: 85.182 |
| - type: recall_at_3 |
| value: 29.747 |
| - type: recall_at_5 |
| value: 34.948 |
| - task: |
| type: Retrieval |
| dataset: |
| type: climate-fever |
| name: MTEB ClimateFEVER |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 9.264 |
| - type: map_at_10 |
| value: 16.09 |
| - type: map_at_100 |
| value: 17.717 |
| - type: map_at_1000 |
| value: 17.903 |
| - type: map_at_3 |
| value: 13.422 |
| - type: map_at_5 |
| value: 14.78 |
| - type: mrr_at_1 |
| value: 20.326 |
| - type: mrr_at_10 |
| value: 31.274 |
| - type: mrr_at_100 |
| value: 32.312999999999995 |
| - type: mrr_at_1000 |
| value: 32.365 |
| - type: mrr_at_3 |
| value: 27.959 |
| - type: mrr_at_5 |
| value: 29.877 |
| - type: ndcg_at_1 |
| value: 20.326 |
| - type: ndcg_at_10 |
| value: 23.358 |
| - type: ndcg_at_100 |
| value: 30.36 |
| - type: ndcg_at_1000 |
| value: 33.883 |
| - type: ndcg_at_3 |
| value: 18.704 |
| - type: ndcg_at_5 |
| value: 20.374 |
| - type: precision_at_1 |
| value: 20.326 |
| - type: precision_at_10 |
| value: 7.303 |
| - type: precision_at_100 |
| value: 1.488 |
| - type: precision_at_1000 |
| value: 0.214 |
| - type: precision_at_3 |
| value: 13.811000000000002 |
| - type: precision_at_5 |
| value: 10.84 |
| - type: recall_at_1 |
| value: 9.264 |
| - type: recall_at_10 |
| value: 29.177999999999997 |
| - type: recall_at_100 |
| value: 53.61900000000001 |
| - type: recall_at_1000 |
| value: 73.48400000000001 |
| - type: recall_at_3 |
| value: 17.738 |
| - type: recall_at_5 |
| value: 22.279 |
| - task: |
| type: Retrieval |
| dataset: |
| type: C-MTEB/CmedqaRetrieval |
| name: MTEB CmedqaRetrieval |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 14.494000000000002 |
| - type: map_at_10 |
| value: 21.37 |
| - type: map_at_100 |
| value: 22.741 |
| - type: map_at_1000 |
| value: 22.911 |
| - type: map_at_3 |
| value: 18.929000000000002 |
| - type: map_at_5 |
| value: 20.244 |
| - type: mrr_at_1 |
| value: 23.105999999999998 |
| - type: mrr_at_10 |
| value: 29.137999999999998 |
| - type: mrr_at_100 |
| value: 30.064 |
| - type: mrr_at_1000 |
| value: 30.152 |
| - type: mrr_at_3 |
| value: 27.119 |
| - type: mrr_at_5 |
| value: 28.301 |
| - type: ndcg_at_1 |
| value: 23.105999999999998 |
| - type: ndcg_at_10 |
| value: 26.182 |
| - type: ndcg_at_100 |
| value: 32.396 |
| - type: ndcg_at_1000 |
| value: 36.177 |
| - type: ndcg_at_3 |
| value: 22.708000000000002 |
| - type: ndcg_at_5 |
| value: 24.137 |
| - type: precision_at_1 |
| value: 23.105999999999998 |
| - type: precision_at_10 |
| value: 6.0040000000000004 |
| - type: precision_at_100 |
| value: 1.119 |
| - type: precision_at_1000 |
| value: 0.161 |
| - type: precision_at_3 |
| value: 13.028 |
| - type: precision_at_5 |
| value: 9.557 |
| - type: recall_at_1 |
| value: 14.494000000000002 |
| - type: recall_at_10 |
| value: 32.910000000000004 |
| - type: recall_at_100 |
| value: 59.202999999999996 |
| - type: recall_at_1000 |
| value: 85.61 |
| - type: recall_at_3 |
| value: 22.397 |
| - type: recall_at_5 |
| value: 26.900000000000002 |
| - task: |
| type: PairClassification |
| dataset: |
| type: C-MTEB/CMNLI |
| name: MTEB Cmnli |
| config: default |
| split: validation |
| revision: None |
| metrics: |
| - type: cos_sim_accuracy |
| value: 74.91280817799158 |
| - type: cos_sim_ap |
| value: 83.32013347926805 |
| - type: cos_sim_f1 |
| value: 76.57387580299788 |
| - type: cos_sim_precision |
| value: 70.63006122852063 |
| - type: cos_sim_recall |
| value: 83.61000701426234 |
| - type: dot_accuracy |
| value: 70.5832832230908 |
| - type: dot_ap |
| value: 75.9647326130666 |
| - type: dot_f1 |
| value: 73.65528072241852 |
| - type: dot_precision |
| value: 63.47487734731856 |
| - type: dot_recall |
| value: 87.72504091653029 |
| - type: euclidean_accuracy |
| value: 74.51593505712569 |
| - type: euclidean_ap |
| value: 83.04382773676555 |
| - type: euclidean_f1 |
| value: 75.7739770513098 |
| - type: euclidean_precision |
| value: 70.5502922797823 |
| - type: euclidean_recall |
| value: 81.83306055646482 |
| - type: manhattan_accuracy |
| value: 74.73241130487071 |
| - type: manhattan_ap |
| value: 83.32768114935021 |
| - type: manhattan_f1 |
| value: 76.09116319071167 |
| - type: manhattan_precision |
| value: 70.42786069651741 |
| - type: manhattan_recall |
| value: 82.74491465980827 |
| - type: max_accuracy |
| value: 74.91280817799158 |
| - type: max_ap |
| value: 83.32768114935021 |
| - type: max_f1 |
| value: 76.57387580299788 |
| - task: |
| type: Retrieval |
| dataset: |
| type: C-MTEB/CovidRetrieval |
| name: MTEB CovidRetrieval |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 55.032000000000004 |
| - type: map_at_10 |
| value: 63.517 |
| - type: map_at_100 |
| value: 64.159 |
| - type: map_at_1000 |
| value: 64.17699999999999 |
| - type: map_at_3 |
| value: 61.503 |
| - type: map_at_5 |
| value: 62.741 |
| - type: mrr_at_1 |
| value: 55.111 |
| - type: mrr_at_10 |
| value: 63.50900000000001 |
| - type: mrr_at_100 |
| value: 64.13499999999999 |
| - type: mrr_at_1000 |
| value: 64.153 |
| - type: mrr_at_3 |
| value: 61.521 |
| - type: mrr_at_5 |
| value: 62.759 |
| - type: ndcg_at_1 |
| value: 55.216 |
| - type: ndcg_at_10 |
| value: 67.569 |
| - type: ndcg_at_100 |
| value: 70.71 |
| - type: ndcg_at_1000 |
| value: 71.211 |
| - type: ndcg_at_3 |
| value: 63.543000000000006 |
| - type: ndcg_at_5 |
| value: 65.718 |
| - type: precision_at_1 |
| value: 55.216 |
| - type: precision_at_10 |
| value: 8.093 |
| - type: precision_at_100 |
| value: 0.96 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: precision_at_3 |
| value: 23.253 |
| - type: precision_at_5 |
| value: 15.026 |
| - type: recall_at_1 |
| value: 55.032000000000004 |
| - type: recall_at_10 |
| value: 80.163 |
| - type: recall_at_100 |
| value: 94.94200000000001 |
| - type: recall_at_1000 |
| value: 98.946 |
| - type: recall_at_3 |
| value: 69.231 |
| - type: recall_at_5 |
| value: 74.49900000000001 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dbpedia-entity |
| name: MTEB DBPedia |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 8.391 |
| - type: map_at_10 |
| value: 16.381999999999998 |
| - type: map_at_100 |
| value: 21.262 |
| - type: map_at_1000 |
| value: 22.461000000000002 |
| - type: map_at_3 |
| value: 12.471 |
| - type: map_at_5 |
| value: 14.016 |
| - type: mrr_at_1 |
| value: 62.25000000000001 |
| - type: mrr_at_10 |
| value: 69.64099999999999 |
| - type: mrr_at_100 |
| value: 70.114 |
| - type: mrr_at_1000 |
| value: 70.128 |
| - type: mrr_at_3 |
| value: 67.958 |
| - type: mrr_at_5 |
| value: 68.996 |
| - type: ndcg_at_1 |
| value: 50.375 |
| - type: ndcg_at_10 |
| value: 34.542 |
| - type: ndcg_at_100 |
| value: 37.265 |
| - type: ndcg_at_1000 |
| value: 44.324000000000005 |
| - type: ndcg_at_3 |
| value: 40.113 |
| - type: ndcg_at_5 |
| value: 37.177 |
| - type: precision_at_1 |
| value: 62.25000000000001 |
| - type: precision_at_10 |
| value: 26.05 |
| - type: precision_at_100 |
| value: 7.632999999999999 |
| - type: precision_at_1000 |
| value: 1.6209999999999998 |
| - type: precision_at_3 |
| value: 42.5 |
| - type: precision_at_5 |
| value: 35.199999999999996 |
| - type: recall_at_1 |
| value: 8.391 |
| - type: recall_at_10 |
| value: 21.099 |
| - type: recall_at_100 |
| value: 40.886 |
| - type: recall_at_1000 |
| value: 63.805 |
| - type: recall_at_3 |
| value: 13.766 |
| - type: recall_at_5 |
| value: 16.128 |
| - task: |
| type: Retrieval |
| dataset: |
| type: C-MTEB/DuRetrieval |
| name: MTEB DuRetrieval |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 21.933 |
| - type: map_at_10 |
| value: 65.739 |
| - type: map_at_100 |
| value: 69.245 |
| - type: map_at_1000 |
| value: 69.33399999999999 |
| - type: map_at_3 |
| value: 44.874 |
| - type: map_at_5 |
| value: 56.242999999999995 |
| - type: mrr_at_1 |
| value: 78.95 |
| - type: mrr_at_10 |
| value: 85.37700000000001 |
| - type: mrr_at_100 |
| value: 85.474 |
| - type: mrr_at_1000 |
| value: 85.481 |
| - type: mrr_at_3 |
| value: 84.63300000000001 |
| - type: mrr_at_5 |
| value: 85.141 |
| - type: ndcg_at_1 |
| value: 78.95 |
| - type: ndcg_at_10 |
| value: 75.81599999999999 |
| - type: ndcg_at_100 |
| value: 80.42399999999999 |
| - type: ndcg_at_1000 |
| value: 81.357 |
| - type: ndcg_at_3 |
| value: 73.821 |
| - type: ndcg_at_5 |
| value: 72.497 |
| - type: precision_at_1 |
| value: 78.95 |
| - type: precision_at_10 |
| value: 37.285000000000004 |
| - type: precision_at_100 |
| value: 4.589 |
| - type: precision_at_1000 |
| value: 0.481 |
| - type: precision_at_3 |
| value: 66.333 |
| - type: precision_at_5 |
| value: 55.879999999999995 |
| - type: recall_at_1 |
| value: 21.933 |
| - type: recall_at_10 |
| value: 77.943 |
| - type: recall_at_100 |
| value: 92.17 |
| - type: recall_at_1000 |
| value: 96.986 |
| - type: recall_at_3 |
| value: 48.079 |
| - type: recall_at_5 |
| value: 62.65500000000001 |
| - task: |
| type: Retrieval |
| dataset: |
| type: C-MTEB/EcomRetrieval |
| name: MTEB EcomRetrieval |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 38.2 |
| - type: map_at_10 |
| value: 46.785 |
| - type: map_at_100 |
| value: 47.635 |
| - type: map_at_1000 |
| value: 47.675 |
| - type: map_at_3 |
| value: 44.583 |
| - type: map_at_5 |
| value: 45.848 |
| - type: mrr_at_1 |
| value: 38.2 |
| - type: mrr_at_10 |
| value: 46.785 |
| - type: mrr_at_100 |
| value: 47.635 |
| - type: mrr_at_1000 |
| value: 47.675 |
| - type: mrr_at_3 |
| value: 44.583 |
| - type: mrr_at_5 |
| value: 45.848 |
| - type: ndcg_at_1 |
| value: 38.2 |
| - type: ndcg_at_10 |
| value: 51.282000000000004 |
| - type: ndcg_at_100 |
| value: 55.608000000000004 |
| - type: ndcg_at_1000 |
| value: 56.726 |
| - type: ndcg_at_3 |
| value: 46.763 |
| - type: ndcg_at_5 |
| value: 49.035000000000004 |
| - type: precision_at_1 |
| value: 38.2 |
| - type: precision_at_10 |
| value: 6.550000000000001 |
| - type: precision_at_100 |
| value: 0.8619999999999999 |
| - type: precision_at_1000 |
| value: 0.095 |
| - type: precision_at_3 |
| value: 17.7 |
| - type: precision_at_5 |
| value: 11.72 |
| - type: recall_at_1 |
| value: 38.2 |
| - type: recall_at_10 |
| value: 65.5 |
| - type: recall_at_100 |
| value: 86.2 |
| - type: recall_at_1000 |
| value: 95.1 |
| - type: recall_at_3 |
| value: 53.1 |
| - type: recall_at_5 |
| value: 58.599999999999994 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/emotion |
| name: MTEB EmotionClassification |
| config: default |
| split: test |
| revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 |
| metrics: |
| - type: accuracy |
| value: 47.88 |
| - type: f1 |
| value: 43.30537129784135 |
| - task: |
| type: Retrieval |
| dataset: |
| type: fever |
| name: MTEB FEVER |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 54.423 |
| - type: map_at_10 |
| value: 66.136 |
| - type: map_at_100 |
| value: 66.557 |
| - type: map_at_1000 |
| value: 66.57300000000001 |
| - type: map_at_3 |
| value: 64.042 |
| - type: map_at_5 |
| value: 65.366 |
| - type: mrr_at_1 |
| value: 58.745999999999995 |
| - type: mrr_at_10 |
| value: 70.456 |
| - type: mrr_at_100 |
| value: 70.801 |
| - type: mrr_at_1000 |
| value: 70.809 |
| - type: mrr_at_3 |
| value: 68.504 |
| - type: mrr_at_5 |
| value: 69.746 |
| - type: ndcg_at_1 |
| value: 58.745999999999995 |
| - type: ndcg_at_10 |
| value: 71.96000000000001 |
| - type: ndcg_at_100 |
| value: 73.83 |
| - type: ndcg_at_1000 |
| value: 74.17 |
| - type: ndcg_at_3 |
| value: 68.033 |
| - type: ndcg_at_5 |
| value: 70.22 |
| - type: precision_at_1 |
| value: 58.745999999999995 |
| - type: precision_at_10 |
| value: 9.397 |
| - type: precision_at_100 |
| value: 1.043 |
| - type: precision_at_1000 |
| value: 0.108 |
| - type: precision_at_3 |
| value: 27.208 |
| - type: precision_at_5 |
| value: 17.561 |
| - type: recall_at_1 |
| value: 54.423 |
| - type: recall_at_10 |
| value: 85.703 |
| - type: recall_at_100 |
| value: 93.989 |
| - type: recall_at_1000 |
| value: 96.35000000000001 |
| - type: recall_at_3 |
| value: 75.05 |
| - type: recall_at_5 |
| value: 80.447 |
| - task: |
| type: Retrieval |
| dataset: |
| type: fiqa |
| name: MTEB FiQA2018 |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 16.286 |
| - type: map_at_10 |
| value: 27.499000000000002 |
| - type: map_at_100 |
| value: 29.176999999999996 |
| - type: map_at_1000 |
| value: 29.354999999999997 |
| - type: map_at_3 |
| value: 23.684 |
| - type: map_at_5 |
| value: 25.544 |
| - type: mrr_at_1 |
| value: 32.87 |
| - type: mrr_at_10 |
| value: 41.906 |
| - type: mrr_at_100 |
| value: 42.739 |
| - type: mrr_at_1000 |
| value: 42.78 |
| - type: mrr_at_3 |
| value: 38.992 |
| - type: mrr_at_5 |
| value: 40.535 |
| - type: ndcg_at_1 |
| value: 32.87 |
| - type: ndcg_at_10 |
| value: 35.124 |
| - type: ndcg_at_100 |
| value: 41.638 |
| - type: ndcg_at_1000 |
| value: 44.869 |
| - type: ndcg_at_3 |
| value: 30.975 |
| - type: ndcg_at_5 |
| value: 32.112 |
| - type: precision_at_1 |
| value: 32.87 |
| - type: precision_at_10 |
| value: 10.062 |
| - type: precision_at_100 |
| value: 1.653 |
| - type: precision_at_1000 |
| value: 0.22599999999999998 |
| - type: precision_at_3 |
| value: 20.833 |
| - type: precision_at_5 |
| value: 15.340000000000002 |
| - type: recall_at_1 |
| value: 16.286 |
| - type: recall_at_10 |
| value: 42.734 |
| - type: recall_at_100 |
| value: 67.582 |
| - type: recall_at_1000 |
| value: 86.735 |
| - type: recall_at_3 |
| value: 28.438000000000002 |
| - type: recall_at_5 |
| value: 33.944 |
| - task: |
| type: Retrieval |
| dataset: |
| type: hotpotqa |
| name: MTEB HotpotQA |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 33.606 |
| - type: map_at_10 |
| value: 46.085 |
| - type: map_at_100 |
| value: 46.796 |
| - type: map_at_1000 |
| value: 46.866 |
| - type: map_at_3 |
| value: 43.614000000000004 |
| - type: map_at_5 |
| value: 45.094 |
| - type: mrr_at_1 |
| value: 67.211 |
| - type: mrr_at_10 |
| value: 73.447 |
| - type: mrr_at_100 |
| value: 73.734 |
| - type: mrr_at_1000 |
| value: 73.752 |
| - type: mrr_at_3 |
| value: 72.233 |
| - type: mrr_at_5 |
| value: 72.982 |
| - type: ndcg_at_1 |
| value: 67.211 |
| - type: ndcg_at_10 |
| value: 55.125 |
| - type: ndcg_at_100 |
| value: 57.904999999999994 |
| - type: ndcg_at_1000 |
| value: 59.40800000000001 |
| - type: ndcg_at_3 |
| value: 51.283 |
| - type: ndcg_at_5 |
| value: 53.32599999999999 |
| - type: precision_at_1 |
| value: 67.211 |
| - type: precision_at_10 |
| value: 11.198 |
| - type: precision_at_100 |
| value: 1.34 |
| - type: precision_at_1000 |
| value: 0.154 |
| - type: precision_at_3 |
| value: 31.631999999999998 |
| - type: precision_at_5 |
| value: 20.591 |
| - type: recall_at_1 |
| value: 33.606 |
| - type: recall_at_10 |
| value: 55.989 |
| - type: recall_at_100 |
| value: 67.01599999999999 |
| - type: recall_at_1000 |
| value: 77.076 |
| - type: recall_at_3 |
| value: 47.448 |
| - type: recall_at_5 |
| value: 51.479 |
| - task: |
| type: Classification |
| dataset: |
| type: C-MTEB/IFlyTek-classification |
| name: MTEB IFlyTek |
| config: default |
| split: validation |
| revision: None |
| metrics: |
| - type: accuracy |
| value: 45.02500961908426 |
| - type: f1 |
| value: 36.80024928040335 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/imdb |
| name: MTEB ImdbClassification |
| config: default |
| split: test |
| revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
| metrics: |
| - type: accuracy |
| value: 77.698 |
| - type: ap |
| value: 72.08492726312224 |
| - type: f1 |
| value: 77.57721549038352 |
| - task: |
| type: Classification |
| dataset: |
| type: C-MTEB/JDReview-classification |
| name: MTEB JDReview |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: accuracy |
| value: 83.63977485928706 |
| - type: ap |
| value: 48.33680179995013 |
| - type: f1 |
| value: 77.42875376726259 |
| - task: |
| type: STS |
| dataset: |
| type: C-MTEB/LCQMC |
| name: MTEB LCQMC |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: cos_sim_pearson |
| value: 67.71826986847978 |
| - type: cos_sim_spearman |
| value: 75.31951271324436 |
| - type: euclidean_pearson |
| value: 73.99129929755692 |
| - type: euclidean_spearman |
| value: 75.50510874612128 |
| - type: manhattan_pearson |
| value: 74.1581557667118 |
| - type: manhattan_spearman |
| value: 75.62495446886778 |
| - task: |
| type: Retrieval |
| dataset: |
| type: C-MTEB/MMarcoRetrieval |
| name: MTEB MMarcoRetrieval |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 64.305 |
| - type: map_at_10 |
| value: 73.286 |
| - type: map_at_100 |
| value: 73.661 |
| - type: map_at_1000 |
| value: 73.675 |
| - type: map_at_3 |
| value: 71.433 |
| - type: map_at_5 |
| value: 72.596 |
| - type: mrr_at_1 |
| value: 66.562 |
| - type: mrr_at_10 |
| value: 73.932 |
| - type: mrr_at_100 |
| value: 74.265 |
| - type: mrr_at_1000 |
| value: 74.278 |
| - type: mrr_at_3 |
| value: 72.333 |
| - type: mrr_at_5 |
| value: 73.322 |
| - type: ndcg_at_1 |
| value: 66.562 |
| - type: ndcg_at_10 |
| value: 76.998 |
| - type: ndcg_at_100 |
| value: 78.684 |
| - type: ndcg_at_1000 |
| value: 79.038 |
| - type: ndcg_at_3 |
| value: 73.491 |
| - type: ndcg_at_5 |
| value: 75.436 |
| - type: precision_at_1 |
| value: 66.562 |
| - type: precision_at_10 |
| value: 9.34 |
| - type: precision_at_100 |
| value: 1.018 |
| - type: precision_at_1000 |
| value: 0.105 |
| - type: precision_at_3 |
| value: 27.683999999999997 |
| - type: precision_at_5 |
| value: 17.645 |
| - type: recall_at_1 |
| value: 64.305 |
| - type: recall_at_10 |
| value: 87.825 |
| - type: recall_at_100 |
| value: 95.451 |
| - type: recall_at_1000 |
| value: 98.17 |
| - type: recall_at_3 |
| value: 78.522 |
| - type: recall_at_5 |
| value: 83.146 |
| - task: |
| type: Retrieval |
| dataset: |
| type: msmarco |
| name: MTEB MSMARCO |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 21.862000000000002 |
| - type: map_at_10 |
| value: 33.635999999999996 |
| - type: map_at_100 |
| value: 34.833 |
| - type: map_at_1000 |
| value: 34.886 |
| - type: map_at_3 |
| value: 29.916999999999998 |
| - type: map_at_5 |
| value: 32.042 |
| - type: mrr_at_1 |
| value: 22.493 |
| - type: mrr_at_10 |
| value: 34.217999999999996 |
| - type: mrr_at_100 |
| value: 35.365 |
| - type: mrr_at_1000 |
| value: 35.411 |
| - type: mrr_at_3 |
| value: 30.585 |
| - type: mrr_at_5 |
| value: 32.659 |
| - type: ndcg_at_1 |
| value: 22.493 |
| - type: ndcg_at_10 |
| value: 40.247 |
| - type: ndcg_at_100 |
| value: 46.025 |
| - type: ndcg_at_1000 |
| value: 47.343 |
| - type: ndcg_at_3 |
| value: 32.696999999999996 |
| - type: ndcg_at_5 |
| value: 36.476 |
| - type: precision_at_1 |
| value: 22.493 |
| - type: precision_at_10 |
| value: 6.334 |
| - type: precision_at_100 |
| value: 0.922 |
| - type: precision_at_1000 |
| value: 0.104 |
| - type: precision_at_3 |
| value: 13.863 |
| - type: precision_at_5 |
| value: 10.232 |
| - type: recall_at_1 |
| value: 21.862000000000002 |
| - type: recall_at_10 |
| value: 60.56700000000001 |
| - type: recall_at_100 |
| value: 87.261 |
| - type: recall_at_1000 |
| value: 97.365 |
| - type: recall_at_3 |
| value: 40.081 |
| - type: recall_at_5 |
| value: 49.16 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/mtop_domain |
| name: MTEB MTOPDomainClassification (en) |
| config: en |
| split: test |
| revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
| metrics: |
| - type: accuracy |
| value: 92.34154126766987 |
| - type: f1 |
| value: 92.05415284766352 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/mtop_intent |
| name: MTEB MTOPIntentClassification (en) |
| config: en |
| split: test |
| revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
| metrics: |
| - type: accuracy |
| value: 70.63155494756043 |
| - type: f1 |
| value: 53.392602505424435 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/amazon_massive_intent |
| name: MTEB MassiveIntentClassification (en) |
| config: en |
| split: test |
| revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
| metrics: |
| - type: accuracy |
| value: 70.39340954942837 |
| - type: f1 |
| value: 68.85705470713275 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/amazon_massive_scenario |
| name: MTEB MassiveScenarioClassification (en) |
| config: en |
| split: test |
| revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
| metrics: |
| - type: accuracy |
| value: 77.18897108271688 |
| - type: f1 |
| value: 77.36699772115247 |
| - task: |
| type: Retrieval |
| dataset: |
| type: C-MTEB/MedicalRetrieval |
| name: MTEB MedicalRetrieval |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 40.699999999999996 |
| - type: map_at_10 |
| value: 45.304 |
| - type: map_at_100 |
| value: 45.862 |
| - type: map_at_1000 |
| value: 45.923 |
| - type: map_at_3 |
| value: 44.433 |
| - type: map_at_5 |
| value: 44.753 |
| - type: mrr_at_1 |
| value: 40.8 |
| - type: mrr_at_10 |
| value: 45.354 |
| - type: mrr_at_100 |
| value: 45.912 |
| - type: mrr_at_1000 |
| value: 45.973000000000006 |
| - type: mrr_at_3 |
| value: 44.483 |
| - type: mrr_at_5 |
| value: 44.803 |
| - type: ndcg_at_1 |
| value: 40.699999999999996 |
| - type: ndcg_at_10 |
| value: 47.477999999999994 |
| - type: ndcg_at_100 |
| value: 50.51 |
| - type: ndcg_at_1000 |
| value: 52.367 |
| - type: ndcg_at_3 |
| value: 45.609 |
| - type: ndcg_at_5 |
| value: 46.186 |
| - type: precision_at_1 |
| value: 40.699999999999996 |
| - type: precision_at_10 |
| value: 5.43 |
| - type: precision_at_100 |
| value: 0.692 |
| - type: precision_at_1000 |
| value: 0.084 |
| - type: precision_at_3 |
| value: 16.333000000000002 |
| - type: precision_at_5 |
| value: 10.08 |
| - type: recall_at_1 |
| value: 40.699999999999996 |
| - type: recall_at_10 |
| value: 54.300000000000004 |
| - type: recall_at_100 |
| value: 69.19999999999999 |
| - type: recall_at_1000 |
| value: 84.3 |
| - type: recall_at_3 |
| value: 49.0 |
| - type: recall_at_5 |
| value: 50.4 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/medrxiv-clustering-p2p |
| name: MTEB MedrxivClusteringP2P |
| config: default |
| split: test |
| revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
| metrics: |
| - type: v_measure |
| value: 31.70883822617504 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/medrxiv-clustering-s2s |
| name: MTEB MedrxivClusteringS2S |
| config: default |
| split: test |
| revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
| metrics: |
| - type: v_measure |
| value: 28.801248513598072 |
| - task: |
| type: Reranking |
| dataset: |
| type: mteb/mind_small |
| name: MTEB MindSmallReranking |
| config: default |
| split: test |
| revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
| metrics: |
| - type: map |
| value: 30.97227673339198 |
| - type: mrr |
| value: 32.03205560232119 |
| - task: |
| type: Reranking |
| dataset: |
| type: C-MTEB/Mmarco-reranking |
| name: MTEB MMarcoReranking |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map |
| value: 25.89977615357687 |
| - type: mrr |
| value: 24.192857142857143 |
| - task: |
| type: Classification |
| dataset: |
| type: C-MTEB/MultilingualSentiment-classification |
| name: MTEB MultilingualSentiment |
| config: default |
| split: validation |
| revision: None |
| metrics: |
| - type: accuracy |
| value: 67.16666666666666 |
| - type: f1 |
| value: 67.15765577091656 |
| - task: |
| type: Retrieval |
| dataset: |
| type: nfcorpus |
| name: MTEB NFCorpus |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 5.079000000000001 |
| - type: map_at_10 |
| value: 12.04 |
| - type: map_at_100 |
| value: 15.375 |
| - type: map_at_1000 |
| value: 16.878 |
| - type: map_at_3 |
| value: 8.851 |
| - type: map_at_5 |
| value: 10.23 |
| - type: mrr_at_1 |
| value: 43.963 |
| - type: mrr_at_10 |
| value: 52.886 |
| - type: mrr_at_100 |
| value: 53.498000000000005 |
| - type: mrr_at_1000 |
| value: 53.54 |
| - type: mrr_at_3 |
| value: 50.876999999999995 |
| - type: mrr_at_5 |
| value: 52.254999999999995 |
| - type: ndcg_at_1 |
| value: 42.415000000000006 |
| - type: ndcg_at_10 |
| value: 33.660000000000004 |
| - type: ndcg_at_100 |
| value: 31.008000000000003 |
| - type: ndcg_at_1000 |
| value: 40.016 |
| - type: ndcg_at_3 |
| value: 39.329 |
| - type: ndcg_at_5 |
| value: 36.687999999999995 |
| - type: precision_at_1 |
| value: 43.963 |
| - type: precision_at_10 |
| value: 25.356 |
| - type: precision_at_100 |
| value: 8.245 |
| - type: precision_at_1000 |
| value: 2.106 |
| - type: precision_at_3 |
| value: 37.255 |
| - type: precision_at_5 |
| value: 31.95 |
| - type: recall_at_1 |
| value: 5.079000000000001 |
| - type: recall_at_10 |
| value: 15.838 |
| - type: recall_at_100 |
| value: 32.159 |
| - type: recall_at_1000 |
| value: 64.91799999999999 |
| - type: recall_at_3 |
| value: 10.152999999999999 |
| - type: recall_at_5 |
| value: 12.4 |
| - task: |
| type: Retrieval |
| dataset: |
| type: nq |
| name: MTEB NQ |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 29.605999999999998 |
| - type: map_at_10 |
| value: 43.518 |
| - type: map_at_100 |
| value: 44.583 |
| - type: map_at_1000 |
| value: 44.622 |
| - type: map_at_3 |
| value: 39.673 |
| - type: map_at_5 |
| value: 41.897 |
| - type: mrr_at_1 |
| value: 33.604 |
| - type: mrr_at_10 |
| value: 46.156000000000006 |
| - type: mrr_at_100 |
| value: 46.974 |
| - type: mrr_at_1000 |
| value: 47.002 |
| - type: mrr_at_3 |
| value: 42.907000000000004 |
| - type: mrr_at_5 |
| value: 44.792 |
| - type: ndcg_at_1 |
| value: 33.575 |
| - type: ndcg_at_10 |
| value: 50.61600000000001 |
| - type: ndcg_at_100 |
| value: 55.129 |
| - type: ndcg_at_1000 |
| value: 56.084 |
| - type: ndcg_at_3 |
| value: 43.297999999999995 |
| - type: ndcg_at_5 |
| value: 46.979 |
| - type: precision_at_1 |
| value: 33.575 |
| - type: precision_at_10 |
| value: 8.297 |
| - type: precision_at_100 |
| value: 1.083 |
| - type: precision_at_1000 |
| value: 0.117 |
| - type: precision_at_3 |
| value: 19.602 |
| - type: precision_at_5 |
| value: 13.934 |
| - type: recall_at_1 |
| value: 29.605999999999998 |
| - type: recall_at_10 |
| value: 69.718 |
| - type: recall_at_100 |
| value: 89.352 |
| - type: recall_at_1000 |
| value: 96.543 |
| - type: recall_at_3 |
| value: 50.617999999999995 |
| - type: recall_at_5 |
| value: 59.031 |
| - task: |
| type: PairClassification |
| dataset: |
| type: C-MTEB/OCNLI |
| name: MTEB Ocnli |
| config: default |
| split: validation |
| revision: None |
| metrics: |
| - type: cos_sim_accuracy |
| value: 65.83649160801299 |
| - type: cos_sim_ap |
| value: 69.86408265006916 |
| - type: cos_sim_f1 |
| value: 70.50709939148074 |
| - type: cos_sim_precision |
| value: 57.2463768115942 |
| - type: cos_sim_recall |
| value: 91.76346356916578 |
| - type: dot_accuracy |
| value: 61.93827828911749 |
| - type: dot_ap |
| value: 64.26140500313572 |
| - type: dot_f1 |
| value: 68.97081413210446 |
| - type: dot_precision |
| value: 54.19432709716355 |
| - type: dot_recall |
| value: 94.82576557550159 |
| - type: euclidean_accuracy |
| value: 66.32376827287493 |
| - type: euclidean_ap |
| value: 70.58216586017075 |
| - type: euclidean_f1 |
| value: 71.31782945736435 |
| - type: euclidean_precision |
| value: 58.11170212765957 |
| - type: euclidean_recall |
| value: 92.29144667370645 |
| - type: manhattan_accuracy |
| value: 66.54033567948024 |
| - type: manhattan_ap |
| value: 70.88996923294056 |
| - type: manhattan_f1 |
| value: 71.45256087321579 |
| - type: manhattan_precision |
| value: 59.30313588850174 |
| - type: manhattan_recall |
| value: 89.86272439281943 |
| - type: max_accuracy |
| value: 66.54033567948024 |
| - type: max_ap |
| value: 70.88996923294056 |
| - type: max_f1 |
| value: 71.45256087321579 |
| - task: |
| type: Classification |
| dataset: |
| type: C-MTEB/OnlineShopping-classification |
| name: MTEB OnlineShopping |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: accuracy |
| value: 90.41 |
| - type: ap |
| value: 88.15736492425235 |
| - type: f1 |
| value: 90.40118324200982 |
| - task: |
| type: STS |
| dataset: |
| type: C-MTEB/PAWSX |
| name: MTEB PAWSX |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: cos_sim_pearson |
| value: 14.718326697461064 |
| - type: cos_sim_spearman |
| value: 17.458017383716168 |
| - type: euclidean_pearson |
| value: 19.416710995216608 |
| - type: euclidean_spearman |
| value: 17.87886266073602 |
| - type: manhattan_pearson |
| value: 19.508696307778063 |
| - type: manhattan_spearman |
| value: 18.026398724663487 |
| - task: |
| type: STS |
| dataset: |
| type: C-MTEB/QBQTC |
| name: MTEB QBQTC |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: cos_sim_pearson |
| value: 31.330102731068386 |
| - type: cos_sim_spearman |
| value: 33.69612492132476 |
| - type: euclidean_pearson |
| value: 33.83912666711584 |
| - type: euclidean_spearman |
| value: 35.58666712573462 |
| - type: manhattan_pearson |
| value: 34.257595977157706 |
| - type: manhattan_spearman |
| value: 36.08587604692898 |
| - task: |
| type: Retrieval |
| dataset: |
| type: quora |
| name: MTEB QuoraRetrieval |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 70.37 |
| - type: map_at_10 |
| value: 84.22699999999999 |
| - type: map_at_100 |
| value: 84.871 |
| - type: map_at_1000 |
| value: 84.88900000000001 |
| - type: map_at_3 |
| value: 81.277 |
| - type: map_at_5 |
| value: 83.16799999999999 |
| - type: mrr_at_1 |
| value: 80.97 |
| - type: mrr_at_10 |
| value: 87.24300000000001 |
| - type: mrr_at_100 |
| value: 87.346 |
| - type: mrr_at_1000 |
| value: 87.347 |
| - type: mrr_at_3 |
| value: 86.258 |
| - type: mrr_at_5 |
| value: 86.914 |
| - type: ndcg_at_1 |
| value: 81.0 |
| - type: ndcg_at_10 |
| value: 88.009 |
| - type: ndcg_at_100 |
| value: 89.251 |
| - type: ndcg_at_1000 |
| value: 89.374 |
| - type: ndcg_at_3 |
| value: 85.169 |
| - type: ndcg_at_5 |
| value: 86.75399999999999 |
| - type: precision_at_1 |
| value: 81.0 |
| - type: precision_at_10 |
| value: 13.343 |
| - type: precision_at_100 |
| value: 1.526 |
| - type: precision_at_1000 |
| value: 0.157 |
| - type: precision_at_3 |
| value: 37.25 |
| - type: precision_at_5 |
| value: 24.504 |
| - type: recall_at_1 |
| value: 70.37 |
| - type: recall_at_10 |
| value: 95.158 |
| - type: recall_at_100 |
| value: 99.39 |
| - type: recall_at_1000 |
| value: 99.98 |
| - type: recall_at_3 |
| value: 86.942 |
| - type: recall_at_5 |
| value: 91.446 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/reddit-clustering |
| name: MTEB RedditClustering |
| config: default |
| split: test |
| revision: 24640382cdbf8abc73003fb0fa6d111a705499eb |
| metrics: |
| - type: v_measure |
| value: 49.71370818375339 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/reddit-clustering-p2p |
| name: MTEB RedditClusteringP2P |
| config: default |
| split: test |
| revision: 282350215ef01743dc01b456c7f5241fa8937f16 |
| metrics: |
| - type: v_measure |
| value: 55.07451965473589 |
| - task: |
| type: Retrieval |
| dataset: |
| type: scidocs |
| name: MTEB SCIDOCS |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 4.508 |
| - type: map_at_10 |
| value: 10.825 |
| - type: map_at_100 |
| value: 12.598 |
| - type: map_at_1000 |
| value: 12.854 |
| - type: map_at_3 |
| value: 7.892 |
| - type: map_at_5 |
| value: 9.349 |
| - type: mrr_at_1 |
| value: 22.2 |
| - type: mrr_at_10 |
| value: 32.611000000000004 |
| - type: mrr_at_100 |
| value: 33.61 |
| - type: mrr_at_1000 |
| value: 33.671 |
| - type: mrr_at_3 |
| value: 29.15 |
| - type: mrr_at_5 |
| value: 31.225 |
| - type: ndcg_at_1 |
| value: 22.2 |
| - type: ndcg_at_10 |
| value: 18.502 |
| - type: ndcg_at_100 |
| value: 25.424999999999997 |
| - type: ndcg_at_1000 |
| value: 30.233999999999998 |
| - type: ndcg_at_3 |
| value: 17.711 |
| - type: ndcg_at_5 |
| value: 15.501000000000001 |
| - type: precision_at_1 |
| value: 22.2 |
| - type: precision_at_10 |
| value: 9.49 |
| - type: precision_at_100 |
| value: 1.941 |
| - type: precision_at_1000 |
| value: 0.31 |
| - type: precision_at_3 |
| value: 16.433 |
| - type: precision_at_5 |
| value: 13.54 |
| - type: recall_at_1 |
| value: 4.508 |
| - type: recall_at_10 |
| value: 19.243 |
| - type: recall_at_100 |
| value: 39.407 |
| - type: recall_at_1000 |
| value: 62.953 |
| - type: recall_at_3 |
| value: 9.993 |
| - type: recall_at_5 |
| value: 13.733 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sickr-sts |
| name: MTEB SICK-R |
| config: default |
| split: test |
| revision: a6ea5a8cab320b040a23452cc28066d9beae2cee |
| metrics: |
| - type: cos_sim_pearson |
| value: 85.88096352325879 |
| - type: cos_sim_spearman |
| value: 80.84882728439892 |
| - type: euclidean_pearson |
| value: 82.89512161923362 |
| - type: euclidean_spearman |
| value: 80.69723454935396 |
| - type: manhattan_pearson |
| value: 82.94365287299226 |
| - type: manhattan_spearman |
| value: 80.64700541831023 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts12-sts |
| name: MTEB STS12 |
| config: default |
| split: test |
| revision: a0d554a64d88156834ff5ae9920b964011b16384 |
| metrics: |
| - type: cos_sim_pearson |
| value: 84.09030569824817 |
| - type: cos_sim_spearman |
| value: 76.10288448289813 |
| - type: euclidean_pearson |
| value: 82.19317617787483 |
| - type: euclidean_spearman |
| value: 78.51206398528993 |
| - type: manhattan_pearson |
| value: 82.50688072451729 |
| - type: manhattan_spearman |
| value: 78.71694597298867 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts13-sts |
| name: MTEB STS13 |
| config: default |
| split: test |
| revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
| metrics: |
| - type: cos_sim_pearson |
| value: 85.04298066236511 |
| - type: cos_sim_spearman |
| value: 85.49051395372348 |
| - type: euclidean_pearson |
| value: 85.7369561800059 |
| - type: euclidean_spearman |
| value: 86.35626949911497 |
| - type: manhattan_pearson |
| value: 85.86766305481635 |
| - type: manhattan_spearman |
| value: 86.5115276036124 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts14-sts |
| name: MTEB STS14 |
| config: default |
| split: test |
| revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
| metrics: |
| - type: cos_sim_pearson |
| value: 83.98107748125086 |
| - type: cos_sim_spearman |
| value: 80.43502071880916 |
| - type: euclidean_pearson |
| value: 82.24603130661005 |
| - type: euclidean_spearman |
| value: 80.94302742946145 |
| - type: manhattan_pearson |
| value: 82.4215619893203 |
| - type: manhattan_spearman |
| value: 81.13824893869541 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts15-sts |
| name: MTEB STS15 |
| config: default |
| split: test |
| revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
| metrics: |
| - type: cos_sim_pearson |
| value: 86.95857345426359 |
| - type: cos_sim_spearman |
| value: 87.7540379885978 |
| - type: euclidean_pearson |
| value: 87.86433964223119 |
| - type: euclidean_spearman |
| value: 88.43585275816753 |
| - type: manhattan_pearson |
| value: 87.90915813062988 |
| - type: manhattan_spearman |
| value: 88.49038031429657 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts16-sts |
| name: MTEB STS16 |
| config: default |
| split: test |
| revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
| metrics: |
| - type: cos_sim_pearson |
| value: 83.84530028548023 |
| - type: cos_sim_spearman |
| value: 85.42197371225963 |
| - type: euclidean_pearson |
| value: 84.12042159341938 |
| - type: euclidean_spearman |
| value: 84.69864997658445 |
| - type: manhattan_pearson |
| value: 84.09772815909784 |
| - type: manhattan_spearman |
| value: 84.63986468736967 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts17-crosslingual-sts |
| name: MTEB STS17 (en-en) |
| config: en-en |
| split: test |
| revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
| metrics: |
| - type: cos_sim_pearson |
| value: 89.89281017946413 |
| - type: cos_sim_spearman |
| value: 89.94783195991867 |
| - type: euclidean_pearson |
| value: 89.19342633226815 |
| - type: euclidean_spearman |
| value: 88.6692137120815 |
| - type: manhattan_pearson |
| value: 89.19006596701496 |
| - type: manhattan_spearman |
| value: 88.65041672073397 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts22-crosslingual-sts |
| name: MTEB STS22 (en) |
| config: en |
| split: test |
| revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
| metrics: |
| - type: cos_sim_pearson |
| value: 65.05176237336566 |
| - type: cos_sim_spearman |
| value: 65.12758602746149 |
| - type: euclidean_pearson |
| value: 67.44468889455905 |
| - type: euclidean_spearman |
| value: 67.42836832904808 |
| - type: manhattan_pearson |
| value: 67.99438187200471 |
| - type: manhattan_spearman |
| value: 67.96190936270705 |
| - task: |
| type: STS |
| dataset: |
| type: C-MTEB/STSB |
| name: MTEB STSB |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: cos_sim_pearson |
| value: 81.36171514729287 |
| - type: cos_sim_spearman |
| value: 81.51752389848613 |
| - type: euclidean_pearson |
| value: 81.14136234145765 |
| - type: euclidean_spearman |
| value: 81.27609983297867 |
| - type: manhattan_pearson |
| value: 81.44966268348165 |
| - type: manhattan_spearman |
| value: 81.53484018091312 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/stsbenchmark-sts |
| name: MTEB STSBenchmark |
| config: default |
| split: test |
| revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
| metrics: |
| - type: cos_sim_pearson |
| value: 86.92195724268996 |
| - type: cos_sim_spearman |
| value: 87.70682082313391 |
| - type: euclidean_pearson |
| value: 86.24220109166684 |
| - type: euclidean_spearman |
| value: 86.51998671092596 |
| - type: manhattan_pearson |
| value: 86.17577571663554 |
| - type: manhattan_spearman |
| value: 86.45961101071687 |
| - task: |
| type: Reranking |
| dataset: |
| type: mteb/scidocs-reranking |
| name: MTEB SciDocsRR |
| config: default |
| split: test |
| revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
| metrics: |
| - type: map |
| value: 78.62106635785725 |
| - type: mrr |
| value: 93.84658279266121 |
| - task: |
| type: Retrieval |
| dataset: |
| type: scifact |
| name: MTEB SciFact |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 53.761 |
| - type: map_at_10 |
| value: 64.56 |
| - type: map_at_100 |
| value: 65.243 |
| - type: map_at_1000 |
| value: 65.269 |
| - type: map_at_3 |
| value: 62.156 |
| - type: map_at_5 |
| value: 63.55 |
| - type: mrr_at_1 |
| value: 56.667 |
| - type: mrr_at_10 |
| value: 66.084 |
| - type: mrr_at_100 |
| value: 66.58500000000001 |
| - type: mrr_at_1000 |
| value: 66.61 |
| - type: mrr_at_3 |
| value: 64.333 |
| - type: mrr_at_5 |
| value: 65.3 |
| - type: ndcg_at_1 |
| value: 56.667 |
| - type: ndcg_at_10 |
| value: 69.43 |
| - type: ndcg_at_100 |
| value: 72.031 |
| - type: ndcg_at_1000 |
| value: 72.75 |
| - type: ndcg_at_3 |
| value: 65.282 |
| - type: ndcg_at_5 |
| value: 67.24900000000001 |
| - type: precision_at_1 |
| value: 56.667 |
| - type: precision_at_10 |
| value: 9.3 |
| - type: precision_at_100 |
| value: 1.0670000000000002 |
| - type: precision_at_1000 |
| value: 0.11199999999999999 |
| - type: precision_at_3 |
| value: 25.778000000000002 |
| - type: precision_at_5 |
| value: 16.866999999999997 |
| - type: recall_at_1 |
| value: 53.761 |
| - type: recall_at_10 |
| value: 82.678 |
| - type: recall_at_100 |
| value: 93.667 |
| - type: recall_at_1000 |
| value: 99.333 |
| - type: recall_at_3 |
| value: 71.578 |
| - type: recall_at_5 |
| value: 76.25 |
| - task: |
| type: PairClassification |
| dataset: |
| type: mteb/sprintduplicatequestions-pairclassification |
| name: MTEB SprintDuplicateQuestions |
| config: default |
| split: test |
| revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
| metrics: |
| - type: cos_sim_accuracy |
| value: 99.80594059405941 |
| - type: cos_sim_ap |
| value: 95.35711574476811 |
| - type: cos_sim_f1 |
| value: 90.12096774193547 |
| - type: cos_sim_precision |
| value: 90.85365853658537 |
| - type: cos_sim_recall |
| value: 89.4 |
| - type: dot_accuracy |
| value: 99.76732673267327 |
| - type: dot_ap |
| value: 93.20624501431367 |
| - type: dot_f1 |
| value: 87.74126238914971 |
| - type: dot_precision |
| value: 91.71210468920393 |
| - type: dot_recall |
| value: 84.1 |
| - type: euclidean_accuracy |
| value: 99.80594059405941 |
| - type: euclidean_ap |
| value: 95.35758863966429 |
| - type: euclidean_f1 |
| value: 90.15075376884421 |
| - type: euclidean_precision |
| value: 90.6060606060606 |
| - type: euclidean_recall |
| value: 89.7 |
| - type: manhattan_accuracy |
| value: 99.80990099009901 |
| - type: manhattan_ap |
| value: 95.48335466728275 |
| - type: manhattan_f1 |
| value: 90.2672718103883 |
| - type: manhattan_precision |
| value: 91.04781281790437 |
| - type: manhattan_recall |
| value: 89.5 |
| - type: max_accuracy |
| value: 99.80990099009901 |
| - type: max_ap |
| value: 95.48335466728275 |
| - type: max_f1 |
| value: 90.2672718103883 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/stackexchange-clustering |
| name: MTEB StackExchangeClustering |
| config: default |
| split: test |
| revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
| metrics: |
| - type: v_measure |
| value: 59.422562431402845 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/stackexchange-clustering-p2p |
| name: MTEB StackExchangeClusteringP2P |
| config: default |
| split: test |
| revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
| metrics: |
| - type: v_measure |
| value: 31.695493629721373 |
| - task: |
| type: Reranking |
| dataset: |
| type: mteb/stackoverflowdupquestions-reranking |
| name: MTEB StackOverflowDupQuestions |
| config: default |
| split: test |
| revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
| metrics: |
| - type: map |
| value: 50.070077950465965 |
| - type: mrr |
| value: 50.72293311263899 |
| - task: |
| type: Summarization |
| dataset: |
| type: mteb/summeval |
| name: MTEB SummEval |
| config: default |
| split: test |
| revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
| metrics: |
| - type: cos_sim_pearson |
| value: 30.59608436984981 |
| - type: cos_sim_spearman |
| value: 30.617289383193103 |
| - type: dot_pearson |
| value: 30.78715584903813 |
| - type: dot_spearman |
| value: 31.269245492805283 |
| - task: |
| type: Reranking |
| dataset: |
| type: C-MTEB/T2Reranking |
| name: MTEB T2Reranking |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map |
| value: 66.49332760690612 |
| - type: mrr |
| value: 76.52668294806075 |
| - task: |
| type: Retrieval |
| dataset: |
| type: C-MTEB/T2Retrieval |
| name: MTEB T2Retrieval |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 24.607 |
| - type: map_at_10 |
| value: 67.009 |
| - type: map_at_100 |
| value: 70.838 |
| - type: map_at_1000 |
| value: 70.954 |
| - type: map_at_3 |
| value: 47.573 |
| - type: map_at_5 |
| value: 58.10999999999999 |
| - type: mrr_at_1 |
| value: 84.333 |
| - type: mrr_at_10 |
| value: 87.822 |
| - type: mrr_at_100 |
| value: 87.969 |
| - type: mrr_at_1000 |
| value: 87.97500000000001 |
| - type: mrr_at_3 |
| value: 87.16000000000001 |
| - type: mrr_at_5 |
| value: 87.587 |
| - type: ndcg_at_1 |
| value: 84.333 |
| - type: ndcg_at_10 |
| value: 76.303 |
| - type: ndcg_at_100 |
| value: 81.05499999999999 |
| - type: ndcg_at_1000 |
| value: 82.218 |
| - type: ndcg_at_3 |
| value: 78.691 |
| - type: ndcg_at_5 |
| value: 76.66 |
| - type: precision_at_1 |
| value: 84.333 |
| - type: precision_at_10 |
| value: 38.019999999999996 |
| - type: precision_at_100 |
| value: 4.7669999999999995 |
| - type: precision_at_1000 |
| value: 0.505 |
| - type: precision_at_3 |
| value: 68.939 |
| - type: precision_at_5 |
| value: 57.306999999999995 |
| - type: recall_at_1 |
| value: 24.607 |
| - type: recall_at_10 |
| value: 74.971 |
| - type: recall_at_100 |
| value: 90.108 |
| - type: recall_at_1000 |
| value: 95.917 |
| - type: recall_at_3 |
| value: 49.586000000000006 |
| - type: recall_at_5 |
| value: 62.232 |
| - task: |
| type: Classification |
| dataset: |
| type: C-MTEB/TNews-classification |
| name: MTEB TNews |
| config: default |
| split: validation |
| revision: None |
| metrics: |
| - type: accuracy |
| value: 47.702 |
| - type: f1 |
| value: 46.274469606672426 |
| - task: |
| type: Retrieval |
| dataset: |
| type: trec-covid |
| name: MTEB TRECCOVID |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 0.252 |
| - type: map_at_10 |
| value: 2.178 |
| - type: map_at_100 |
| value: 12.781999999999998 |
| - type: map_at_1000 |
| value: 29.494999999999997 |
| - type: map_at_3 |
| value: 0.73 |
| - type: map_at_5 |
| value: 1.169 |
| - type: mrr_at_1 |
| value: 94.0 |
| - type: mrr_at_10 |
| value: 97.0 |
| - type: mrr_at_100 |
| value: 97.0 |
| - type: mrr_at_1000 |
| value: 97.0 |
| - type: mrr_at_3 |
| value: 97.0 |
| - type: mrr_at_5 |
| value: 97.0 |
| - type: ndcg_at_1 |
| value: 88.0 |
| - type: ndcg_at_10 |
| value: 83.21 |
| - type: ndcg_at_100 |
| value: 63.31 |
| - type: ndcg_at_1000 |
| value: 54.734 |
| - type: ndcg_at_3 |
| value: 87.408 |
| - type: ndcg_at_5 |
| value: 86.20100000000001 |
| - type: precision_at_1 |
| value: 94.0 |
| - type: precision_at_10 |
| value: 88.2 |
| - type: precision_at_100 |
| value: 64.68 |
| - type: precision_at_1000 |
| value: 23.966 |
| - type: precision_at_3 |
| value: 93.333 |
| - type: precision_at_5 |
| value: 91.60000000000001 |
| - type: recall_at_1 |
| value: 0.252 |
| - type: recall_at_10 |
| value: 2.307 |
| - type: recall_at_100 |
| value: 15.703 |
| - type: recall_at_1000 |
| value: 51.111 |
| - type: recall_at_3 |
| value: 0.749 |
| - type: recall_at_5 |
| value: 1.212 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (sqi-eng) |
| config: sqi-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 16.8 |
| - type: f1 |
| value: 13.168299935527422 |
| - type: precision |
| value: 12.209559281760876 |
| - type: recall |
| value: 16.8 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (fry-eng) |
| config: fry-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 35.83815028901734 |
| - type: f1 |
| value: 29.0852500101055 |
| - type: precision |
| value: 26.965317919075147 |
| - type: recall |
| value: 35.83815028901734 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (kur-eng) |
| config: kur-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 15.121951219512194 |
| - type: f1 |
| value: 11.844149203614325 |
| - type: precision |
| value: 11.042929292929294 |
| - type: recall |
| value: 15.121951219512194 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (tur-eng) |
| config: tur-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 9.9 |
| - type: f1 |
| value: 7.1396348187007215 |
| - type: precision |
| value: 6.501835713997978 |
| - type: recall |
| value: 9.9 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (deu-eng) |
| config: deu-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 76.6 |
| - type: f1 |
| value: 72.73241758241758 |
| - type: precision |
| value: 71.18867647058823 |
| - type: recall |
| value: 76.6 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (nld-eng) |
| config: nld-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 42.0 |
| - type: f1 |
| value: 36.81003102453103 |
| - type: precision |
| value: 35.19870269535562 |
| - type: recall |
| value: 42.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ron-eng) |
| config: ron-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 35.3 |
| - type: f1 |
| value: 30.353777056277053 |
| - type: precision |
| value: 28.773956778515604 |
| - type: recall |
| value: 35.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ang-eng) |
| config: ang-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 35.82089552238806 |
| - type: f1 |
| value: 27.44136460554371 |
| - type: precision |
| value: 24.340796019900495 |
| - type: recall |
| value: 35.82089552238806 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ido-eng) |
| config: ido-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 51.800000000000004 |
| - type: f1 |
| value: 45.82491836793846 |
| - type: precision |
| value: 43.729303094622864 |
| - type: recall |
| value: 51.800000000000004 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (jav-eng) |
| config: jav-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 25.853658536585368 |
| - type: f1 |
| value: 19.79869362796192 |
| - type: precision |
| value: 18.250680214094846 |
| - type: recall |
| value: 25.853658536585368 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (isl-eng) |
| config: isl-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 9.0 |
| - type: f1 |
| value: 6.926590762281661 |
| - type: precision |
| value: 6.507185696775364 |
| - type: recall |
| value: 9.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (slv-eng) |
| config: slv-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 14.33778857837181 |
| - type: f1 |
| value: 10.888963524130242 |
| - type: precision |
| value: 10.189272116928368 |
| - type: recall |
| value: 14.33778857837181 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (cym-eng) |
| config: cym-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 11.304347826086957 |
| - type: f1 |
| value: 8.459121175343064 |
| - type: precision |
| value: 7.7218644669759975 |
| - type: recall |
| value: 11.304347826086957 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (kaz-eng) |
| config: kaz-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 8.521739130434783 |
| - type: f1 |
| value: 6.751744703151353 |
| - type: precision |
| value: 6.387004921960017 |
| - type: recall |
| value: 8.521739130434783 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (est-eng) |
| config: est-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 7.3 |
| - type: f1 |
| value: 5.626766011766011 |
| - type: precision |
| value: 5.1270385799923 |
| - type: recall |
| value: 7.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (heb-eng) |
| config: heb-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 3.2 |
| - type: f1 |
| value: 1.91950282507703 |
| - type: precision |
| value: 1.6684431360304504 |
| - type: recall |
| value: 3.2 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (gla-eng) |
| config: gla-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 5.790108564535585 |
| - type: f1 |
| value: 4.128499324411468 |
| - type: precision |
| value: 3.8151453928788914 |
| - type: recall |
| value: 5.790108564535585 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (mar-eng) |
| config: mar-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 70.3 |
| - type: f1 |
| value: 65.18318181818181 |
| - type: precision |
| value: 63.126911976911984 |
| - type: recall |
| value: 70.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (lat-eng) |
| config: lat-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 45.300000000000004 |
| - type: f1 |
| value: 38.339152873270514 |
| - type: precision |
| value: 36.130903304212126 |
| - type: recall |
| value: 45.300000000000004 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (bel-eng) |
| config: bel-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 16.0 |
| - type: f1 |
| value: 12.172850459161385 |
| - type: precision |
| value: 11.27855570316309 |
| - type: recall |
| value: 16.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (pms-eng) |
| config: pms-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 37.714285714285715 |
| - type: f1 |
| value: 32.188793178089945 |
| - type: precision |
| value: 30.457500778089013 |
| - type: recall |
| value: 37.714285714285715 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (gle-eng) |
| config: gle-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 6.5 |
| - type: f1 |
| value: 4.528544131928126 |
| - type: precision |
| value: 4.171387799947767 |
| - type: recall |
| value: 6.5 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (pes-eng) |
| config: pes-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 21.0 |
| - type: f1 |
| value: 17.006564035803166 |
| - type: precision |
| value: 15.844832112332114 |
| - type: recall |
| value: 21.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (nob-eng) |
| config: nob-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 25.5 |
| - type: f1 |
| value: 22.79430820164996 |
| - type: precision |
| value: 21.938476924594045 |
| - type: recall |
| value: 25.5 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (bul-eng) |
| config: bul-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 33.7 |
| - type: f1 |
| value: 26.898922166422164 |
| - type: precision |
| value: 24.939117884031678 |
| - type: recall |
| value: 33.7 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (cbk-eng) |
| config: cbk-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 69.0 |
| - type: f1 |
| value: 63.68992285492286 |
| - type: precision |
| value: 61.72837301587302 |
| - type: recall |
| value: 69.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (hun-eng) |
| config: hun-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 7.3999999999999995 |
| - type: f1 |
| value: 5.5655686223658565 |
| - type: precision |
| value: 5.119921502146487 |
| - type: recall |
| value: 7.3999999999999995 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (uig-eng) |
| config: uig-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 1.5 |
| - type: f1 |
| value: 1.001208686507139 |
| - type: precision |
| value: 0.9683730903243098 |
| - type: recall |
| value: 1.5 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (rus-eng) |
| config: rus-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 69.0 |
| - type: f1 |
| value: 62.61056277056276 |
| - type: precision |
| value: 59.96357142857143 |
| - type: recall |
| value: 69.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (spa-eng) |
| config: spa-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 98.3 |
| - type: f1 |
| value: 97.76666666666668 |
| - type: precision |
| value: 97.51666666666668 |
| - type: recall |
| value: 98.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (hye-eng) |
| config: hye-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 2.0215633423180592 |
| - type: f1 |
| value: 1.5634923413129036 |
| - type: precision |
| value: 1.4895885785373653 |
| - type: recall |
| value: 2.0215633423180592 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (tel-eng) |
| config: tel-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 83.33333333333334 |
| - type: f1 |
| value: 79.3019943019943 |
| - type: precision |
| value: 77.45726495726495 |
| - type: recall |
| value: 83.33333333333334 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (afr-eng) |
| config: afr-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 23.400000000000002 |
| - type: f1 |
| value: 18.655079988631996 |
| - type: precision |
| value: 17.338269096494905 |
| - type: recall |
| value: 23.400000000000002 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (mon-eng) |
| config: mon-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 6.363636363636363 |
| - type: f1 |
| value: 4.48376251469035 |
| - type: precision |
| value: 4.071778641679957 |
| - type: recall |
| value: 6.363636363636363 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (arz-eng) |
| config: arz-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 77.56813417190776 |
| - type: f1 |
| value: 73.16561844863732 |
| - type: precision |
| value: 71.3440484509667 |
| - type: recall |
| value: 77.56813417190776 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (hrv-eng) |
| config: hrv-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 17.299999999999997 |
| - type: f1 |
| value: 13.693204564375854 |
| - type: precision |
| value: 12.830651358081276 |
| - type: recall |
| value: 17.299999999999997 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (nov-eng) |
| config: nov-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 59.92217898832685 |
| - type: f1 |
| value: 53.29591938541354 |
| - type: precision |
| value: 50.58736335000926 |
| - type: recall |
| value: 59.92217898832685 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (gsw-eng) |
| config: gsw-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 25.64102564102564 |
| - type: f1 |
| value: 19.31404777558624 |
| - type: precision |
| value: 17.413105413105416 |
| - type: recall |
| value: 25.64102564102564 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (nds-eng) |
| config: nds-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 29.7 |
| - type: f1 |
| value: 24.44977050316952 |
| - type: precision |
| value: 22.798075396825396 |
| - type: recall |
| value: 29.7 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ukr-eng) |
| config: ukr-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 32.2 |
| - type: f1 |
| value: 25.423187804627435 |
| - type: precision |
| value: 23.404003309492442 |
| - type: recall |
| value: 32.2 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (uzb-eng) |
| config: uzb-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 9.11214953271028 |
| - type: f1 |
| value: 5.910063827286792 |
| - type: precision |
| value: 5.296401380795872 |
| - type: recall |
| value: 9.11214953271028 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (lit-eng) |
| config: lit-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 7.199999999999999 |
| - type: f1 |
| value: 5.816726797396153 |
| - type: precision |
| value: 5.508698718788661 |
| - type: recall |
| value: 7.199999999999999 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ina-eng) |
| config: ina-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 87.2 |
| - type: f1 |
| value: 83.88333333333333 |
| - type: precision |
| value: 82.42833333333333 |
| - type: recall |
| value: 87.2 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (lfn-eng) |
| config: lfn-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 53.7 |
| - type: f1 |
| value: 48.25312435500516 |
| - type: precision |
| value: 46.34107401656314 |
| - type: recall |
| value: 53.7 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (zsm-eng) |
| config: zsm-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 88.1 |
| - type: f1 |
| value: 85.21690476190476 |
| - type: precision |
| value: 83.96761904761905 |
| - type: recall |
| value: 88.1 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ita-eng) |
| config: ita-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 78.10000000000001 |
| - type: f1 |
| value: 73.38746031746032 |
| - type: precision |
| value: 71.47583333333334 |
| - type: recall |
| value: 78.10000000000001 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (cmn-eng) |
| config: cmn-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 96.1 |
| - type: f1 |
| value: 95.08333333333333 |
| - type: precision |
| value: 94.58333333333334 |
| - type: recall |
| value: 96.1 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (lvs-eng) |
| config: lvs-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 9.0 |
| - type: f1 |
| value: 6.952605595133894 |
| - type: precision |
| value: 6.457724621713984 |
| - type: recall |
| value: 9.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (glg-eng) |
| config: glg-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 84.7 |
| - type: f1 |
| value: 80.97880952380953 |
| - type: precision |
| value: 79.36428571428571 |
| - type: recall |
| value: 84.7 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ceb-eng) |
| config: ceb-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 10.5 |
| - type: f1 |
| value: 8.146458694813958 |
| - type: precision |
| value: 7.618942433110826 |
| - type: recall |
| value: 10.5 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (bre-eng) |
| config: bre-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 8.4 |
| - type: f1 |
| value: 6.144921607886653 |
| - type: precision |
| value: 5.5261043562899586 |
| - type: recall |
| value: 8.4 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ben-eng) |
| config: ben-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 84.39999999999999 |
| - type: f1 |
| value: 80.65333333333334 |
| - type: precision |
| value: 78.97833333333332 |
| - type: recall |
| value: 84.39999999999999 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (swg-eng) |
| config: swg-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 28.57142857142857 |
| - type: f1 |
| value: 22.767379679144387 |
| - type: precision |
| value: 21.2016369047619 |
| - type: recall |
| value: 28.57142857142857 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (arq-eng) |
| config: arq-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 34.24807903402854 |
| - type: f1 |
| value: 29.241572730305222 |
| - type: precision |
| value: 27.6428310072657 |
| - type: recall |
| value: 34.24807903402854 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (kab-eng) |
| config: kab-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 2.9000000000000004 |
| - type: f1 |
| value: 1.9156734696693711 |
| - type: precision |
| value: 1.7528460881307182 |
| - type: recall |
| value: 2.9000000000000004 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (fra-eng) |
| config: fra-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 94.89999999999999 |
| - type: f1 |
| value: 93.53333333333332 |
| - type: precision |
| value: 92.90666666666667 |
| - type: recall |
| value: 94.89999999999999 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (por-eng) |
| config: por-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 95.0 |
| - type: f1 |
| value: 93.61666666666666 |
| - type: precision |
| value: 92.93333333333332 |
| - type: recall |
| value: 95.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (tat-eng) |
| config: tat-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 6.3 |
| - type: f1 |
| value: 4.920070356472795 |
| - type: precision |
| value: 4.565811270125224 |
| - type: recall |
| value: 6.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (oci-eng) |
| config: oci-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 47.4 |
| - type: f1 |
| value: 41.08392857142857 |
| - type: precision |
| value: 38.999704968944094 |
| - type: recall |
| value: 47.4 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (pol-eng) |
| config: pol-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 18.2 |
| - type: f1 |
| value: 14.826165036734295 |
| - type: precision |
| value: 13.988559330454489 |
| - type: recall |
| value: 18.2 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (war-eng) |
| config: war-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 13.3 |
| - type: f1 |
| value: 10.73451225789461 |
| - type: precision |
| value: 10.06524508030025 |
| - type: recall |
| value: 13.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (aze-eng) |
| config: aze-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 9.3 |
| - type: f1 |
| value: 7.613044370901514 |
| - type: precision |
| value: 7.184100384035204 |
| - type: recall |
| value: 9.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (vie-eng) |
| config: vie-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 97.0 |
| - type: f1 |
| value: 96.05 |
| - type: precision |
| value: 95.58333333333334 |
| - type: recall |
| value: 97.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (nno-eng) |
| config: nno-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 19.8 |
| - type: f1 |
| value: 16.070523504273503 |
| - type: precision |
| value: 14.848185626325227 |
| - type: recall |
| value: 19.8 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (cha-eng) |
| config: cha-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 29.1970802919708 |
| - type: f1 |
| value: 22.579707397225647 |
| - type: precision |
| value: 20.792945550165477 |
| - type: recall |
| value: 29.1970802919708 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (mhr-eng) |
| config: mhr-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 4.3 |
| - type: f1 |
| value: 2.884495496452018 |
| - type: precision |
| value: 2.6280916815877506 |
| - type: recall |
| value: 4.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (dan-eng) |
| config: dan-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 28.7 |
| - type: f1 |
| value: 24.9056519214062 |
| - type: precision |
| value: 23.800155414494334 |
| - type: recall |
| value: 28.7 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ell-eng) |
| config: ell-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 9.5 |
| - type: f1 |
| value: 6.723431537130878 |
| - type: precision |
| value: 6.078266616597544 |
| - type: recall |
| value: 9.5 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (amh-eng) |
| config: amh-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 1.7857142857142856 |
| - type: f1 |
| value: 0.4579590594653929 |
| - type: precision |
| value: 0.32939943654229364 |
| - type: recall |
| value: 1.7857142857142856 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (pam-eng) |
| config: pam-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 9.1 |
| - type: f1 |
| value: 7.1794182614770845 |
| - type: precision |
| value: 6.81138018671376 |
| - type: recall |
| value: 9.1 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (hsb-eng) |
| config: hsb-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 15.113871635610765 |
| - type: f1 |
| value: 12.353104530336957 |
| - type: precision |
| value: 11.66106754766342 |
| - type: recall |
| value: 15.113871635610765 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (srp-eng) |
| config: srp-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 18.4 |
| - type: f1 |
| value: 15.091645001025805 |
| - type: precision |
| value: 14.200823959052217 |
| - type: recall |
| value: 18.4 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (epo-eng) |
| config: epo-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 33.2 |
| - type: f1 |
| value: 28.066634199134192 |
| - type: precision |
| value: 26.54372717117398 |
| - type: recall |
| value: 33.2 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (kzj-eng) |
| config: kzj-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 7.6 |
| - type: f1 |
| value: 5.992580343865051 |
| - type: precision |
| value: 5.7409125738839055 |
| - type: recall |
| value: 7.6 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (awa-eng) |
| config: awa-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 52.81385281385281 |
| - type: f1 |
| value: 46.86834810211434 |
| - type: precision |
| value: 45.13687899402185 |
| - type: recall |
| value: 52.81385281385281 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (fao-eng) |
| config: fao-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 16.030534351145036 |
| - type: f1 |
| value: 12.902313597194603 |
| - type: precision |
| value: 12.19757977391565 |
| - type: recall |
| value: 16.030534351145036 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (mal-eng) |
| config: mal-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 94.75982532751091 |
| - type: f1 |
| value: 93.11984473556527 |
| - type: precision |
| value: 92.3216885007278 |
| - type: recall |
| value: 94.75982532751091 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ile-eng) |
| config: ile-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 70.19999999999999 |
| - type: f1 |
| value: 64.41237595737596 |
| - type: precision |
| value: 62.074285714285715 |
| - type: recall |
| value: 70.19999999999999 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (bos-eng) |
| config: bos-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 19.2090395480226 |
| - type: f1 |
| value: 14.986259497894084 |
| - type: precision |
| value: 14.08083152750014 |
| - type: recall |
| value: 19.2090395480226 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (cor-eng) |
| config: cor-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 5.800000000000001 |
| - type: f1 |
| value: 4.004811414639001 |
| - type: precision |
| value: 3.611296721493974 |
| - type: recall |
| value: 5.800000000000001 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (cat-eng) |
| config: cat-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 93.10000000000001 |
| - type: f1 |
| value: 91.17333333333335 |
| - type: precision |
| value: 90.27833333333334 |
| - type: recall |
| value: 93.10000000000001 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (eus-eng) |
| config: eus-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 68.2 |
| - type: f1 |
| value: 63.805870279146134 |
| - type: precision |
| value: 62.064924029458915 |
| - type: recall |
| value: 68.2 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (yue-eng) |
| config: yue-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 88.9 |
| - type: f1 |
| value: 86.38250000000001 |
| - type: precision |
| value: 85.345 |
| - type: recall |
| value: 88.9 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (swe-eng) |
| config: swe-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 26.3 |
| - type: f1 |
| value: 21.72601907540825 |
| - type: precision |
| value: 20.3161132602622 |
| - type: recall |
| value: 26.3 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (dtp-eng) |
| config: dtp-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 6.6000000000000005 |
| - type: f1 |
| value: 5.4107919446503585 |
| - type: precision |
| value: 5.143205186348676 |
| - type: recall |
| value: 6.6000000000000005 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (kat-eng) |
| config: kat-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 1.2064343163538873 |
| - type: f1 |
| value: 0.7118331023204635 |
| - type: precision |
| value: 0.6930197065411955 |
| - type: recall |
| value: 1.2064343163538873 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (jpn-eng) |
| config: jpn-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 78.0 |
| - type: f1 |
| value: 73.95134920634919 |
| - type: precision |
| value: 72.3770634920635 |
| - type: recall |
| value: 78.0 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (csb-eng) |
| config: csb-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 12.648221343873518 |
| - type: f1 |
| value: 10.259994816302727 |
| - type: precision |
| value: 9.677206851119895 |
| - type: recall |
| value: 12.648221343873518 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (xho-eng) |
| config: xho-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 10.56338028169014 |
| - type: f1 |
| value: 7.792644757433489 |
| - type: precision |
| value: 7.299087316692951 |
| - type: recall |
| value: 10.56338028169014 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (orv-eng) |
| config: orv-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 8.1437125748503 |
| - type: f1 |
| value: 5.6113303405098724 |
| - type: precision |
| value: 5.156075980223929 |
| - type: recall |
| value: 8.1437125748503 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ind-eng) |
| config: ind-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 92.5 |
| - type: f1 |
| value: 90.53999999999999 |
| - type: precision |
| value: 89.64500000000001 |
| - type: recall |
| value: 92.5 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (tuk-eng) |
| config: tuk-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 8.374384236453201 |
| - type: f1 |
| value: 5.831645092728836 |
| - type: precision |
| value: 5.241568776051535 |
| - type: recall |
| value: 8.374384236453201 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (max-eng) |
| config: max-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 45.42253521126761 |
| - type: f1 |
| value: 40.878561970111264 |
| - type: precision |
| value: 39.52681669728516 |
| - type: recall |
| value: 45.42253521126761 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (swh-eng) |
| config: swh-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 32.05128205128205 |
| - type: f1 |
| value: 25.433010420698523 |
| - type: precision |
| value: 23.545685308843208 |
| - type: recall |
| value: 32.05128205128205 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (hin-eng) |
| config: hin-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 94.6 |
| - type: f1 |
| value: 92.86666666666666 |
| - type: precision |
| value: 92.01666666666667 |
| - type: recall |
| value: 94.6 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (dsb-eng) |
| config: dsb-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 14.822546972860126 |
| - type: f1 |
| value: 12.439321820122155 |
| - type: precision |
| value: 11.940341857811413 |
| - type: recall |
| value: 14.822546972860126 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ber-eng) |
| config: ber-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 6.7 |
| - type: f1 |
| value: 5.534443298607457 |
| - type: precision |
| value: 5.299107273391812 |
| - type: recall |
| value: 6.7 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (tam-eng) |
| config: tam-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 87.94788273615634 |
| - type: f1 |
| value: 84.65798045602605 |
| - type: precision |
| value: 83.2084690553746 |
| - type: recall |
| value: 87.94788273615634 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (slk-eng) |
| config: slk-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 13.8 |
| - type: f1 |
| value: 11.356912127897372 |
| - type: precision |
| value: 10.778191051205624 |
| - type: recall |
| value: 13.8 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (tgl-eng) |
| config: tgl-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 13.700000000000001 |
| - type: f1 |
| value: 10.74774895608627 |
| - type: precision |
| value: 9.966243757837463 |
| - type: recall |
| value: 13.700000000000001 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ast-eng) |
| config: ast-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 76.37795275590551 |
| - type: f1 |
| value: 71.24671916010499 |
| - type: precision |
| value: 69.20697412823397 |
| - type: recall |
| value: 76.37795275590551 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (mkd-eng) |
| config: mkd-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 18.099999999999998 |
| - type: f1 |
| value: 13.934122253809159 |
| - type: precision |
| value: 12.815974391105971 |
| - type: recall |
| value: 18.099999999999998 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (khm-eng) |
| config: khm-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 0.6925207756232686 |
| - type: f1 |
| value: 0.08966600365830146 |
| - type: precision |
| value: 0.05066184676394412 |
| - type: recall |
| value: 0.6925207756232686 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ces-eng) |
| config: ces-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 11.1 |
| - type: f1 |
| value: 8.28646043238052 |
| - type: precision |
| value: 7.686198801198802 |
| - type: recall |
| value: 11.1 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (tzl-eng) |
| config: tzl-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 38.46153846153847 |
| - type: f1 |
| value: 31.640899949723472 |
| - type: precision |
| value: 29.298878205128204 |
| - type: recall |
| value: 38.46153846153847 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (urd-eng) |
| config: urd-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 81.2 |
| - type: f1 |
| value: 76.77103174603175 |
| - type: precision |
| value: 74.96511904761905 |
| - type: recall |
| value: 81.2 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (ara-eng) |
| config: ara-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 90.60000000000001 |
| - type: f1 |
| value: 88.20666666666665 |
| - type: precision |
| value: 87.14833333333334 |
| - type: recall |
| value: 90.60000000000001 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (kor-eng) |
| config: kor-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 35.699999999999996 |
| - type: f1 |
| value: 29.159127620745267 |
| - type: precision |
| value: 27.109529030910608 |
| - type: recall |
| value: 35.699999999999996 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (yid-eng) |
| config: yid-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 0.9433962264150944 |
| - type: f1 |
| value: 0.28088681664921333 |
| - type: precision |
| value: 0.22694150916099465 |
| - type: recall |
| value: 0.9433962264150944 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (fin-eng) |
| config: fin-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 7.5 |
| - type: f1 |
| value: 5.825362182391272 |
| - type: precision |
| value: 5.526187577939453 |
| - type: recall |
| value: 7.5 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (tha-eng) |
| config: tha-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 4.197080291970803 |
| - type: f1 |
| value: 3.079215618580677 |
| - type: precision |
| value: 2.8501768792419 |
| - type: recall |
| value: 4.197080291970803 |
| - task: |
| type: BitextMining |
| dataset: |
| type: mteb/tatoeba-bitext-mining |
| name: MTEB Tatoeba (wuu-eng) |
| config: wuu-eng |
| split: test |
| revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
| metrics: |
| - type: accuracy |
| value: 87.9 |
| - type: f1 |
| value: 84.60499999999999 |
| - type: precision |
| value: 83.11428571428571 |
| - type: recall |
| value: 87.9 |
| - task: |
| type: Clustering |
| dataset: |
| type: C-MTEB/ThuNewsClusteringP2P |
| name: MTEB ThuNewsClusteringP2P |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: v_measure |
| value: 50.23655676494653 |
| - task: |
| type: Clustering |
| dataset: |
| type: C-MTEB/ThuNewsClusteringS2S |
| name: MTEB ThuNewsClusteringS2S |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: v_measure |
| value: 49.54033078256682 |
| - task: |
| type: Retrieval |
| dataset: |
| type: webis-touche2020 |
| name: MTEB Touche2020 |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 2.299 |
| - type: map_at_10 |
| value: 9.232999999999999 |
| - type: map_at_100 |
| value: 15.156 |
| - type: map_at_1000 |
| value: 16.63 |
| - type: map_at_3 |
| value: 4.2250000000000005 |
| - type: map_at_5 |
| value: 6.078 |
| - type: mrr_at_1 |
| value: 30.612000000000002 |
| - type: mrr_at_10 |
| value: 45.158 |
| - type: mrr_at_100 |
| value: 45.9 |
| - type: mrr_at_1000 |
| value: 45.910000000000004 |
| - type: mrr_at_3 |
| value: 39.456 |
| - type: mrr_at_5 |
| value: 42.925000000000004 |
| - type: ndcg_at_1 |
| value: 29.592000000000002 |
| - type: ndcg_at_10 |
| value: 25.166 |
| - type: ndcg_at_100 |
| value: 35.35 |
| - type: ndcg_at_1000 |
| value: 46.67 |
| - type: ndcg_at_3 |
| value: 24.545 |
| - type: ndcg_at_5 |
| value: 25.112000000000002 |
| - type: precision_at_1 |
| value: 30.612000000000002 |
| - type: precision_at_10 |
| value: 23.673 |
| - type: precision_at_100 |
| value: 7.428999999999999 |
| - type: precision_at_1000 |
| value: 1.482 |
| - type: precision_at_3 |
| value: 23.810000000000002 |
| - type: precision_at_5 |
| value: 25.306 |
| - type: recall_at_1 |
| value: 2.299 |
| - type: recall_at_10 |
| value: 16.801 |
| - type: recall_at_100 |
| value: 45.506 |
| - type: recall_at_1000 |
| value: 79.985 |
| - type: recall_at_3 |
| value: 5.069 |
| - type: recall_at_5 |
| value: 8.863999999999999 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/toxic_conversations_50k |
| name: MTEB ToxicConversationsClassification |
| config: default |
| split: test |
| revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c |
| metrics: |
| - type: accuracy |
| value: 72.1314 |
| - type: ap |
| value: 14.605968497007712 |
| - type: f1 |
| value: 55.37284214772282 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/tweet_sentiment_extraction |
| name: MTEB TweetSentimentExtractionClassification |
| config: default |
| split: test |
| revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
| metrics: |
| - type: accuracy |
| value: 61.044142614601014 |
| - type: f1 |
| value: 61.30028928459138 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/twentynewsgroups-clustering |
| name: MTEB TwentyNewsgroupsClustering |
| config: default |
| split: test |
| revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
| metrics: |
| - type: v_measure |
| value: 41.28707371610032 |
| - task: |
| type: PairClassification |
| dataset: |
| type: mteb/twittersemeval2015-pairclassification |
| name: MTEB TwitterSemEval2015 |
| config: default |
| split: test |
| revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
| metrics: |
| - type: cos_sim_accuracy |
| value: 85.09864695714371 |
| - type: cos_sim_ap |
| value: 70.63738634684302 |
| - type: cos_sim_f1 |
| value: 66.12903225806453 |
| - type: cos_sim_precision |
| value: 64.22178020885131 |
| - type: cos_sim_recall |
| value: 68.15303430079156 |
| - type: dot_accuracy |
| value: 83.59063002920665 |
| - type: dot_ap |
| value: 66.68356189934075 |
| - type: dot_f1 |
| value: 63.27201851626264 |
| - type: dot_precision |
| value: 58.76895225164064 |
| - type: dot_recall |
| value: 68.52242744063325 |
| - type: euclidean_accuracy |
| value: 85.027120462538 |
| - type: euclidean_ap |
| value: 69.99328290454234 |
| - type: euclidean_f1 |
| value: 65.23797657612758 |
| - type: euclidean_precision |
| value: 61.803588290840416 |
| - type: euclidean_recall |
| value: 69.07651715039577 |
| - type: manhattan_accuracy |
| value: 85.02115992132086 |
| - type: manhattan_ap |
| value: 69.91284274429754 |
| - type: manhattan_f1 |
| value: 65.19297407097623 |
| - type: manhattan_precision |
| value: 59.5763267088884 |
| - type: manhattan_recall |
| value: 71.97889182058047 |
| - type: max_accuracy |
| value: 85.09864695714371 |
| - type: max_ap |
| value: 70.63738634684302 |
| - type: max_f1 |
| value: 66.12903225806453 |
| - task: |
| type: PairClassification |
| dataset: |
| type: mteb/twitterurlcorpus-pairclassification |
| name: MTEB TwitterURLCorpus |
| config: default |
| split: test |
| revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
| metrics: |
| - type: cos_sim_accuracy |
| value: 89.119804400978 |
| - type: cos_sim_ap |
| value: 86.1777422918812 |
| - type: cos_sim_f1 |
| value: 78.57841293719444 |
| - type: cos_sim_precision |
| value: 76.80488163505366 |
| - type: cos_sim_recall |
| value: 80.4357868801971 |
| - type: dot_accuracy |
| value: 88.86366282454303 |
| - type: dot_ap |
| value: 84.1891332504211 |
| - type: dot_f1 |
| value: 78.31691507672025 |
| - type: dot_precision |
| value: 74.67700258397933 |
| - type: dot_recall |
| value: 82.32984293193716 |
| - type: euclidean_accuracy |
| value: 88.74141343578997 |
| - type: euclidean_ap |
| value: 85.60421594792011 |
| - type: euclidean_f1 |
| value: 77.79556879538262 |
| - type: euclidean_precision |
| value: 75.32991995384727 |
| - type: euclidean_recall |
| value: 80.42808746535263 |
| - type: manhattan_accuracy |
| value: 88.7782822990647 |
| - type: manhattan_ap |
| value: 85.61374819166252 |
| - type: manhattan_f1 |
| value: 77.78237795927583 |
| - type: manhattan_precision |
| value: 76.08423532876813 |
| - type: manhattan_recall |
| value: 79.55805358792732 |
| - type: max_accuracy |
| value: 89.119804400978 |
| - type: max_ap |
| value: 86.1777422918812 |
| - type: max_f1 |
| value: 78.57841293719444 |
| - task: |
| type: Retrieval |
| dataset: |
| type: C-MTEB/VideoRetrieval |
| name: MTEB VideoRetrieval |
| config: default |
| split: dev |
| revision: None |
| metrics: |
| - type: map_at_1 |
| value: 41.8 |
| - type: map_at_10 |
| value: 51.456999999999994 |
| - type: map_at_100 |
| value: 52.107000000000006 |
| - type: map_at_1000 |
| value: 52.141999999999996 |
| - type: map_at_3 |
| value: 48.717 |
| - type: map_at_5 |
| value: 50.452 |
| - type: mrr_at_1 |
| value: 41.8 |
| - type: mrr_at_10 |
| value: 51.441 |
| - type: mrr_at_100 |
| value: 52.091 |
| - type: mrr_at_1000 |
| value: 52.125 |
| - type: mrr_at_3 |
| value: 48.699999999999996 |
| - type: mrr_at_5 |
| value: 50.434999999999995 |
| - type: ndcg_at_1 |
| value: 41.8 |
| - type: ndcg_at_10 |
| value: 56.537000000000006 |
| - type: ndcg_at_100 |
| value: 59.901 |
| - type: ndcg_at_1000 |
| value: 60.889 |
| - type: ndcg_at_3 |
| value: 51.019999999999996 |
| - type: ndcg_at_5 |
| value: 54.106 |
| - type: precision_at_1 |
| value: 41.8 |
| - type: precision_at_10 |
| value: 7.26 |
| - type: precision_at_100 |
| value: 0.8880000000000001 |
| - type: precision_at_1000 |
| value: 0.097 |
| - type: precision_at_3 |
| value: 19.233 |
| - type: precision_at_5 |
| value: 13.020000000000001 |
| - type: recall_at_1 |
| value: 41.8 |
| - type: recall_at_10 |
| value: 72.6 |
| - type: recall_at_100 |
| value: 88.8 |
| - type: recall_at_1000 |
| value: 96.7 |
| - type: recall_at_3 |
| value: 57.699999999999996 |
| - type: recall_at_5 |
| value: 65.10000000000001 |
| - task: |
| type: Classification |
| dataset: |
| type: C-MTEB/waimai-classification |
| name: MTEB Waimai |
| config: default |
| split: test |
| revision: None |
| metrics: |
| - type: accuracy |
| value: 84.07 |
| - type: ap |
| value: 65.23766736490957 |
| - type: f1 |
| value: 82.17794239849368 |
| --- |
| |
| # Model Card for udever-bloom |
|
|
| <!-- Provide a quick summary of what the model is/does. --> |
|
|
| `udever-bloom-3b` is finetuned from [bigscience/bloom-3b](https://huggingface.co/bigscience/bloom-3b) via [BitFit](https://aclanthology.org/2022.acl-short.1/) on MS MARCO Passage Ranking, SNLI and MultiNLI data. |
| It is a universal embedding model across tasks, natural and programming languages. |
| (From the technical view, `udever` is merely with some minor improvements to `sgpt-bloom`) |
|
|
| <div align=center><img width="338" height="259" src="https://user-images.githubusercontent.com/26690193/277643721-cdb7f227-cae5-40e1-b6e1-a201bde00339.png" /></div> |
|
|
|
|
| ## Model Details |
|
|
| ### Model Description |
|
|
| - **Developed by:** Alibaba Group |
| - **Model type:** Transformer-based Language Model (decoder-only) |
| - **Language(s) (NLP):** Multiple; see [bloom training data](https://huggingface.co/bigscience/bloom-3b#training-data) |
| - **Finetuned from model :** [bigscience/bloom-3b](https://huggingface.co/bigscience/bloom-3b) |
|
|
| ### Model Sources |
|
|
| <!-- Provide the basic links for the model. --> |
|
|
| - **Repository:** [github.com/izhx/uni-rep](https://github.com/izhx/uni-rep) |
| - **Paper :** [Language Models are Universal Embedders](https://arxiv.org/pdf/2310.08232.pdf) |
| - **Training Date :** 2023-06 |
|
|
|
|
|
|
| ## How to Get Started with the Model |
|
|
| Use the code below to get started with the model. |
|
|
| ```python |
| import torch |
| from transformers import AutoTokenizer, BloomModel |
| |
| tokenizer = AutoTokenizer.from_pretrained('izhx/udever-bloom-3b') |
| model = BloomModel.from_pretrained('izhx/udever-bloom-3b') |
| |
| boq, eoq, bod, eod = '[BOQ]', '[EOQ]', '[BOD]', '[EOD]' |
| eoq_id, eod_id = tokenizer.convert_tokens_to_ids([eoq, eod]) |
| |
| if tokenizer.padding_side != 'left': |
| print('!!!', tokenizer.padding_side) |
| tokenizer.padding_side = 'left' |
| |
| |
| def encode(texts: list, is_query: bool = True, max_length=300): |
| bos = boq if is_query else bod |
| eos_id = eoq_id if is_query else eod_id |
| texts = [bos + t for t in texts] |
| encoding = tokenizer( |
| texts, truncation=True, max_length=max_length - 1, padding=True |
| ) |
| for ids, mask in zip(encoding['input_ids'], encoding['attention_mask']): |
| ids.append(eos_id) |
| mask.append(1) |
| inputs = tokenizer.pad(encoding, return_tensors='pt') |
| with torch.inference_mode(): |
| outputs = model(**inputs) |
| embeds = outputs.last_hidden_state[:, -1] |
| return embeds |
| |
| encode(['I am Bert', 'You are Elmo']) |
| |
| ``` |
|
|
| ## Training Details |
|
|
| ### Training Data |
|
|
| <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. --> |
|
|
| - MS MARCO Passage Ranking, retrieved by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86) |
| - SNLI and MultiNLI (https://sbert.net/datasets/AllNLI.tsv.gz) |
| |
| |
| ### Training Procedure |
| |
| <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. --> |
| |
| #### Preprocessing |
| |
| MS MARCO hard negatives provided by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86). |
| Negatives for SNLI and MultiNLI are randomly sampled. |
|
|
|
|
| #### Training Hyperparameters |
|
|
| - **Training regime:** tf32, BitFit |
| - **Batch size:** 1024 |
| - **Epochs:** 3 |
| - **Optimizer:** AdamW |
| - **Learning rate:** 1e-4 |
| - **Scheduler:** constant with warmup. |
| - **Warmup:** 0.25 epoch |
|
|
|
|
| ## Evaluation |
|
|
| ### Table 1: Massive Text Embedding Benchmark [MTEB](https://huggingface.co/spaces/mteb/leaderboard) |
|
|
| | MTEB | Avg. | Class. | Clust. | PairClass. | Rerank. | Retr. | STS | Summ. | |
| |-----------------------------|--------------|--------------|--------------|--------------|--------------|--------------|--------------|--------| |
| | #Datasets ➡️ | 56 | 12 | 11 | 3 | 4 | 15 | 10 | 1 | |
| || |
| | bge-large-en-v1.5 | **64.23** | **75.97** | 46.08| **87.12** | **60.03** | **54.29** | 83.11| 31.61 | |
| | bge-base-en-v1.5 | 63.55| 75.53| 45.77| 86.55| 58.86| 53.25| 82.4| 31.07 | |
| | gte-large | 63.13| 73.33| **46.84** | 85| 59.13| 52.22| **83.35** | 31.66 | |
| | gte-base | 62.39| 73.01| 46.2| 84.57| 58.61| 51.14| 82.3| 31.17 | |
| | e5-large-v2 | 62.25| 75.24| 44.49| 86.03| 56.61| 50.56| 82.05| 30.19 | |
| | instructor-xl | 61.79| 73.12| 44.74| 86.62| 57.29| 49.26| 83.06| 32.32 | |
| | instructor-large | 61.59| 73.86| 45.29| 85.89| 57.54| 47.57| 83.15| 31.84 | |
| | e5-base-v2 | 61.5 | 73.84| 43.8| 85.73| 55.91| 50.29| 81.05| 30.28 | |
| | e5-large | 61.42| 73.14| 43.33| 85.94| 56.53| 49.99| 82.06| 30.97 | |
| | text-embedding-ada-002 (OpenAI API) | 60.99| 70.93| 45.9 | 84.89| 56.32| 49.25| 80.97| 30.8 | |
| | e5-base | 60.44| 72.63| 42.11| 85.09| 55.7 | 48.75| 80.96| 31.01 | |
| | SGPT-5.8B-msmarco | 58.93| 68.13| 40.34| 82 | 56.56| 50.25| 78.1 | 31.46 | |
| | sgpt-bloom-7b1-msmarco | 57.59| 66.19| 38.93| 81.9 | 55.65| 48.22| 77.74| **33.6** | |
| || |
| | Udever-bloom-560m | 55.80| 68.04| 36.89| 81.05| 52.60| 41.19| 79.93| 32.06 | |
| | Udever-bloom-1b1 | 58.28| 70.18| 39.11| 83.11| 54.28| 45.27| 81.52| 31.10 | |
| | Udever-bloom-3b | 59.86| 71.91| 40.74| 84.06| 54.90| 47.67| 82.37| 30.62 | |
| | Udever-bloom-7b1 | 60.63 | 72.13| 40.81| 85.40| 55.91| 49.34| 83.01| 30.97 | |
|
|
|
|
| ### Table 2: [CodeSearchNet](https://github.com/github/CodeSearchNet) |
|
|
| | CodeSearchNet | Go | Ruby | Python | Java | JS | PHP | Avg. | |
| |-|-|-|-|-|-|-|-| |
| | CodeBERT | 69.3 | 70.6 | 84.0 | 86.8 | 74.8 | 70.6 | 76.0 | |
| | GraphCodeBERT | 84.1 | 73.2 | 87.9 | 75.7 | 71.1 | 72.5 | 77.4 | |
| | cpt-code S | **97.7** | **86.3** | 99.8 | 94.0 | 86.0 | 96.7 | 93.4 | |
| | cpt-code M | 97.5 | 85.5 | **99.9** | **94.4** | **86.5** | **97.2** | **93.5** | |
| | sgpt-bloom-7b1-msmarco | 76.79 | 69.25 | 95.68 | 77.93 | 70.35 | 73.45 | 77.24 | |
| || |
| | Udever-bloom-560m | 75.38 | 66.67 | 96.23 | 78.99 | 69.39 | 73.69 | 76.73 | |
| | Udever-bloom-1b1 | 78.76 | 72.85 | 97.67 | 82.77 | 74.38 | 78.97 | 80.90 | |
| | Udever-bloom-3b | 80.63 | 75.40 | 98.02 | 83.88 | 76.18 | 79.67 | 82.29 | |
| | Udever-bloom-7b1 | 79.37 | 76.59 | 98.38 | 84.68 | 77.49 | 80.03 | 82.76 | |
|
|
|
|
| ### Table 3: Chinese multi-domain retrieval [Multi-cpr](https://dl.acm.org/doi/10.1145/3477495.3531736) |
|
|
| | | | |E-commerce | | Entertainment video | | Medical | | |
| |--|--|--|--|--|--|--|--|--| |
| | Model | Train | Backbone | MRR@10 | Recall@1k | MRR@10 | Recall@1k | MRR@10 | Recall@1k | |
| || |
| | BM25 | - | - | 0.225 | 0.815 | 0.225 | 0.780 | 0.187 | 0.482 | |
| | Doc2Query | - | - | 0.239 | 0.826 | 0.238 | 0.794 | 0.210 | 0.505 | |
| | DPR-1 | In-Domain | BERT | 0.270 | 0.921 | 0.254 | 0.934 | 0.327 | 0.747 | |
| | DPR-2 | In-Domain | BERT-CT | 0.289 | **0.926** | 0.263 | **0.935** | 0.339 | **0.769** | |
| | text-embedding-ada-002 | General | GPT | 0.183 | 0.825 | 0.159 | 0.786 | 0.245 | 0.593 | |
| | sgpt-bloom-7b1-msmarco | General | BLOOM | 0.242 | 0.840 | 0.227 | 0.829 | 0.311 | 0.675 | |
| || |
| | Udever-bloom-560m | General | BLOOM | 0.156 | 0.802 | 0.149 | 0.749 | 0.245 | 0.571 | |
| | Udever-bloom-1b1 | General | BLOOM | 0.244 | 0.863 | 0.208 | 0.815 | 0.241 | 0.557 | |
| | Udever-bloom-3b | General | BLOOM | 0.267 | 0.871 | 0.228 | 0.836 | 0.288 | 0.619 | |
| | Udever-bloom-7b1 | General | BLOOM | **0.296** | 0.889 | **0.267** | 0.907 | **0.343** | 0.705 | |
|
|
| #### More results refer to [paper](https://arxiv.org/pdf/2310.08232.pdf) section 3. |
|
|
|
|
|
|
| ## Technical Specifications |
|
|
| ### Model Architecture and Objective |
|
|
| - Model: [bigscience/bloom-3b](https://huggingface.co/bigscience/bloom-3b). |
| - Objective: Constrastive loss with hard negatives (refer to [paper](https://arxiv.org/pdf/2310.08232.pdf) section 2.2). |
|
|
|
|
| ### Compute Infrastructure |
|
|
| - Nvidia A100 SXM4 80GB. |
| - torch 2.0.0, transformers 4.29.2. |
|
|
|
|
| ## Citation |
|
|
| **BibTeX:** |
|
|
| ```BibTeX |
| @article{zhang2023language, |
| title={Language Models are Universal Embedders}, |
| author={Zhang, Xin and Li, Zehan and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Zhang, Min}, |
| journal={arXiv preprint arXiv:2310.08232}, |
| year={2023} |
| } |
| ``` |
|
|
