| | --- |
| | model-index: |
| | - name: Quark-Emb-8B |
| | results: |
| | - dataset: |
| | config: default |
| | name: MTEB AFQMC (default) |
| | revision: latest2023 |
| | split: validation |
| | type: C-MTEB/AFQMC |
| | metrics: |
| | - type: cosine_pearson |
| | value: 52.87704664791064 |
| | - type: cosine_spearman |
| | value: 53.567003436521375 |
| | - type: manhattan_pearson |
| | value: 52.07472780799189 |
| | - type: manhattan_spearman |
| | value: 53.5368469974003 |
| | - type: euclidean_pearson |
| | value: 52.074186684368016 |
| | - type: euclidean_spearman |
| | value: 53.515536447088074 |
| | - type: main_score |
| | value: 53.567003436521375 |
| | task: |
| | type: STS |
| | - dataset: |
| | config: default |
| | name: MTEB ATEC (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/ATEC |
| | metrics: |
| | - type: cosine_pearson |
| | value: 59.13301114775821 |
| | - type: cosine_spearman |
| | value: 53.42152760117668 |
| | - type: manhattan_pearson |
| | value: 60.05185745744783 |
| | - type: manhattan_spearman |
| | value: 53.36914545708813 |
| | - type: euclidean_pearson |
| | value: 60.17725014927802 |
| | - type: euclidean_spearman |
| | value: 53.431110991334485 |
| | - type: main_score |
| | value: 53.42152760117668 |
| | task: |
| | type: STS |
| | - dataset: |
| | config: zh |
| | name: MTEB AmazonReviewsClassification (zh) |
| | revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| | split: test |
| | type: mteb/amazon_reviews_multi |
| | metrics: |
| | - type: accuracy |
| | value: 54.74600000000001 |
| | - type: accuracy_stderr |
| | value: 1.060492338491892 |
| | - type: f1 |
| | value: 53.49846112279175 |
| | - type: f1_stderr |
| | value: 1.729174511160517 |
| | - type: main_score |
| | value: 54.74600000000001 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: zh |
| | name: MTEB AmazonReviewsClassification (zh) |
| | revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| | split: validation |
| | type: mteb/amazon_reviews_multi |
| | metrics: |
| | - type: accuracy |
| | value: 53.468 |
| | - type: accuracy_stderr |
| | value: 1.0129639677698308 |
| | - type: f1 |
| | value: 52.21987651679265 |
| | - type: f1_stderr |
| | value: 1.7016822177116173 |
| | - type: main_score |
| | value: 53.468 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: default |
| | name: MTEB BQ (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/BQ |
| | metrics: |
| | - type: cosine_pearson |
| | value: 69.640366232364 |
| | - type: cosine_spearman |
| | value: 70.65881213017273 |
| | - type: manhattan_pearson |
| | value: 67.76837799100343 |
| | - type: manhattan_spearman |
| | value: 70.5046111101055 |
| | - type: euclidean_pearson |
| | value: 67.83004194158737 |
| | - type: euclidean_spearman |
| | value: 70.60927547682859 |
| | - type: main_score |
| | value: 70.65881213017273 |
| | task: |
| | type: STS |
| | - dataset: |
| | config: default |
| | name: MTEB CLSClusteringP2P (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/CLSClusteringP2P |
| | metrics: |
| | - type: v_measure |
| | value: 62.32714079793593 |
| | - type: v_measure_std |
| | value: 1.5782386182731478 |
| | - type: main_score |
| | value: 62.32714079793593 |
| | task: |
| | type: Clustering |
| | - dataset: |
| | config: default |
| | name: MTEB CLSClusteringS2S (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/CLSClusteringS2S |
| | metrics: |
| | - type: v_measure |
| | value: 59.29532340833129 |
| | - type: v_measure_std |
| | value: 1.5258658358346424 |
| | - type: main_score |
| | value: 59.29532340833129 |
| | task: |
| | type: Clustering |
| | - dataset: |
| | config: default |
| | name: MTEB CMedQAv1 |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/CMedQAv1-reranking |
| | metrics: |
| | - type: map |
| | value: 88.02263756085355 |
| | - type: mrr |
| | value: 90.18928571428572 |
| | - type: main_score |
| | value: 88.02263756085355 |
| | task: |
| | type: Reranking |
| | - dataset: |
| | config: default |
| | name: MTEB CMedQAv2 |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/CMedQAv2-reranking |
| | metrics: |
| | - type: map |
| | value: 88.81199829110464 |
| | - type: mrr |
| | value: 90.81817460317461 |
| | - type: main_score |
| | value: 88.81199829110464 |
| | task: |
| | type: Reranking |
| | - dataset: |
| | config: default |
| | name: MTEB CmedqaRetrieval (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/CmedqaRetrieval |
| | metrics: |
| | - type: map_at_1 |
| | value: 27.448 |
| | - type: map_at_10 |
| | value: 40.794000000000004 |
| | - type: map_at_100 |
| | value: 42.606 |
| | - type: map_at_1000 |
| | value: 42.711 |
| | - type: map_at_20 |
| | value: 41.778 |
| | - type: map_at_3 |
| | value: 36.429 |
| | - type: map_at_5 |
| | value: 38.841 |
| | - type: mrr_at_1 |
| | value: 41.510000000000005 |
| | - type: mrr_at_10 |
| | value: 49.986999999999995 |
| | - type: mrr_at_100 |
| | value: 50.908 |
| | - type: mrr_at_1000 |
| | value: 50.946000000000005 |
| | - type: mrr_at_20 |
| | value: 50.531000000000006 |
| | - type: mrr_at_3 |
| | value: 47.562 |
| | - type: mrr_at_5 |
| | value: 48.882 |
| | - type: ndcg_at_1 |
| | value: 41.510000000000005 |
| | - type: ndcg_at_10 |
| | value: 47.620000000000005 |
| | - type: ndcg_at_100 |
| | value: 54.586999999999996 |
| | - type: ndcg_at_1000 |
| | value: 56.324 |
| | - type: ndcg_at_20 |
| | value: 50.332 |
| | - type: ndcg_at_3 |
| | value: 42.27 |
| | - type: ndcg_at_5 |
| | value: 44.421 |
| | - type: precision_at_1 |
| | value: 41.510000000000005 |
| | - type: precision_at_10 |
| | value: 10.45 |
| | - type: precision_at_100 |
| | value: 1.6179999999999999 |
| | - type: precision_at_1000 |
| | value: 0.184 |
| | - type: precision_at_20 |
| | value: 6.1339999999999995 |
| | - type: precision_at_3 |
| | value: 23.823 |
| | - type: precision_at_5 |
| | value: 17.089 |
| | - type: recall_at_1 |
| | value: 27.448 |
| | - type: recall_at_10 |
| | value: 58.629 |
| | - type: recall_at_100 |
| | value: 87.26899999999999 |
| | - type: recall_at_1000 |
| | value: 98.713 |
| | - type: recall_at_20 |
| | value: 67.929 |
| | - type: recall_at_3 |
| | value: 42.331 |
| | - type: recall_at_5 |
| | value: 49.193999999999996 |
| | - type: main_score |
| | value: 47.620000000000005 |
| | task: |
| | type: Retrieval |
| | - dataset: |
| | config: default |
| | name: MTEB Cmnli (default) |
| | revision: latest2023 |
| | split: validation |
| | type: C-MTEB/CMNLI |
| | metrics: |
| | - type: cos_sim_accuracy |
| | value: 91.040288634997 |
| | - type: cos_sim_accuracy_threshold |
| | value: 96.31753556207411 |
| | - type: cos_sim_ap |
| | value: 95.94857244353375 |
| | - type: cos_sim_f1 |
| | value: 91.47971901200997 |
| | - type: cos_sim_f1_threshold |
| | value: 96.01236110428391 |
| | - type: cos_sim_precision |
| | value: 88.74477907232358 |
| | - type: cos_sim_recall |
| | value: 94.388590133271 |
| | - type: dot_accuracy |
| | value: 80.7696933253157 |
| | - type: dot_accuracy_threshold |
| | value: 58.4022485296251 |
| | - type: dot_ap |
| | value: 89.17817373664943 |
| | - type: dot_f1 |
| | value: 81.62572172534811 |
| | - type: dot_f1_threshold |
| | value: 58.18378730482039 |
| | - type: dot_precision |
| | value: 79.12642669007901 |
| | - type: dot_recall |
| | value: 84.28805237315876 |
| | - type: euclidean_accuracy |
| | value: 90.92002405291642 |
| | - type: euclidean_accuracy_threshold |
| | value: 21.553298629922512 |
| | - type: euclidean_ap |
| | value: 95.90941014786691 |
| | - type: euclidean_f1 |
| | value: 91.45241317095173 |
| | - type: euclidean_f1_threshold |
| | value: 22.109074422645463 |
| | - type: euclidean_precision |
| | value: 88.32498366368982 |
| | - type: euclidean_recall |
| | value: 94.80944587327565 |
| | - type: manhattan_accuracy |
| | value: 90.94407696933253 |
| | - type: manhattan_accuracy_threshold |
| | value: 524.1466620016906 |
| | - type: manhattan_ap |
| | value: 95.89310684813798 |
| | - type: manhattan_f1 |
| | value: 91.50400541577343 |
| | - type: manhattan_f1_threshold |
| | value: 525.5181014869215 |
| | - type: manhattan_precision |
| | value: 88.4212821631051 |
| | - type: manhattan_recall |
| | value: 94.80944587327565 |
| | - type: max_accuracy |
| | value: 91.040288634997 |
| | - type: max_ap |
| | value: 95.94857244353375 |
| | - type: max_f1 |
| | value: 91.50400541577343 |
| | task: |
| | type: PairClassification |
| | - dataset: |
| | config: default |
| | name: MTEB CovidRetrieval (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/CovidRetrieval |
| | metrics: |
| | - type: map_at_1 |
| | value: 78.82000000000001 |
| | - type: map_at_10 |
| | value: 85.51100000000001 |
| | - type: map_at_100 |
| | value: 85.67099999999999 |
| | - type: map_at_1000 |
| | value: 85.672 |
| | - type: map_at_20 |
| | value: 85.641 |
| | - type: map_at_3 |
| | value: 84.321 |
| | - type: map_at_5 |
| | value: 85.048 |
| | - type: mrr_at_1 |
| | value: 78.925 |
| | - type: mrr_at_10 |
| | value: 85.548 |
| | - type: mrr_at_100 |
| | value: 85.698 |
| | - type: mrr_at_1000 |
| | value: 85.699 |
| | - type: mrr_at_20 |
| | value: 85.669 |
| | - type: mrr_at_3 |
| | value: 84.45700000000001 |
| | - type: mrr_at_5 |
| | value: 85.12100000000001 |
| | - type: ndcg_at_1 |
| | value: 78.925 |
| | - type: ndcg_at_10 |
| | value: 88.359 |
| | - type: ndcg_at_100 |
| | value: 88.98899999999999 |
| | - type: ndcg_at_1000 |
| | value: 89.017 |
| | - type: ndcg_at_20 |
| | value: 88.776 |
| | - type: ndcg_at_3 |
| | value: 86.086 |
| | - type: ndcg_at_5 |
| | value: 87.336 |
| | - type: precision_at_1 |
| | value: 78.925 |
| | - type: precision_at_10 |
| | value: 9.789 |
| | - type: precision_at_100 |
| | value: 1.0070000000000001 |
| | - type: precision_at_1000 |
| | value: 0.101 |
| | - type: precision_at_20 |
| | value: 4.979 |
| | - type: precision_at_3 |
| | value: 30.488 |
| | - type: precision_at_5 |
| | value: 18.925 |
| | - type: recall_at_1 |
| | value: 78.82000000000001 |
| | - type: recall_at_10 |
| | value: 96.997 |
| | - type: recall_at_100 |
| | value: 99.684 |
| | - type: recall_at_1000 |
| | value: 99.895 |
| | - type: recall_at_20 |
| | value: 98.52499999999999 |
| | - type: recall_at_3 |
| | value: 91.01700000000001 |
| | - type: recall_at_5 |
| | value: 93.994 |
| | - type: main_score |
| | value: 88.359 |
| | task: |
| | type: Retrieval |
| | - dataset: |
| | config: default |
| | name: MTEB DuRetrieval (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/DuRetrieval |
| | metrics: |
| | - type: map_at_1 |
| | value: 28.03 |
| | - type: map_at_10 |
| | value: 85.60600000000001 |
| | - type: map_at_100 |
| | value: 88.14800000000001 |
| | - type: map_at_1000 |
| | value: 88.169 |
| | - type: map_at_20 |
| | value: 87.591 |
| | - type: map_at_3 |
| | value: 60.06 |
| | - type: map_at_5 |
| | value: 75.564 |
| | - type: mrr_at_1 |
| | value: 94.05 |
| | - type: mrr_at_10 |
| | value: 96.043 |
| | - type: mrr_at_100 |
| | value: 96.075 |
| | - type: mrr_at_1000 |
| | value: 96.077 |
| | - type: mrr_at_20 |
| | value: 96.06099999999999 |
| | - type: mrr_at_3 |
| | value: 95.875 |
| | - type: mrr_at_5 |
| | value: 96.017 |
| | - type: ndcg_at_1 |
| | value: 94.05 |
| | - type: ndcg_at_10 |
| | value: 91.58800000000001 |
| | - type: ndcg_at_100 |
| | value: 93.536 |
| | - type: ndcg_at_1000 |
| | value: 93.726 |
| | - type: ndcg_at_20 |
| | value: 92.64099999999999 |
| | - type: ndcg_at_3 |
| | value: 90.865 |
| | - type: ndcg_at_5 |
| | value: 89.972 |
| | - type: precision_at_1 |
| | value: 94.05 |
| | - type: precision_at_10 |
| | value: 43.19 |
| | - type: precision_at_100 |
| | value: 4.859 |
| | - type: precision_at_1000 |
| | value: 0.49 |
| | - type: precision_at_20 |
| | value: 23.3 |
| | - type: precision_at_3 |
| | value: 81.0 |
| | - type: precision_at_5 |
| | value: 68.36 |
| | - type: recall_at_1 |
| | value: 28.03 |
| | - type: recall_at_10 |
| | value: 92.095 |
| | - type: recall_at_100 |
| | value: 98.764 |
| | - type: recall_at_1000 |
| | value: 99.71 |
| | - type: recall_at_20 |
| | value: 95.87 |
| | - type: recall_at_3 |
| | value: 61.949 |
| | - type: recall_at_5 |
| | value: 79.41 |
| | - type: main_score |
| | value: 91.58800000000001 |
| | task: |
| | type: Retrieval |
| | - dataset: |
| | config: default |
| | name: MTEB EcomRetrieval (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/EcomRetrieval |
| | metrics: |
| | - type: map_at_1 |
| | value: 58.599999999999994 |
| | - type: map_at_10 |
| | value: 68.88499999999999 |
| | - type: map_at_100 |
| | value: 69.269 |
| | - type: map_at_1000 |
| | value: 69.274 |
| | - type: map_at_20 |
| | value: 69.17699999999999 |
| | - type: map_at_3 |
| | value: 66.167 |
| | - type: map_at_5 |
| | value: 68.082 |
| | - type: mrr_at_1 |
| | value: 58.599999999999994 |
| | - type: mrr_at_10 |
| | value: 68.88499999999999 |
| | - type: mrr_at_100 |
| | value: 69.269 |
| | - type: mrr_at_1000 |
| | value: 69.274 |
| | - type: mrr_at_20 |
| | value: 69.17699999999999 |
| | - type: mrr_at_3 |
| | value: 66.167 |
| | - type: mrr_at_5 |
| | value: 68.082 |
| | - type: ndcg_at_1 |
| | value: 58.599999999999994 |
| | - type: ndcg_at_10 |
| | value: 74.018 |
| | - type: ndcg_at_100 |
| | value: 75.72 |
| | - type: ndcg_at_1000 |
| | value: 75.851 |
| | - type: ndcg_at_20 |
| | value: 75.08 |
| | - type: ndcg_at_3 |
| | value: 68.64 |
| | - type: ndcg_at_5 |
| | value: 72.075 |
| | - type: precision_at_1 |
| | value: 58.599999999999994 |
| | - type: precision_at_10 |
| | value: 9.01 |
| | - type: precision_at_100 |
| | value: 0.9769999999999999 |
| | - type: precision_at_1000 |
| | value: 0.099 |
| | - type: precision_at_20 |
| | value: 4.715 |
| | - type: precision_at_3 |
| | value: 25.267 |
| | - type: precision_at_5 |
| | value: 16.82 |
| | - type: recall_at_1 |
| | value: 58.599999999999994 |
| | - type: recall_at_10 |
| | value: 90.10000000000001 |
| | - type: recall_at_100 |
| | value: 97.7 |
| | - type: recall_at_1000 |
| | value: 98.7 |
| | - type: recall_at_20 |
| | value: 94.3 |
| | - type: recall_at_3 |
| | value: 75.8 |
| | - type: recall_at_5 |
| | value: 84.1 |
| | - type: main_score |
| | value: 74.018 |
| | task: |
| | type: Retrieval |
| | - dataset: |
| | config: default |
| | name: MTEB IFlyTek (default) |
| | revision: latest2023 |
| | split: validation |
| | type: C-MTEB/IFlyTek-classification |
| | metrics: |
| | - type: accuracy |
| | value: 55.79838399384378 |
| | - type: accuracy_stderr |
| | value: 0.273588131352537 |
| | - type: f1 |
| | value: 42.23811666656058 |
| | - type: f1_stderr |
| | value: 0.2317340030986553 |
| | - type: main_score |
| | value: 55.79838399384378 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: default |
| | name: MTEB JDReview (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/JDReview-classification |
| | metrics: |
| | - type: accuracy |
| | value: 89.11819887429644 |
| | - type: accuracy_stderr |
| | value: 1.5149440328845287 |
| | - type: ap |
| | value: 60.17445086411222 |
| | - type: ap_stderr |
| | value: 3.4864563160430384 |
| | - type: f1 |
| | value: 84.14324891240739 |
| | - type: f1_stderr |
| | value: 1.804154595730216 |
| | - type: main_score |
| | value: 89.11819887429644 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: default |
| | name: MTEB LCQMC (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/LCQMC |
| | metrics: |
| | - type: cosine_pearson |
| | value: 80.86541640109346 |
| | - type: cosine_spearman |
| | value: 79.60409318173409 |
| | - type: manhattan_pearson |
| | value: 81.12725142112909 |
| | - type: manhattan_spearman |
| | value: 79.61120096401483 |
| | - type: euclidean_pearson |
| | value: 81.1558178459699 |
| | - type: euclidean_spearman |
| | value: 79.63206760369867 |
| | - type: main_score |
| | value: 79.60409318173409 |
| | task: |
| | type: STS |
| | - dataset: |
| | config: default |
| | name: MTEB MMarcoReranking (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/Mmarco-reranking |
| | metrics: |
| | - type: map |
| | value: 30.290346963620866 |
| | - type: mrr |
| | value: 29.661507936507935 |
| | - type: main_score |
| | value: 30.290346963620866 |
| | task: |
| | type: Reranking |
| | - dataset: |
| | config: default |
| | name: MTEB MMarcoRetrieval (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/MMarcoRetrieval |
| | metrics: |
| | - type: map_at_1 |
| | value: 67.801 |
| | - type: map_at_10 |
| | value: 76.771 |
| | - type: map_at_100 |
| | value: 77.08 |
| | - type: map_at_1000 |
| | value: 77.091 |
| | - type: map_at_20 |
| | value: 76.982 |
| | - type: map_at_3 |
| | value: 75.035 |
| | - type: map_at_5 |
| | value: 76.171 |
| | - type: mrr_at_1 |
| | value: 70.057 |
| | - type: mrr_at_10 |
| | value: 77.387 |
| | - type: mrr_at_100 |
| | value: 77.65100000000001 |
| | - type: mrr_at_1000 |
| | value: 77.661 |
| | - type: mrr_at_20 |
| | value: 77.566 |
| | - type: mrr_at_3 |
| | value: 75.90299999999999 |
| | - type: mrr_at_5 |
| | value: 76.848 |
| | - type: ndcg_at_1 |
| | value: 70.057 |
| | - type: ndcg_at_10 |
| | value: 80.37100000000001 |
| | - type: ndcg_at_100 |
| | value: 81.71300000000001 |
| | - type: ndcg_at_1000 |
| | value: 81.982 |
| | - type: ndcg_at_20 |
| | value: 81.074 |
| | - type: ndcg_at_3 |
| | value: 77.12 |
| | - type: ndcg_at_5 |
| | value: 79.00500000000001 |
| | - type: precision_at_1 |
| | value: 70.057 |
| | - type: precision_at_10 |
| | value: 9.643 |
| | - type: precision_at_100 |
| | value: 1.031 |
| | - type: precision_at_1000 |
| | value: 0.105 |
| | - type: precision_at_20 |
| | value: 4.973000000000001 |
| | - type: precision_at_3 |
| | value: 28.959000000000003 |
| | - type: precision_at_5 |
| | value: 18.384 |
| | - type: recall_at_1 |
| | value: 67.801 |
| | - type: recall_at_10 |
| | value: 90.821 |
| | - type: recall_at_100 |
| | value: 96.809 |
| | - type: recall_at_1000 |
| | value: 98.87899999999999 |
| | - type: recall_at_20 |
| | value: 93.49300000000001 |
| | - type: recall_at_3 |
| | value: 82.26 |
| | - type: recall_at_5 |
| | value: 86.725 |
| | - type: main_score |
| | value: 80.37100000000001 |
| | task: |
| | type: Retrieval |
| | - dataset: |
| | config: zh-CN |
| | name: MTEB MassiveIntentClassification (zh-CN) |
| | revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
| | split: test |
| | type: mteb/amazon_massive_intent |
| | metrics: |
| | - type: accuracy |
| | value: 81.4862138533961 |
| | - type: accuracy_stderr |
| | value: 1.024666951162929 |
| | - type: f1 |
| | value: 78.57865898474617 |
| | - type: f1_stderr |
| | value: 1.1662766217911715 |
| | - type: main_score |
| | value: 81.4862138533961 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: zh-CN |
| | name: MTEB MassiveScenarioClassification (zh-CN) |
| | revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
| | split: test |
| | type: mteb/amazon_massive_scenario |
| | metrics: |
| | - type: accuracy |
| | value: 86.43913920645595 |
| | - type: accuracy_stderr |
| | value: 0.6198364524624383 |
| | - type: f1 |
| | value: 85.45514450914429 |
| | - type: f1_stderr |
| | value: 0.775295463718716 |
| | - type: main_score |
| | value: 86.43913920645595 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: default |
| | name: MTEB MedicalRetrieval (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/MedicalRetrieval |
| | metrics: |
| | - type: map_at_1 |
| | value: 60.0 |
| | - type: map_at_10 |
| | value: 66.673 |
| | - type: map_at_100 |
| | value: 67.239 |
| | - type: map_at_1000 |
| | value: 67.26299999999999 |
| | - type: map_at_20 |
| | value: 67.01899999999999 |
| | - type: map_at_3 |
| | value: 65.333 |
| | - type: map_at_5 |
| | value: 66.063 |
| | - type: mrr_at_1 |
| | value: 60.099999999999994 |
| | - type: mrr_at_10 |
| | value: 66.739 |
| | - type: mrr_at_100 |
| | value: 67.306 |
| | - type: mrr_at_1000 |
| | value: 67.33 |
| | - type: mrr_at_20 |
| | value: 67.086 |
| | - type: mrr_at_3 |
| | value: 65.4 |
| | - type: mrr_at_5 |
| | value: 66.13 |
| | - type: ndcg_at_1 |
| | value: 60.0 |
| | - type: ndcg_at_10 |
| | value: 69.786 |
| | - type: ndcg_at_100 |
| | value: 72.693 |
| | - type: ndcg_at_1000 |
| | value: 73.373 |
| | - type: ndcg_at_20 |
| | value: 71.032 |
| | - type: ndcg_at_3 |
| | value: 67.024 |
| | - type: ndcg_at_5 |
| | value: 68.34 |
| | - type: precision_at_1 |
| | value: 60.0 |
| | - type: precision_at_10 |
| | value: 7.95 |
| | - type: precision_at_100 |
| | value: 0.935 |
| | - type: precision_at_1000 |
| | value: 0.099 |
| | - type: precision_at_20 |
| | value: 4.22 |
| | - type: precision_at_3 |
| | value: 23.967 |
| | - type: precision_at_5 |
| | value: 15.02 |
| | - type: recall_at_1 |
| | value: 60.0 |
| | - type: recall_at_10 |
| | value: 79.5 |
| | - type: recall_at_100 |
| | value: 93.5 |
| | - type: recall_at_1000 |
| | value: 98.9 |
| | - type: recall_at_20 |
| | value: 84.39999999999999 |
| | - type: recall_at_3 |
| | value: 71.89999999999999 |
| | - type: recall_at_5 |
| | value: 75.1 |
| | - type: main_score |
| | value: 69.786 |
| | task: |
| | type: Retrieval |
| | - dataset: |
| | config: default |
| | name: MTEB MultilingualSentiment (default) |
| | revision: latest2023 |
| | split: validation |
| | type: C-MTEB/MultilingualSentiment-classification |
| | metrics: |
| | - type: accuracy |
| | value: 81.19666666666667 |
| | - type: accuracy_stderr |
| | value: 0.6507175526550155 |
| | - type: f1 |
| | value: 81.3717120301294 |
| | - type: f1_stderr |
| | value: 0.629161893845245 |
| | - type: main_score |
| | value: 81.19666666666667 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: default |
| | name: MTEB Ocnli (default) |
| | revision: latest2023 |
| | split: validation |
| | type: C-MTEB/OCNLI |
| | metrics: |
| | - type: cos_sim_accuracy |
| | value: 89.92961559285327 |
| | - type: cos_sim_accuracy_threshold |
| | value: 95.72410743295985 |
| | - type: cos_sim_ap |
| | value: 94.04072585697942 |
| | - type: cos_sim_f1 |
| | value: 90.6060606060606 |
| | - type: cos_sim_f1_threshold |
| | value: 95.71085030120679 |
| | - type: cos_sim_precision |
| | value: 86.83446272991287 |
| | - type: cos_sim_recall |
| | value: 94.72016895459345 |
| | - type: dot_accuracy |
| | value: 84.29886302111532 |
| | - type: dot_accuracy_threshold |
| | value: 58.39834018444028 |
| | - type: dot_ap |
| | value: 90.71149047430606 |
| | - type: dot_f1 |
| | value: 84.83263598326361 |
| | - type: dot_f1_threshold |
| | value: 58.39834018444028 |
| | - type: dot_precision |
| | value: 84.04145077720207 |
| | - type: dot_recall |
| | value: 85.6388595564942 |
| | - type: euclidean_accuracy |
| | value: 89.87547374120194 |
| | - type: euclidean_accuracy_threshold |
| | value: 22.827705768877962 |
| | - type: euclidean_ap |
| | value: 93.87312138426815 |
| | - type: euclidean_f1 |
| | value: 90.5982905982906 |
| | - type: euclidean_f1_threshold |
| | value: 22.940558234199905 |
| | - type: euclidean_precision |
| | value: 86.468330134357 |
| | - type: euclidean_recall |
| | value: 95.14255543822597 |
| | - type: manhattan_accuracy |
| | value: 89.44233892799134 |
| | - type: manhattan_accuracy_threshold |
| | value: 544.6256417358975 |
| | - type: manhattan_ap |
| | value: 93.8313800715528 |
| | - type: manhattan_f1 |
| | value: 90.16641452344932 |
| | - type: manhattan_f1_threshold |
| | value: 544.6256417358975 |
| | - type: manhattan_precision |
| | value: 86.2934362934363 |
| | - type: manhattan_recall |
| | value: 94.40337909186906 |
| | - type: max_accuracy |
| | value: 89.92961559285327 |
| | - type: max_ap |
| | value: 94.04072585697942 |
| | - type: max_f1 |
| | value: 90.6060606060606 |
| | task: |
| | type: PairClassification |
| | - dataset: |
| | config: default |
| | name: MTEB OnlineShopping (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/OnlineShopping-classification |
| | metrics: |
| | - type: accuracy |
| | value: 93.83999999999999 |
| | - type: accuracy_stderr |
| | value: 0.4521061822182879 |
| | - type: ap |
| | value: 92.19373645628713 |
| | - type: ap_stderr |
| | value: 0.2927396159644918 |
| | - type: f1 |
| | value: 93.83158946571204 |
| | - type: f1_stderr |
| | value: 0.4472553159438725 |
| | - type: main_score |
| | value: 93.83999999999999 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: default |
| | name: MTEB PAWSX (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/PAWSX |
| | metrics: |
| | - type: cosine_pearson |
| | value: 50.30680596662101 |
| | - type: cosine_spearman |
| | value: 52.41534063346883 |
| | - type: manhattan_pearson |
| | value: 51.81137421589127 |
| | - type: manhattan_spearman |
| | value: 52.40332176267904 |
| | - type: euclidean_pearson |
| | value: 51.842454511431235 |
| | - type: euclidean_spearman |
| | value: 52.4062829337432 |
| | - type: main_score |
| | value: 52.41534063346883 |
| | task: |
| | type: STS |
| | - dataset: |
| | config: default |
| | name: MTEB QBQTC (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/QBQTC |
| | metrics: |
| | - type: cosine_pearson |
| | value: 58.31070289198933 |
| | - type: cosine_spearman |
| | value: 57.966010447080684 |
| | - type: manhattan_pearson |
| | value: 54.99874211888254 |
| | - type: manhattan_spearman |
| | value: 57.796012247889195 |
| | - type: euclidean_pearson |
| | value: 55.138798573277455 |
| | - type: euclidean_spearman |
| | value: 57.95150876116391 |
| | - type: main_score |
| | value: 57.966010447080684 |
| | task: |
| | type: STS |
| | - dataset: |
| | config: zh |
| | name: MTEB STS22 (zh) |
| | revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
| | split: test |
| | type: mteb/sts22-crosslingual-sts |
| | metrics: |
| | - type: cosine_pearson |
| | value: 75.26028549682404 |
| | - type: cosine_spearman |
| | value: 73.9002967678025 |
| | - type: manhattan_pearson |
| | value: 73.47220514464013 |
| | - type: manhattan_spearman |
| | value: 73.74093326288234 |
| | - type: euclidean_pearson |
| | value: 73.59040445366989 |
| | - type: euclidean_spearman |
| | value: 73.9002967678025 |
| | - type: main_score |
| | value: 73.9002967678025 |
| | task: |
| | type: STS |
| | - dataset: |
| | config: default |
| | name: MTEB STSB (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/STSB |
| | metrics: |
| | - type: cosine_pearson |
| | value: 78.17081373176123 |
| | - type: cosine_spearman |
| | value: 78.61566426272397 |
| | - type: manhattan_pearson |
| | value: 77.66643088697434 |
| | - type: manhattan_spearman |
| | value: 78.94692354474782 |
| | - type: euclidean_pearson |
| | value: 77.69471041307843 |
| | - type: euclidean_spearman |
| | value: 78.92513847741967 |
| | - type: main_score |
| | value: 78.61566426272397 |
| | task: |
| | type: STS |
| | - dataset: |
| | config: default |
| | name: MTEB T2Reranking (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/T2Reranking |
| | metrics: |
| | - type: map |
| | value: 68.13018101639273 |
| | - type: mrr |
| | value: 79.13973922902494 |
| | - type: main_score |
| | value: 68.13018101639273 |
| | task: |
| | type: Reranking |
| | - dataset: |
| | config: default |
| | name: MTEB T2Retrieval (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/T2Retrieval |
| | metrics: |
| | - type: map_at_1 |
| | value: 28.591 |
| | - type: map_at_10 |
| | value: 80.979 |
| | - type: map_at_100 |
| | value: 84.411 |
| | - type: map_at_1000 |
| | value: 84.458 |
| | - type: map_at_20 |
| | value: 83.68100000000001 |
| | - type: map_at_3 |
| | value: 56.967999999999996 |
| | - type: map_at_5 |
| | value: 70.098 |
| | - type: mrr_at_1 |
| | value: 92.12700000000001 |
| | - type: mrr_at_10 |
| | value: 94.094 |
| | - type: mrr_at_100 |
| | value: 94.161 |
| | - type: mrr_at_1000 |
| | value: 94.164 |
| | - type: mrr_at_20 |
| | value: 94.14 |
| | - type: mrr_at_3 |
| | value: 93.753 |
| | - type: mrr_at_5 |
| | value: 93.98100000000001 |
| | - type: ndcg_at_1 |
| | value: 92.12700000000001 |
| | - type: ndcg_at_10 |
| | value: 87.586 |
| | - type: ndcg_at_100 |
| | value: 90.58500000000001 |
| | - type: ndcg_at_1000 |
| | value: 91.05 |
| | - type: ndcg_at_20 |
| | value: 89.132 |
| | - type: ndcg_at_3 |
| | value: 88.86800000000001 |
| | - type: ndcg_at_5 |
| | value: 87.673 |
| | - type: precision_at_1 |
| | value: 92.12700000000001 |
| | - type: precision_at_10 |
| | value: 43.35 |
| | - type: precision_at_100 |
| | value: 5.06 |
| | - type: precision_at_1000 |
| | value: 0.517 |
| | - type: precision_at_20 |
| | value: 23.895 |
| | - type: precision_at_3 |
| | value: 77.664 |
| | - type: precision_at_5 |
| | value: 65.231 |
| | - type: recall_at_1 |
| | value: 28.591 |
| | - type: recall_at_10 |
| | value: 86.342 |
| | - type: recall_at_100 |
| | value: 96.274 |
| | - type: recall_at_1000 |
| | value: 98.666 |
| | - type: recall_at_20 |
| | value: 91.741 |
| | - type: recall_at_3 |
| | value: 58.386 |
| | - type: recall_at_5 |
| | value: 72.942 |
| | - type: main_score |
| | value: 87.586 |
| | task: |
| | type: Retrieval |
| | - dataset: |
| | config: default |
| | name: MTEB TNews (default) |
| | revision: latest2023 |
| | split: validation |
| | type: C-MTEB/TNews-classification |
| | metrics: |
| | - type: accuracy |
| | value: 58.057 |
| | - type: accuracy_stderr |
| | value: 0.4056365368159032 |
| | - type: f1 |
| | value: 56.16542257610506 |
| | - type: f1_stderr |
| | value: 0.49560443919264746 |
| | - type: main_score |
| | value: 58.057 |
| | task: |
| | type: Classification |
| | - dataset: |
| | config: default |
| | name: MTEB ThuNewsClusteringP2P (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/ThuNewsClusteringP2P |
| | metrics: |
| | - type: v_measure |
| | value: 83.43086890900754 |
| | - type: v_measure_std |
| | value: 1.3242733220406704 |
| | - type: main_score |
| | value: 83.43086890900754 |
| | task: |
| | type: Clustering |
| | - dataset: |
| | config: default |
| | name: MTEB ThuNewsClusteringS2S (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/ThuNewsClusteringS2S |
| | metrics: |
| | - type: v_measure |
| | value: 80.17922689954183 |
| | - type: v_measure_std |
| | value: 2.1732975942130612 |
| | - type: main_score |
| | value: 80.17922689954183 |
| | task: |
| | type: Clustering |
| | - dataset: |
| | config: default |
| | name: MTEB VideoRetrieval (default) |
| | revision: latest2023 |
| | split: dev |
| | type: C-MTEB/VideoRetrieval |
| | metrics: |
| | - type: map_at_1 |
| | value: 68.60000000000001 |
| | - type: map_at_10 |
| | value: 77.518 |
| | - type: map_at_100 |
| | value: 77.815 |
| | - type: map_at_1000 |
| | value: 77.82 |
| | - type: map_at_20 |
| | value: 77.73299999999999 |
| | - type: map_at_3 |
| | value: 76.167 |
| | - type: map_at_5 |
| | value: 76.932 |
| | - type: mrr_at_1 |
| | value: 68.60000000000001 |
| | - type: mrr_at_10 |
| | value: 77.518 |
| | - type: mrr_at_100 |
| | value: 77.815 |
| | - type: mrr_at_1000 |
| | value: 77.82 |
| | - type: mrr_at_20 |
| | value: 77.73299999999999 |
| | - type: mrr_at_3 |
| | value: 76.167 |
| | - type: mrr_at_5 |
| | value: 76.932 |
| | - type: ndcg_at_1 |
| | value: 68.60000000000001 |
| | - type: ndcg_at_10 |
| | value: 81.339 |
| | - type: ndcg_at_100 |
| | value: 82.646 |
| | - type: ndcg_at_1000 |
| | value: 82.76599999999999 |
| | - type: ndcg_at_20 |
| | value: 82.107 |
| | - type: ndcg_at_3 |
| | value: 78.569 |
| | - type: ndcg_at_5 |
| | value: 79.937 |
| | - type: precision_at_1 |
| | value: 68.60000000000001 |
| | - type: precision_at_10 |
| | value: 9.31 |
| | - type: precision_at_100 |
| | value: 0.989 |
| | - type: precision_at_1000 |
| | value: 0.1 |
| | - type: precision_at_20 |
| | value: 4.805000000000001 |
| | - type: precision_at_3 |
| | value: 28.499999999999996 |
| | - type: precision_at_5 |
| | value: 17.76 |
| | - type: recall_at_1 |
| | value: 68.60000000000001 |
| | - type: recall_at_10 |
| | value: 93.10000000000001 |
| | - type: recall_at_100 |
| | value: 98.9 |
| | - type: recall_at_1000 |
| | value: 99.8 |
| | - type: recall_at_20 |
| | value: 96.1 |
| | - type: recall_at_3 |
| | value: 85.5 |
| | - type: recall_at_5 |
| | value: 88.8 |
| | - type: main_score |
| | value: 81.339 |
| | task: |
| | type: Retrieval |
| | - dataset: |
| | config: default |
| | name: MTEB Waimai (default) |
| | revision: latest2023 |
| | split: test |
| | type: C-MTEB/waimai-classification |
| | metrics: |
| | - type: accuracy |
| | value: 90.63000000000001 |
| | - type: accuracy_stderr |
| | value: 0.49203658400570216 |
| | - type: ap |
| | value: 77.93466200571231 |
| | - type: ap_stderr |
| | value: 1.2006502477223735 |
| | - type: f1 |
| | value: 89.36361097500829 |
| | - type: f1_stderr |
| | value: 0.43660966359249054 |
| | - type: main_score |
| | value: 90.63000000000001 |
| | task: |
| | type: Classification |
| | tags: |
| | - mteb |
| | --- |
| | |
| | # quark-llm-embedding-8B |
| |
|
| | - Chinese Text Embedding Model developed by Alibaba Quark-LLM Team. Details will be published later. |
