diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/AngryTweetsClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/AngryTweetsClassification.json new file mode 100644 index 0000000000..4a59352bbb --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/AngryTweetsClassification.json @@ -0,0 +1,140 @@ +{ + "dataset_revision": "20b0e6081892e78179356fada741b7afa381443d", + "task_name": "AngryTweetsClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.65234, + "f1": 0.646658, + "f1_weighted": 0.652289, + "precision": 0.64586, + "precision_weighted": 0.652426, + "recall": 0.647671, + "recall_weighted": 0.65234, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.630372, + "f1": 0.62591, + "f1_weighted": 0.631856, + "precision": 0.626206, + "precision_weighted": 0.636705, + "recall": 0.629388, + "recall_weighted": 0.630372, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.622732, + "f1": 0.607674, + "f1_weighted": 0.612095, + "precision": 0.621059, + "precision_weighted": 0.620348, + "recall": 0.611781, + "recall_weighted": 0.622732, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.602674, + "f1": 0.586887, + "f1_weighted": 0.58976, + "precision": 0.600826, + "precision_weighted": 0.606676, + "recall": 0.602285, + "recall_weighted": 0.602674, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.633238, + "f1": 0.622984, + "f1_weighted": 0.628778, + "precision": 0.623645, + "precision_weighted": 0.628259, + "recall": 0.625949, + "recall_weighted": 0.633238, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.669532, + "f1": 0.652856, + "f1_weighted": 0.662098, + "precision": 0.664424, + "precision_weighted": 0.665637, + "recall": 0.652453, + "recall_weighted": 0.669532, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.617956, + "f1": 0.611358, + "f1_weighted": 0.619668, + "precision": 0.611766, + "precision_weighted": 0.623168, + "recall": 0.61297, + "recall_weighted": 0.617956, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.626552, + "f1": 0.617906, + "f1_weighted": 0.621257, + "precision": 0.618909, + "precision_weighted": 0.624355, + "recall": 0.625337, + "recall_weighted": 0.626552, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.624642, + "f1": 0.619009, + "f1_weighted": 0.62718, + "precision": 0.620339, + "precision_weighted": 0.63122, + "recall": 0.619127, + "recall_weighted": 0.624642, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.659981, + "f1": 0.644923, + "f1_weighted": 0.651536, + "precision": 0.651914, + "precision_weighted": 0.658052, + "recall": 0.651982, + "recall_weighted": 0.659981, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.634002, + "f1": 0.623617, + "f1_weighted": 0.629652, + "precision": 0.628495, + "precision_weighted": 0.634685, + "recall": 0.627894, + "recall_weighted": 0.634002, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.634002, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 14.389760732650757, + "kg_co2_emissions": null, + "date": 1776145298.035166 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/BornholmBitextMining.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/BornholmBitextMining.json new file mode 100644 index 0000000000..c906d6f67d --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/BornholmBitextMining.json @@ -0,0 +1,23 @@ +{ + "dataset_revision": "5b02048bd75e79275aa91a1fce6cdfd3f4a391cb", + "task_name": "BornholmBitextMining", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "precision": 0.337652, + "recall": 0.436, + "f1": 0.360316, + "accuracy": 0.436, + "main_score": 0.360316, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 7.278884649276733, + "kg_co2_emissions": null, + "date": null +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DKHateClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DKHateClassification.json new file mode 100644 index 0000000000..04007a9fe5 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DKHateClassification.json @@ -0,0 +1,140 @@ +{ + "dataset_revision": "59d12749a3c91a186063c7d729ec392fda94681c", + "task_name": "DKHateClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.677812, + "f1": 0.600142, + "f1_weighted": 0.732448, + "precision": 0.630987, + "precision_weighted": 0.899955, + "recall": 0.795054, + "recall_weighted": 0.677812, + "ap": 0.948129, + "ap_weighted": 0.948129 + }, + { + "accuracy": 0.74772, + "f1": 0.64658, + "f1_weighted": 0.788521, + "precision": 0.642425, + "precision_weighted": 0.89012, + "recall": 0.793149, + "recall_weighted": 0.74772, + "ap": 0.946424, + "ap_weighted": 0.946424 + }, + { + "accuracy": 0.808511, + "f1": 0.685024, + "f1_weighted": 0.833088, + "precision": 0.66066, + "precision_weighted": 0.881988, + "recall": 0.775576, + "recall_weighted": 0.808511, + "ap": 0.941006, + "ap_weighted": 0.941006 + }, + { + "accuracy": 0.753799, + "f1": 0.645233, + "f1_weighted": 0.792573, + "precision": 0.637723, + "precision_weighted": 0.882773, + "recall": 0.775703, + "recall_weighted": 0.753799, + "ap": 0.941631, + "ap_weighted": 0.941631 + }, + { + "accuracy": 0.765957, + "f1": 0.669096, + "f1_weighted": 0.803505, + "precision": 0.659049, + "precision_weighted": 0.901309, + "recall": 0.824483, + "recall_weighted": 0.765957, + "ap": 0.954777, + "ap_weighted": 0.954777 + }, + { + "accuracy": 0.674772, + "f1": 0.574025, + "f1_weighted": 0.729553, + "precision": 0.596218, + "precision_weighted": 0.86201, + "recall": 0.709646, + "recall_weighted": 0.674772, + "ap": 0.925033, + "ap_weighted": 0.925033 + }, + { + "accuracy": 0.708207, + "f1": 0.600203, + "f1_weighted": 0.756209, + "precision": 0.608894, + "precision_weighted": 0.867107, + "recall": 0.728743, + "recall_weighted": 0.708207, + "ap": 0.929703, + "ap_weighted": 0.929703 + }, + { + "accuracy": 0.741641, + "f1": 0.644469, + "f1_weighted": 0.784013, + "precision": 0.643435, + "precision_weighted": 0.893506, + "recall": 0.800136, + "recall_weighted": 0.741641, + "ap": 0.94843, + "ap_weighted": 0.94843 + }, + { + "accuracy": 0.799392, + "f1": 0.683654, + "f1_weighted": 0.827309, + "precision": 0.661025, + "precision_weighted": 0.88736, + "recall": 0.791286, + "recall_weighted": 0.799392, + "ap": 0.945257, + "ap_weighted": 0.945257 + }, + { + "accuracy": 0.693009, + "f1": 0.56186, + "f1_weighted": 0.741826, + "precision": 0.57265, + "precision_weighted": 0.835736, + "recall": 0.64685, + "recall_weighted": 0.693009, + "ap": 0.909165, + "ap_weighted": 0.909165 + } + ], + "accuracy": 0.737082, + "f1": 0.631029, + "f1_weighted": 0.778904, + "precision": 0.631307, + "precision_weighted": 0.880186, + "recall": 0.764062, + "recall_weighted": 0.737082, + "ap": 0.938955, + "ap_weighted": 0.938955, + "main_score": 0.737082, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 12.273958921432495, + "kg_co2_emissions": null, + "date": 1776145382.034961 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DalajClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DalajClassification.json new file mode 100644 index 0000000000..05035bfd1a --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DalajClassification.json @@ -0,0 +1,140 @@ +{ + "dataset_revision": "2a460943e43451e4c9aed3c0adafc83ccd6e3ee1", + "task_name": "DalajClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.496622, + "f1": 0.495439, + "f1_weighted": 0.495439, + "precision": 0.49659, + "precision_weighted": 0.49659, + "recall": 0.496622, + "recall_weighted": 0.496622, + "ap": 0.498321, + "ap_weighted": 0.498321 + }, + { + "accuracy": 0.504505, + "f1": 0.502232, + "f1_weighted": 0.502232, + "precision": 0.504588, + "precision_weighted": 0.504588, + "recall": 0.504505, + "recall_weighted": 0.504505, + "ap": 0.50227, + "ap_weighted": 0.50227 + }, + { + "accuracy": 0.502252, + "f1": 0.495859, + "f1_weighted": 0.495859, + "precision": 0.502373, + "precision_weighted": 0.502373, + "recall": 0.502252, + "recall_weighted": 0.502252, + "ap": 0.501133, + "ap_weighted": 0.501133 + }, + { + "accuracy": 0.512387, + "f1": 0.50407, + "f1_weighted": 0.50407, + "precision": 0.513278, + "precision_weighted": 0.513278, + "recall": 0.512387, + "recall_weighted": 0.512387, + "ap": 0.506401, + "ap_weighted": 0.506401 + }, + { + "accuracy": 0.503378, + "f1": 0.49687, + "f1_weighted": 0.49687, + "precision": 0.503563, + "precision_weighted": 0.503563, + "recall": 0.503378, + "recall_weighted": 0.503378, + "ap": 0.501698, + "ap_weighted": 0.501698 + }, + { + "accuracy": 0.503378, + "f1": 0.5021, + "f1_weighted": 0.5021, + "precision": 0.503413, + "precision_weighted": 0.503413, + "recall": 0.503378, + "recall_weighted": 0.503378, + "ap": 0.501702, + "ap_weighted": 0.501702 + }, + { + "accuracy": 0.492117, + "f1": 0.491497, + "f1_weighted": 0.491497, + "precision": 0.492079, + "precision_weighted": 0.492079, + "recall": 0.492117, + "recall_weighted": 0.492117, + "ap": 0.496117, + "ap_weighted": 0.496117 + }, + { + "accuracy": 0.504505, + "f1": 0.504464, + "f1_weighted": 0.504464, + "precision": 0.504506, + "precision_weighted": 0.504506, + "recall": 0.504505, + "recall_weighted": 0.504505, + "ap": 0.502273, + "ap_weighted": 0.502273 + }, + { + "accuracy": 0.509009, + "f1": 0.508807, + "f1_weighted": 0.508807, + "precision": 0.509024, + "precision_weighted": 0.509024, + "recall": 0.509009, + "recall_weighted": 0.509009, + "ap": 0.504589, + "ap_weighted": 0.504589 + }, + { + "accuracy": 0.506757, + "f1": 0.506634, + "f1_weighted": 0.506634, + "precision": 0.506763, + "precision_weighted": 0.506763, + "recall": 0.506757, + "recall_weighted": 0.506757, + "ap": 0.503426, + "ap_weighted": 0.503426 + } + ], + "accuracy": 0.503491, + "f1": 0.500797, + "f1_weighted": 0.500797, + "precision": 0.503618, + "precision_weighted": 0.503618, + "recall": 0.503491, + "recall_weighted": 0.503491, + "ap": 0.501793, + "ap_weighted": 0.501793, + "main_score": 0.503491, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 10.061943292617798, + "kg_co2_emissions": null, + "date": 1776145367.354351 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DanFeverRetrieval.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DanFeverRetrieval.json new file mode 100644 index 0000000000..643bdfa2f6 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DanFeverRetrieval.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "3b17b754ed5bf356582b93ec11d1c72469f7bb9c", + "task_name": "DanFeverRetrieval", + "mteb_version": "2.12.16", + "scores": { + "train": [ + { + "ndcg_at_1": 0.29327, + "ndcg_at_3": 0.37395, + "ndcg_at_5": 0.38325, + "ndcg_at_10": 0.38854, + "ndcg_at_20": 0.39123, + "ndcg_at_100": 0.39263, + "ndcg_at_1000": 0.39278, + "map_at_1": 0.29311, + "map_at_3": 0.3547, + "map_at_5": 0.35989, + "map_at_10": 0.36211, + "map_at_20": 0.36287, + "map_at_100": 0.36309, + "map_at_1000": 0.3631, + "recall_at_1": 0.29311, + "recall_at_3": 0.42931, + "recall_at_5": 0.45175, + "recall_at_10": 0.46791, + "recall_at_20": 0.47842, + "recall_at_100": 0.48549, + "recall_at_1000": 0.48658, + "accuracy": 0.29311, + "precision_at_1": 0.29327, + "precision_at_3": 0.14321, + "precision_at_5": 0.09041, + "precision_at_10": 0.04682, + "precision_at_20": 0.02394, + "precision_at_100": 0.00486, + "precision_at_1000": 0.00049, + "mrr_at_1": 0.293268, + "mrr_at_3": 0.354804, + "mrr_at_5": 0.35999, + "mrr_at_10": 0.362213, + "mrr_at_20": 0.36295, + "mrr_at_100": 0.363175, + "mrr_at_1000": 0.363181, + "nauc_ndcg_at_1_max": 0.352189, + "nauc_ndcg_at_1_std": -0.025439, + "nauc_ndcg_at_1_diff1": 0.36703, + "nauc_ndcg_at_3_max": 0.337893, + "nauc_ndcg_at_3_std": 0.005419, + "nauc_ndcg_at_3_diff1": 0.277502, + "nauc_ndcg_at_5_max": 0.328105, + "nauc_ndcg_at_5_std": 0.015649, + "nauc_ndcg_at_5_diff1": 0.268126, + "nauc_ndcg_at_10_max": 0.31826, + "nauc_ndcg_at_10_std": 0.019036, + "nauc_ndcg_at_10_diff1": 0.26201, + "nauc_ndcg_at_20_max": 0.312988, + "nauc_ndcg_at_20_std": 0.021511, + "nauc_ndcg_at_20_diff1": 0.258052, + "nauc_ndcg_at_100_max": 0.31018, + "nauc_ndcg_at_100_std": 0.023184, + "nauc_ndcg_at_100_diff1": 0.256161, + "nauc_ndcg_at_1000_max": 0.309792, + "nauc_ndcg_at_1000_std": 0.022928, + "nauc_ndcg_at_1000_diff1": 0.255973, + "nauc_map_at_1_max": 0.352435, + "nauc_map_at_1_std": -0.026245, + "nauc_map_at_1_diff1": 0.36706, + "nauc_map_at_3_max": 0.3417, + "nauc_map_at_3_std": -0.002959, + "nauc_map_at_3_diff1": 0.298275, + "nauc_map_at_5_max": 0.336295, + "nauc_map_at_5_std": 0.002757, + "nauc_map_at_5_diff1": 0.293116, + "nauc_map_at_10_max": 0.332161, + "nauc_map_at_10_std": 0.004199, + "nauc_map_at_10_diff1": 0.290586, + "nauc_map_at_20_max": 0.330689, + "nauc_map_at_20_std": 0.004996, + "nauc_map_at_20_diff1": 0.289476, + "nauc_map_at_100_max": 0.330243, + "nauc_map_at_100_std": 0.005307, + "nauc_map_at_100_diff1": 0.289181, + "nauc_map_at_1000_max": 0.330225, + "nauc_map_at_1000_std": 0.005294, + "nauc_map_at_1000_diff1": 0.289174, + "nauc_recall_at_1_max": 0.352435, + "nauc_recall_at_1_std": -0.026245, + "nauc_recall_at_1_diff1": 0.36706, + "nauc_recall_at_3_max": 0.32718, + "nauc_recall_at_3_std": 0.028904, + "nauc_recall_at_3_diff1": 0.21936, + "nauc_recall_at_5_max": 0.303446, + "nauc_recall_at_5_std": 0.053495, + "nauc_recall_at_5_diff1": 0.196311, + "nauc_recall_at_10_max": 0.273475, + "nauc_recall_at_10_std": 0.063815, + "nauc_recall_at_10_diff1": 0.177252, + "nauc_recall_at_20_max": 0.252741, + "nauc_recall_at_20_std": 0.072984, + "nauc_recall_at_20_diff1": 0.16153, + "nauc_recall_at_100_max": 0.238398, + "nauc_recall_at_100_std": 0.080909, + "nauc_recall_at_100_diff1": 0.151682, + "nauc_recall_at_1000_max": 0.235486, + "nauc_recall_at_1000_std": 0.079038, + "nauc_recall_at_1000_diff1": 0.150198, + "nauc_precision_at_1_max": 0.352189, + "nauc_precision_at_1_std": -0.025439, + "nauc_precision_at_1_diff1": 0.36703, + "nauc_precision_at_3_max": 0.326616, + "nauc_precision_at_3_std": 0.030404, + "nauc_precision_at_3_diff1": 0.2193, + "nauc_precision_at_5_max": 0.302895, + "nauc_precision_at_5_std": 0.054988, + "nauc_precision_at_5_diff1": 0.196267, + "nauc_precision_at_10_max": 0.272942, + "nauc_precision_at_10_std": 0.065309, + "nauc_precision_at_10_diff1": 0.177221, + "nauc_precision_at_20_max": 0.251536, + "nauc_precision_at_20_std": 0.07513, + "nauc_precision_at_20_diff1": 0.160838, + "nauc_precision_at_100_max": 0.237223, + "nauc_precision_at_100_std": 0.083042, + "nauc_precision_at_100_diff1": 0.151011, + "nauc_precision_at_1000_max": 0.234317, + "nauc_precision_at_1000_std": 0.081176, + "nauc_precision_at_1000_diff1": 0.14953, + "nauc_mrr_at_1_max": 0.352189, + "nauc_mrr_at_1_std": -0.025439, + "nauc_mrr_at_1_diff1": 0.36703, + "nauc_mrr_at_3_max": 0.341482, + "nauc_mrr_at_3_std": -0.00244, + "nauc_mrr_at_3_diff1": 0.297971, + "nauc_mrr_at_5_max": 0.336078, + "nauc_mrr_at_5_std": 0.003277, + "nauc_mrr_at_5_diff1": 0.292812, + "nauc_mrr_at_10_max": 0.331944, + "nauc_mrr_at_10_std": 0.004719, + "nauc_mrr_at_10_diff1": 0.290282, + "nauc_mrr_at_20_max": 0.33052, + "nauc_mrr_at_20_std": 0.0054, + "nauc_mrr_at_20_diff1": 0.289239, + "nauc_mrr_at_100_max": 0.330073, + "nauc_mrr_at_100_std": 0.005711, + "nauc_mrr_at_100_diff1": 0.288944, + "nauc_mrr_at_1000_max": 0.330055, + "nauc_mrr_at_1000_std": 0.005699, + "nauc_mrr_at_1000_diff1": 0.288937, + "hit_rate_at_1": 0.29327, + "hit_rate_at_3": 0.42947, + "hit_rate_at_5": 0.45191, + "hit_rate_at_10": 0.46807, + "hit_rate_at_20": 0.47842, + "hit_rate_at_100": 0.48549, + "hit_rate_at_1000": 0.48658, + "main_score": 0.38854, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 77.34244537353516, + "kg_co2_emissions": null, + "date": 1776146383.721085 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DanishPoliticalCommentsClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DanishPoliticalCommentsClassification.json new file mode 100644 index 0000000000..9bca42843a --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/DanishPoliticalCommentsClassification.json @@ -0,0 +1,140 @@ +{ + "dataset_revision": "d743dcd5abb03d5ab357757a0e83522fc6696fcd", + "task_name": "DanishPoliticalCommentsClassification", + "mteb_version": "2.12.16", + "scores": { + "train": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.469192, + "f1": 0.429662, + "f1_weighted": 0.483237, + "precision": 0.421168, + "precision_weighted": 0.540663, + "recall": 0.503452, + "recall_weighted": 0.469192, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.490008, + "f1": 0.445036, + "f1_weighted": 0.50556, + "precision": 0.433064, + "precision_weighted": 0.558839, + "recall": 0.519739, + "recall_weighted": 0.490008, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.467111, + "f1": 0.421259, + "f1_weighted": 0.483827, + "precision": 0.409605, + "precision_weighted": 0.54419, + "recall": 0.488929, + "recall_weighted": 0.467111, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.494033, + "f1": 0.448934, + "f1_weighted": 0.503776, + "precision": 0.429679, + "precision_weighted": 0.548848, + "recall": 0.533504, + "recall_weighted": 0.494033, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.481404, + "f1": 0.430478, + "f1_weighted": 0.493466, + "precision": 0.415097, + "precision_weighted": 0.538467, + "recall": 0.512375, + "recall_weighted": 0.481404, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.399944, + "f1": 0.371867, + "f1_weighted": 0.408458, + "precision": 0.36879, + "precision_weighted": 0.493675, + "recall": 0.492153, + "recall_weighted": 0.399944, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.449903, + "f1": 0.434185, + "f1_weighted": 0.459662, + "precision": 0.426631, + "precision_weighted": 0.546546, + "recall": 0.514436, + "recall_weighted": 0.449903, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.445185, + "f1": 0.413437, + "f1_weighted": 0.462021, + "precision": 0.409629, + "precision_weighted": 0.549092, + "recall": 0.511329, + "recall_weighted": 0.445185, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.455315, + "f1": 0.413992, + "f1_weighted": 0.458473, + "precision": 0.400085, + "precision_weighted": 0.512663, + "recall": 0.513996, + "recall_weighted": 0.455315, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.470303, + "f1": 0.428646, + "f1_weighted": 0.485482, + "precision": 0.420232, + "precision_weighted": 0.550852, + "recall": 0.51912, + "recall_weighted": 0.470303, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.46224, + "f1": 0.42375, + "f1_weighted": 0.474396, + "precision": 0.413398, + "precision_weighted": 0.538384, + "recall": 0.510903, + "recall_weighted": 0.46224, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.46224, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 53.2066924571991, + "kg_co2_emissions": null, + "date": 1776145353.995309 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/LccSentimentClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/LccSentimentClassification.json new file mode 100644 index 0000000000..5269dd3c2c --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/LccSentimentClassification.json @@ -0,0 +1,140 @@ +{ + "dataset_revision": "de7ba3406ee55ea2cc52a0a41408fa6aede6d3c6", + "task_name": "LccSentimentClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.666667, + "f1": 0.651434, + "f1_weighted": 0.667275, + "precision": 0.644548, + "precision_weighted": 0.699391, + "recall": 0.701862, + "recall_weighted": 0.666667, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.6, + "f1": 0.603205, + "f1_weighted": 0.596154, + "precision": 0.606231, + "precision_weighted": 0.674767, + "recall": 0.680999, + "recall_weighted": 0.6, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.62, + "f1": 0.618764, + "f1_weighted": 0.625753, + "precision": 0.632625, + "precision_weighted": 0.708774, + "recall": 0.688516, + "recall_weighted": 0.62, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.646667, + "f1": 0.639983, + "f1_weighted": 0.649101, + "precision": 0.633333, + "precision_weighted": 0.692, + "recall": 0.695518, + "recall_weighted": 0.646667, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.626667, + "f1": 0.626464, + "f1_weighted": 0.627045, + "precision": 0.613087, + "precision_weighted": 0.662712, + "recall": 0.676755, + "recall_weighted": 0.626667, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.64, + "f1": 0.629847, + "f1_weighted": 0.643508, + "precision": 0.617672, + "precision_weighted": 0.655457, + "recall": 0.650886, + "recall_weighted": 0.64, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.626667, + "f1": 0.603435, + "f1_weighted": 0.629871, + "precision": 0.603372, + "precision_weighted": 0.647722, + "recall": 0.625903, + "recall_weighted": 0.626667, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.653333, + "f1": 0.616293, + "f1_weighted": 0.651723, + "precision": 0.621724, + "precision_weighted": 0.650699, + "recall": 0.611586, + "recall_weighted": 0.653333, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.64, + "f1": 0.621587, + "f1_weighted": 0.644216, + "precision": 0.627606, + "precision_weighted": 0.683349, + "recall": 0.66549, + "recall_weighted": 0.64, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.666667, + "f1": 0.656025, + "f1_weighted": 0.670798, + "precision": 0.644569, + "precision_weighted": 0.699483, + "recall": 0.690778, + "recall_weighted": 0.666667, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.638667, + "f1": 0.626704, + "f1_weighted": 0.640544, + "precision": 0.624477, + "precision_weighted": 0.677435, + "recall": 0.668829, + "recall_weighted": 0.638667, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.638667, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 4.4757959842681885, + "kg_co2_emissions": null, + "date": 1776145388.551351 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/MassiveIntentClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/MassiveIntentClassification.json new file mode 100644 index 0000000000..a419003523 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/MassiveIntentClassification.json @@ -0,0 +1,782 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "2.12.16", + "scores": { + "validation": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.698967, + "f1": 0.658797, + "f1_weighted": 0.689277, + "precision": 0.641829, + "precision_weighted": 0.728223, + "recall": 0.751086, + "recall_weighted": 0.698967, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.715199, + "f1": 0.663182, + "f1_weighted": 0.710414, + "precision": 0.64888, + "precision_weighted": 0.755711, + "recall": 0.755822, + "recall_weighted": 0.715199, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.697491, + "f1": 0.636587, + "f1_weighted": 0.688781, + "precision": 0.621534, + "precision_weighted": 0.748061, + "recall": 0.740715, + "recall_weighted": 0.697491, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.717659, + "f1": 0.67253, + "f1_weighted": 0.709815, + "precision": 0.658975, + "precision_weighted": 0.76571, + "recall": 0.767253, + "recall_weighted": 0.717659, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.692081, + "f1": 0.6409, + "f1_weighted": 0.676018, + "precision": 0.624141, + "precision_weighted": 0.709358, + "recall": 0.741159, + "recall_weighted": 0.692081, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.694048, + "f1": 0.667737, + "f1_weighted": 0.684636, + "precision": 0.653182, + "precision_weighted": 0.748802, + "recall": 0.767314, + "recall_weighted": 0.694048, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.677324, + "f1": 0.642274, + "f1_weighted": 0.666909, + "precision": 0.627651, + "precision_weighted": 0.708838, + "recall": 0.745357, + "recall_weighted": 0.677324, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.669946, + "f1": 0.635324, + "f1_weighted": 0.660813, + "precision": 0.624306, + "precision_weighted": 0.727887, + "recall": 0.737495, + "recall_weighted": 0.669946, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.66847, + "f1": 0.640645, + "f1_weighted": 0.648083, + "precision": 0.640177, + "precision_weighted": 0.747128, + "recall": 0.747508, + "recall_weighted": 0.66847, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.708313, + "f1": 0.650918, + "f1_weighted": 0.698145, + "precision": 0.633391, + "precision_weighted": 0.749403, + "recall": 0.747393, + "recall_weighted": 0.708313, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.69395, + "f1": 0.650889, + "f1_weighted": 0.683289, + "precision": 0.637407, + "precision_weighted": 0.738912, + "recall": 0.75011, + "recall_weighted": 0.69395, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.69395, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.695032, + "f1": 0.651829, + "f1_weighted": 0.684895, + "precision": 0.636874, + "precision_weighted": 0.743707, + "recall": 0.753379, + "recall_weighted": 0.695032, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.711756, + "f1": 0.66695, + "f1_weighted": 0.706776, + "precision": 0.655723, + "precision_weighted": 0.767466, + "recall": 0.763109, + "recall_weighted": 0.711756, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.710772, + "f1": 0.655996, + "f1_weighted": 0.700718, + "precision": 0.634129, + "precision_weighted": 0.755661, + "recall": 0.755044, + "recall_weighted": 0.710772, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.712248, + "f1": 0.649667, + "f1_weighted": 0.70187, + "precision": 0.635196, + "precision_weighted": 0.763725, + "recall": 0.74966, + "recall_weighted": 0.712248, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.701426, + "f1": 0.646761, + "f1_weighted": 0.684312, + "precision": 0.630893, + "precision_weighted": 0.731892, + "recall": 0.749538, + "recall_weighted": 0.701426, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.686178, + "f1": 0.643546, + "f1_weighted": 0.674721, + "precision": 0.627609, + "precision_weighted": 0.738493, + "recall": 0.739895, + "recall_weighted": 0.686178, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.684211, + "f1": 0.635026, + "f1_weighted": 0.673729, + "precision": 0.617283, + "precision_weighted": 0.720835, + "recall": 0.733433, + "recall_weighted": 0.684211, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.687654, + "f1": 0.640142, + "f1_weighted": 0.676902, + "precision": 0.622317, + "precision_weighted": 0.740404, + "recall": 0.743406, + "recall_weighted": 0.687654, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.665519, + "f1": 0.631615, + "f1_weighted": 0.64044, + "precision": 0.632071, + "precision_weighted": 0.741313, + "recall": 0.739055, + "recall_weighted": 0.665519, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.697491, + "f1": 0.651435, + "f1_weighted": 0.689207, + "precision": 0.640546, + "precision_weighted": 0.74653, + "recall": 0.745319, + "recall_weighted": 0.697491, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.695229, + "f1": 0.647297, + "f1_weighted": 0.683357, + "precision": 0.633264, + "precision_weighted": 0.745003, + "recall": 0.747184, + "recall_weighted": 0.695229, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.695229, + "hf_subset": "nb", + "languages": [ + "nob-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.719134, + "f1": 0.683157, + "f1_weighted": 0.710721, + "precision": 0.671386, + "precision_weighted": 0.757733, + "recall": 0.7674, + "recall_weighted": 0.719134, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.735366, + "f1": 0.687739, + "f1_weighted": 0.726568, + "precision": 0.671874, + "precision_weighted": 0.764145, + "recall": 0.776283, + "recall_weighted": 0.735366, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.713232, + "f1": 0.661468, + "f1_weighted": 0.701144, + "precision": 0.637501, + "precision_weighted": 0.740906, + "recall": 0.756323, + "recall_weighted": 0.713232, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.727496, + "f1": 0.675199, + "f1_weighted": 0.716988, + "precision": 0.673027, + "precision_weighted": 0.787085, + "recall": 0.757542, + "recall_weighted": 0.727496, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.706837, + "f1": 0.656831, + "f1_weighted": 0.690972, + "precision": 0.640494, + "precision_weighted": 0.72809, + "recall": 0.751633, + "recall_weighted": 0.706837, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.702902, + "f1": 0.673747, + "f1_weighted": 0.691963, + "precision": 0.655331, + "precision_weighted": 0.742111, + "recall": 0.7649, + "recall_weighted": 0.702902, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.708805, + "f1": 0.675111, + "f1_weighted": 0.696355, + "precision": 0.651527, + "precision_weighted": 0.722699, + "recall": 0.770091, + "recall_weighted": 0.708805, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.688146, + "f1": 0.658682, + "f1_weighted": 0.681705, + "precision": 0.645475, + "precision_weighted": 0.743439, + "recall": 0.752099, + "recall_weighted": 0.688146, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.688146, + "f1": 0.664052, + "f1_weighted": 0.673337, + "precision": 0.662508, + "precision_weighted": 0.76289, + "recall": 0.765317, + "recall_weighted": 0.688146, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.730448, + "f1": 0.677616, + "f1_weighted": 0.726023, + "precision": 0.663196, + "precision_weighted": 0.770779, + "recall": 0.761468, + "recall_weighted": 0.730448, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.712051, + "f1": 0.67136, + "f1_weighted": 0.701578, + "precision": 0.657232, + "precision_weighted": 0.751988, + "recall": 0.762306, + "recall_weighted": 0.712051, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.712051, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ] + } + ], + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.705447, + "f1": 0.676364, + "f1_weighted": 0.696773, + "precision": 0.665646, + "precision_weighted": 0.767796, + "recall": 0.762384, + "recall_weighted": 0.705447, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.7115, + "f1": 0.677787, + "f1_weighted": 0.707959, + "precision": 0.660969, + "precision_weighted": 0.766123, + "recall": 0.7734, + "recall_weighted": 0.7115, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.682919, + "f1": 0.638227, + "f1_weighted": 0.677951, + "precision": 0.632212, + "precision_weighted": 0.753071, + "recall": 0.72739, + "recall_weighted": 0.682919, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.704102, + "f1": 0.666516, + "f1_weighted": 0.701917, + "precision": 0.654971, + "precision_weighted": 0.764029, + "recall": 0.747029, + "recall_weighted": 0.704102, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.691997, + "f1": 0.649125, + "f1_weighted": 0.679785, + "precision": 0.646904, + "precision_weighted": 0.767841, + "recall": 0.734249, + "recall_weighted": 0.691997, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.682246, + "f1": 0.676285, + "f1_weighted": 0.673348, + "precision": 0.666512, + "precision_weighted": 0.763516, + "recall": 0.77382, + "recall_weighted": 0.682246, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.683927, + "f1": 0.662612, + "f1_weighted": 0.67561, + "precision": 0.6462, + "precision_weighted": 0.746831, + "recall": 0.777339, + "recall_weighted": 0.683927, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.67115, + "f1": 0.642312, + "f1_weighted": 0.664035, + "precision": 0.6298, + "precision_weighted": 0.740718, + "recall": 0.741824, + "recall_weighted": 0.67115, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.659718, + "f1": 0.642383, + "f1_weighted": 0.644365, + "precision": 0.644387, + "precision_weighted": 0.745512, + "recall": 0.752245, + "recall_weighted": 0.659718, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.701748, + "f1": 0.671201, + "f1_weighted": 0.695011, + "precision": 0.660274, + "precision_weighted": 0.761232, + "recall": 0.777733, + "recall_weighted": 0.701748, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.689475, + "f1": 0.660281, + "f1_weighted": 0.681675, + "precision": 0.650787, + "precision_weighted": 0.757667, + "recall": 0.756741, + "recall_weighted": 0.689475, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.689475, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.697041, + "f1": 0.674342, + "f1_weighted": 0.694169, + "precision": 0.666634, + "precision_weighted": 0.768341, + "recall": 0.778676, + "recall_weighted": 0.697041, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.697041, + "f1": 0.676996, + "f1_weighted": 0.693604, + "precision": 0.659686, + "precision_weighted": 0.757961, + "recall": 0.766611, + "recall_weighted": 0.697041, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.695024, + "f1": 0.663371, + "f1_weighted": 0.688037, + "precision": 0.644806, + "precision_weighted": 0.75243, + "recall": 0.759849, + "recall_weighted": 0.695024, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.703093, + "f1": 0.674835, + "f1_weighted": 0.701003, + "precision": 0.661094, + "precision_weighted": 0.761135, + "recall": 0.758026, + "recall_weighted": 0.703093, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.677202, + "f1": 0.660737, + "f1_weighted": 0.663966, + "precision": 0.658424, + "precision_weighted": 0.745751, + "recall": 0.767894, + "recall_weighted": 0.677202, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.678211, + "f1": 0.665516, + "f1_weighted": 0.670559, + "precision": 0.653997, + "precision_weighted": 0.74679, + "recall": 0.760857, + "recall_weighted": 0.678211, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.680565, + "f1": 0.655754, + "f1_weighted": 0.67526, + "precision": 0.653617, + "precision_weighted": 0.773281, + "recall": 0.766115, + "recall_weighted": 0.680565, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.683255, + "f1": 0.652046, + "f1_weighted": 0.678771, + "precision": 0.639541, + "precision_weighted": 0.752767, + "recall": 0.746147, + "recall_weighted": 0.683255, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.664425, + "f1": 0.650847, + "f1_weighted": 0.64769, + "precision": 0.648019, + "precision_weighted": 0.744891, + "recall": 0.76527, + "recall_weighted": 0.664425, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.696369, + "f1": 0.669709, + "f1_weighted": 0.692736, + "precision": 0.660023, + "precision_weighted": 0.761392, + "recall": 0.778769, + "recall_weighted": 0.696369, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.687223, + "f1": 0.664415, + "f1_weighted": 0.68058, + "precision": 0.654584, + "precision_weighted": 0.756474, + "recall": 0.764821, + "recall_weighted": 0.687223, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.687223, + "hf_subset": "nb", + "languages": [ + "nob-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.72764, + "f1": 0.703036, + "f1_weighted": 0.717335, + "precision": 0.691007, + "precision_weighted": 0.76947, + "recall": 0.795349, + "recall_weighted": 0.72764, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.739072, + "f1": 0.713465, + "f1_weighted": 0.733655, + "precision": 0.697569, + "precision_weighted": 0.792597, + "recall": 0.796504, + "recall_weighted": 0.739072, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.711836, + "f1": 0.685051, + "f1_weighted": 0.70129, + "precision": 0.664134, + "precision_weighted": 0.74859, + "recall": 0.774532, + "recall_weighted": 0.711836, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.737054, + "f1": 0.708436, + "f1_weighted": 0.73125, + "precision": 0.696145, + "precision_weighted": 0.787378, + "recall": 0.779069, + "recall_weighted": 0.737054, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.706456, + "f1": 0.679514, + "f1_weighted": 0.689255, + "precision": 0.675673, + "precision_weighted": 0.777221, + "recall": 0.780734, + "recall_weighted": 0.706456, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.693006, + "f1": 0.682601, + "f1_weighted": 0.682595, + "precision": 0.67106, + "precision_weighted": 0.760192, + "recall": 0.770614, + "recall_weighted": 0.693006, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.710491, + "f1": 0.685183, + "f1_weighted": 0.703261, + "precision": 0.679772, + "precision_weighted": 0.794419, + "recall": 0.795811, + "recall_weighted": 0.710491, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.705111, + "f1": 0.685643, + "f1_weighted": 0.698997, + "precision": 0.672428, + "precision_weighted": 0.765703, + "recall": 0.785304, + "recall_weighted": 0.705111, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.683927, + "f1": 0.673933, + "f1_weighted": 0.666945, + "precision": 0.670205, + "precision_weighted": 0.762835, + "recall": 0.780411, + "recall_weighted": 0.683927, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.718561, + "f1": 0.688272, + "f1_weighted": 0.71373, + "precision": 0.674863, + "precision_weighted": 0.777158, + "recall": 0.787614, + "recall_weighted": 0.718561, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.713315, + "f1": 0.690513, + "f1_weighted": 0.703831, + "precision": 0.679286, + "precision_weighted": 0.773556, + "recall": 0.784594, + "recall_weighted": 0.713315, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.713315, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 281.16170954704285, + "kg_co2_emissions": null, + "date": 1776145672.540869 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/MassiveScenarioClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/MassiveScenarioClassification.json new file mode 100644 index 0000000000..42ebb793f6 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/MassiveScenarioClassification.json @@ -0,0 +1,782 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "2.12.16", + "scores": { + "validation": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.759961, + "f1": 0.74811, + "f1_weighted": 0.757169, + "precision": 0.727175, + "precision_weighted": 0.789304, + "recall": 0.809771, + "recall_weighted": 0.759961, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.744712, + "f1": 0.733513, + "f1_weighted": 0.747019, + "precision": 0.719716, + "precision_weighted": 0.790165, + "recall": 0.798108, + "recall_weighted": 0.744712, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.737826, + "f1": 0.722745, + "f1_weighted": 0.736033, + "precision": 0.706735, + "precision_weighted": 0.775814, + "recall": 0.791161, + "recall_weighted": 0.737826, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.742253, + "f1": 0.727698, + "f1_weighted": 0.74576, + "precision": 0.715407, + "precision_weighted": 0.78424, + "recall": 0.78898, + "recall_weighted": 0.742253, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.745204, + "f1": 0.732479, + "f1_weighted": 0.739654, + "precision": 0.729115, + "precision_weighted": 0.792757, + "recall": 0.799266, + "recall_weighted": 0.745204, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.691589, + "f1": 0.682404, + "f1_weighted": 0.68237, + "precision": 0.669587, + "precision_weighted": 0.748515, + "recall": 0.761162, + "recall_weighted": 0.691589, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.738318, + "f1": 0.721744, + "f1_weighted": 0.739048, + "precision": 0.709342, + "precision_weighted": 0.782827, + "recall": 0.782744, + "recall_weighted": 0.738318, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.72061, + "f1": 0.714341, + "f1_weighted": 0.723516, + "precision": 0.700187, + "precision_weighted": 0.769014, + "recall": 0.773871, + "recall_weighted": 0.72061, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.739793, + "f1": 0.731917, + "f1_weighted": 0.738311, + "precision": 0.709402, + "precision_weighted": 0.774211, + "recall": 0.794994, + "recall_weighted": 0.739793, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.743728, + "f1": 0.736939, + "f1_weighted": 0.744859, + "precision": 0.720954, + "precision_weighted": 0.78603, + "recall": 0.80085, + "recall_weighted": 0.743728, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.736399, + "f1": 0.725189, + "f1_weighted": 0.735374, + "precision": 0.710762, + "precision_weighted": 0.779288, + "recall": 0.790091, + "recall_weighted": 0.736399, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.736399, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.753074, + "f1": 0.747927, + "f1_weighted": 0.753694, + "precision": 0.727372, + "precision_weighted": 0.787305, + "recall": 0.805531, + "recall_weighted": 0.753074, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.730939, + "f1": 0.724458, + "f1_weighted": 0.728819, + "precision": 0.711418, + "precision_weighted": 0.776947, + "recall": 0.795473, + "recall_weighted": 0.730939, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.751107, + "f1": 0.738007, + "f1_weighted": 0.749122, + "precision": 0.718489, + "precision_weighted": 0.779731, + "recall": 0.802019, + "recall_weighted": 0.751107, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.727496, + "f1": 0.71159, + "f1_weighted": 0.730213, + "precision": 0.699447, + "precision_weighted": 0.774487, + "recall": 0.777735, + "recall_weighted": 0.727496, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.730939, + "f1": 0.716334, + "f1_weighted": 0.729539, + "precision": 0.709778, + "precision_weighted": 0.781947, + "recall": 0.784344, + "recall_weighted": 0.730939, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.705362, + "f1": 0.69884, + "f1_weighted": 0.696344, + "precision": 0.682737, + "precision_weighted": 0.744671, + "recall": 0.767404, + "recall_weighted": 0.705362, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.72848, + "f1": 0.71636, + "f1_weighted": 0.727641, + "precision": 0.703547, + "precision_weighted": 0.769957, + "recall": 0.779911, + "recall_weighted": 0.72848, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.715691, + "f1": 0.709623, + "f1_weighted": 0.718324, + "precision": 0.695093, + "precision_weighted": 0.757397, + "recall": 0.766391, + "recall_weighted": 0.715691, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.760944, + "f1": 0.748321, + "f1_weighted": 0.759899, + "precision": 0.727254, + "precision_weighted": 0.78938, + "recall": 0.803198, + "recall_weighted": 0.760944, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.742253, + "f1": 0.730584, + "f1_weighted": 0.745354, + "precision": 0.71512, + "precision_weighted": 0.785178, + "recall": 0.792732, + "recall_weighted": 0.742253, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.734629, + "f1": 0.724204, + "f1_weighted": 0.733895, + "precision": 0.709026, + "precision_weighted": 0.7747, + "recall": 0.787474, + "recall_weighted": 0.734629, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.734629, + "hf_subset": "nb", + "languages": [ + "nob-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.755042, + "f1": 0.750157, + "f1_weighted": 0.75485, + "precision": 0.728179, + "precision_weighted": 0.786842, + "recall": 0.807542, + "recall_weighted": 0.755042, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.743237, + "f1": 0.730774, + "f1_weighted": 0.742573, + "precision": 0.712512, + "precision_weighted": 0.782272, + "recall": 0.797897, + "recall_weighted": 0.743237, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.75455, + "f1": 0.737805, + "f1_weighted": 0.751522, + "precision": 0.718753, + "precision_weighted": 0.791149, + "recall": 0.809464, + "recall_weighted": 0.75455, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.735366, + "f1": 0.722404, + "f1_weighted": 0.736494, + "precision": 0.707939, + "precision_weighted": 0.769513, + "recall": 0.781564, + "recall_weighted": 0.735366, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.741269, + "f1": 0.723687, + "f1_weighted": 0.738068, + "precision": 0.713341, + "precision_weighted": 0.788124, + "recall": 0.793179, + "recall_weighted": 0.741269, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.714215, + "f1": 0.704749, + "f1_weighted": 0.70795, + "precision": 0.685051, + "precision_weighted": 0.755531, + "recall": 0.776733, + "recall_weighted": 0.714215, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.74422, + "f1": 0.725719, + "f1_weighted": 0.744175, + "precision": 0.711658, + "precision_weighted": 0.78489, + "recall": 0.787898, + "recall_weighted": 0.74422, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.718151, + "f1": 0.718871, + "f1_weighted": 0.721449, + "precision": 0.706469, + "precision_weighted": 0.764858, + "recall": 0.77618, + "recall_weighted": 0.718151, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.776685, + "f1": 0.763316, + "f1_weighted": 0.776739, + "precision": 0.741831, + "precision_weighted": 0.803919, + "recall": 0.816582, + "recall_weighted": 0.776685, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.753074, + "f1": 0.74412, + "f1_weighted": 0.757045, + "precision": 0.728438, + "precision_weighted": 0.798165, + "recall": 0.804724, + "recall_weighted": 0.753074, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.743581, + "f1": 0.73216, + "f1_weighted": 0.743086, + "precision": 0.715417, + "precision_weighted": 0.782526, + "recall": 0.795176, + "recall_weighted": 0.743581, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.743581, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ] + } + ], + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.756221, + "f1": 0.7493, + "f1_weighted": 0.755026, + "precision": 0.729628, + "precision_weighted": 0.782855, + "recall": 0.80298, + "recall_weighted": 0.756221, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.746133, + "f1": 0.734685, + "f1_weighted": 0.746176, + "precision": 0.722547, + "precision_weighted": 0.785399, + "recall": 0.794211, + "recall_weighted": 0.746133, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.745461, + "f1": 0.735699, + "f1_weighted": 0.742642, + "precision": 0.719244, + "precision_weighted": 0.778858, + "recall": 0.797837, + "recall_weighted": 0.745461, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.732011, + "f1": 0.722891, + "f1_weighted": 0.735894, + "precision": 0.712228, + "precision_weighted": 0.772258, + "recall": 0.775057, + "recall_weighted": 0.732011, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.729321, + "f1": 0.717686, + "f1_weighted": 0.722863, + "precision": 0.717106, + "precision_weighted": 0.77786, + "recall": 0.781618, + "recall_weighted": 0.729321, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.70343, + "f1": 0.690212, + "f1_weighted": 0.696605, + "precision": 0.678169, + "precision_weighted": 0.756062, + "recall": 0.76101, + "recall_weighted": 0.70343, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.738399, + "f1": 0.722478, + "f1_weighted": 0.738166, + "precision": 0.709158, + "precision_weighted": 0.780254, + "recall": 0.783165, + "recall_weighted": 0.738399, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.72495, + "f1": 0.72291, + "f1_weighted": 0.72686, + "precision": 0.708779, + "precision_weighted": 0.768833, + "recall": 0.779286, + "recall_weighted": 0.72495, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.748487, + "f1": 0.742496, + "f1_weighted": 0.748572, + "precision": 0.722283, + "precision_weighted": 0.786117, + "recall": 0.804003, + "recall_weighted": 0.748487, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.742771, + "f1": 0.736623, + "f1_weighted": 0.743807, + "precision": 0.72146, + "precision_weighted": 0.782768, + "recall": 0.79562, + "recall_weighted": 0.742771, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.736718, + "f1": 0.727498, + "f1_weighted": 0.735661, + "precision": 0.71406, + "precision_weighted": 0.777126, + "recall": 0.787479, + "recall_weighted": 0.736718, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.736718, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.753194, + "f1": 0.750755, + "f1_weighted": 0.755063, + "precision": 0.732824, + "precision_weighted": 0.788021, + "recall": 0.80405, + "recall_weighted": 0.753194, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.738063, + "f1": 0.730846, + "f1_weighted": 0.737598, + "precision": 0.717971, + "precision_weighted": 0.784119, + "recall": 0.797245, + "recall_weighted": 0.738063, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.749832, + "f1": 0.738067, + "f1_weighted": 0.747492, + "precision": 0.725541, + "precision_weighted": 0.788601, + "recall": 0.802288, + "recall_weighted": 0.749832, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.725958, + "f1": 0.714278, + "f1_weighted": 0.727701, + "precision": 0.699553, + "precision_weighted": 0.770025, + "recall": 0.77622, + "recall_weighted": 0.725958, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.720915, + "f1": 0.711949, + "f1_weighted": 0.717274, + "precision": 0.705585, + "precision_weighted": 0.770227, + "recall": 0.778078, + "recall_weighted": 0.720915, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.717888, + "f1": 0.704981, + "f1_weighted": 0.711412, + "precision": 0.690004, + "precision_weighted": 0.755981, + "recall": 0.767387, + "recall_weighted": 0.717888, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.734364, + "f1": 0.71733, + "f1_weighted": 0.734402, + "precision": 0.70442, + "precision_weighted": 0.777048, + "recall": 0.778022, + "recall_weighted": 0.734364, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.734028, + "f1": 0.732571, + "f1_weighted": 0.73803, + "precision": 0.721756, + "precision_weighted": 0.781999, + "recall": 0.786715, + "recall_weighted": 0.734028, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.752522, + "f1": 0.74601, + "f1_weighted": 0.752853, + "precision": 0.724524, + "precision_weighted": 0.786371, + "recall": 0.801794, + "recall_weighted": 0.752522, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.746469, + "f1": 0.741809, + "f1_weighted": 0.748414, + "precision": 0.727076, + "precision_weighted": 0.792305, + "recall": 0.806467, + "recall_weighted": 0.746469, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.737323, + "f1": 0.72886, + "f1_weighted": 0.737024, + "precision": 0.714925, + "precision_weighted": 0.77947, + "recall": 0.789827, + "recall_weighted": 0.737323, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.737323, + "hf_subset": "nb", + "languages": [ + "nob-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.764627, + "f1": 0.764755, + "f1_weighted": 0.764425, + "precision": 0.744331, + "precision_weighted": 0.790582, + "recall": 0.814218, + "recall_weighted": 0.764627, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.749832, + "f1": 0.739039, + "f1_weighted": 0.748728, + "precision": 0.722773, + "precision_weighted": 0.787108, + "recall": 0.799463, + "recall_weighted": 0.749832, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.754539, + "f1": 0.741335, + "f1_weighted": 0.751921, + "precision": 0.725982, + "precision_weighted": 0.79362, + "recall": 0.807427, + "recall_weighted": 0.754539, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.750841, + "f1": 0.737083, + "f1_weighted": 0.753417, + "precision": 0.722675, + "precision_weighted": 0.785292, + "recall": 0.790358, + "recall_weighted": 0.750841, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.736046, + "f1": 0.724702, + "f1_weighted": 0.731289, + "precision": 0.720693, + "precision_weighted": 0.788691, + "recall": 0.79299, + "recall_weighted": 0.736046, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.736046, + "f1": 0.723761, + "f1_weighted": 0.731195, + "precision": 0.704525, + "precision_weighted": 0.774703, + "recall": 0.788981, + "recall_weighted": 0.736046, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.754539, + "f1": 0.738965, + "f1_weighted": 0.755452, + "precision": 0.723439, + "precision_weighted": 0.797568, + "recall": 0.802375, + "recall_weighted": 0.754539, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.732011, + "f1": 0.735666, + "f1_weighted": 0.735001, + "precision": 0.723883, + "precision_weighted": 0.778478, + "recall": 0.792515, + "recall_weighted": 0.732011, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.779085, + "f1": 0.773804, + "f1_weighted": 0.779562, + "precision": 0.752004, + "precision_weighted": 0.808122, + "recall": 0.825403, + "recall_weighted": 0.779085, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.756893, + "f1": 0.757293, + "f1_weighted": 0.759871, + "precision": 0.741327, + "precision_weighted": 0.802082, + "recall": 0.815378, + "recall_weighted": 0.756893, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.751446, + "f1": 0.74364, + "f1_weighted": 0.751086, + "precision": 0.728163, + "precision_weighted": 0.790625, + "recall": 0.802911, + "recall_weighted": 0.751446, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.751446, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 168.35132765769958, + "kg_co2_emissions": null, + "date": 1776145844.108714 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NoRecClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NoRecClassification.json new file mode 100644 index 0000000000..667dc11fe4 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NoRecClassification.json @@ -0,0 +1,140 @@ +{ + "dataset_revision": "5b740b7c42c73d586420812a35745fc37118862f", + "task_name": "NoRecClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.57373, + "f1": 0.56797, + "f1_weighted": 0.588237, + "precision": 0.604951, + "precision_weighted": 0.672753, + "recall": 0.633062, + "recall_weighted": 0.57373, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.54541, + "f1": 0.531506, + "f1_weighted": 0.552113, + "precision": 0.528339, + "precision_weighted": 0.577134, + "recall": 0.560279, + "recall_weighted": 0.54541, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.498535, + "f1": 0.491121, + "f1_weighted": 0.513874, + "precision": 0.547557, + "precision_weighted": 0.62159, + "recall": 0.562589, + "recall_weighted": 0.498535, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.582031, + "f1": 0.562922, + "f1_weighted": 0.592175, + "precision": 0.563731, + "precision_weighted": 0.623322, + "recall": 0.592204, + "recall_weighted": 0.582031, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.531738, + "f1": 0.522528, + "f1_weighted": 0.539423, + "precision": 0.52764, + "precision_weighted": 0.580372, + "recall": 0.566792, + "recall_weighted": 0.531738, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.605469, + "f1": 0.586599, + "f1_weighted": 0.611194, + "precision": 0.583051, + "precision_weighted": 0.635554, + "recall": 0.611012, + "recall_weighted": 0.605469, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.552734, + "f1": 0.547203, + "f1_weighted": 0.566333, + "precision": 0.576932, + "precision_weighted": 0.647044, + "recall": 0.608124, + "recall_weighted": 0.552734, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.578125, + "f1": 0.5357, + "f1_weighted": 0.58068, + "precision": 0.537436, + "precision_weighted": 0.590644, + "recall": 0.539955, + "recall_weighted": 0.578125, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.541992, + "f1": 0.540332, + "f1_weighted": 0.543471, + "precision": 0.542673, + "precision_weighted": 0.590126, + "recall": 0.595884, + "recall_weighted": 0.541992, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.563477, + "f1": 0.55853, + "f1_weighted": 0.561071, + "precision": 0.573946, + "precision_weighted": 0.595307, + "recall": 0.57224, + "recall_weighted": 0.563477, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.557324, + "f1": 0.544441, + "f1_weighted": 0.564857, + "precision": 0.558626, + "precision_weighted": 0.613385, + "recall": 0.584214, + "recall_weighted": 0.557324, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.557324, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 19.01038956642151, + "kg_co2_emissions": null, + "date": 1776145909.459024 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorQuadRetrieval.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorQuadRetrieval.json new file mode 100644 index 0000000000..cc81aee02a --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorQuadRetrieval.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "9dcfcdb2aa578dd178330d49bf564248935f7fbe", + "task_name": "NorQuadRetrieval", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.27539, + "ndcg_at_3": 0.2219, + "ndcg_at_5": 0.23734, + "ndcg_at_10": 0.25756, + "ndcg_at_20": 0.27263, + "ndcg_at_100": 0.30168, + "ndcg_at_1000": 0.34236, + "map_at_1": 0.1377, + "map_at_3": 0.18042, + "map_at_5": 0.18792, + "map_at_10": 0.19631, + "map_at_20": 0.20027, + "map_at_100": 0.20393, + "map_at_1000": 0.20542, + "recall_at_1": 0.1377, + "recall_at_3": 0.21143, + "recall_at_5": 0.24219, + "recall_at_10": 0.29297, + "recall_at_20": 0.3418, + "recall_at_100": 0.47168, + "recall_at_1000": 0.75049, + "accuracy": 0.1377, + "precision_at_1": 0.27539, + "precision_at_3": 0.14095, + "precision_at_5": 0.09688, + "precision_at_10": 0.05859, + "precision_at_20": 0.03418, + "precision_at_100": 0.00943, + "precision_at_1000": 0.0015, + "mrr_at_1": 0.275391, + "mrr_at_3": 0.320801, + "mrr_at_5": 0.333545, + "mrr_at_10": 0.344134, + "mrr_at_20": 0.34958, + "mrr_at_100": 0.354783, + "mrr_at_1000": 0.355661, + "nauc_ndcg_at_1_max": 0.302251, + "nauc_ndcg_at_1_std": -0.11661, + "nauc_ndcg_at_1_diff1": 0.465799, + "nauc_ndcg_at_3_max": 0.327393, + "nauc_ndcg_at_3_std": -0.1252, + "nauc_ndcg_at_3_diff1": 0.45142, + "nauc_ndcg_at_5_max": 0.316204, + "nauc_ndcg_at_5_std": -0.125304, + "nauc_ndcg_at_5_diff1": 0.435629, + "nauc_ndcg_at_10_max": 0.312423, + "nauc_ndcg_at_10_std": -0.118243, + "nauc_ndcg_at_10_diff1": 0.42736, + "nauc_ndcg_at_20_max": 0.314467, + "nauc_ndcg_at_20_std": -0.114857, + "nauc_ndcg_at_20_diff1": 0.42548, + "nauc_ndcg_at_100_max": 0.31809, + "nauc_ndcg_at_100_std": -0.10401, + "nauc_ndcg_at_100_diff1": 0.42354, + "nauc_ndcg_at_1000_max": 0.329051, + "nauc_ndcg_at_1000_std": -0.10045, + "nauc_ndcg_at_1000_diff1": 0.430522, + "nauc_map_at_1_max": 0.302251, + "nauc_map_at_1_std": -0.11661, + "nauc_map_at_1_diff1": 0.465799, + "nauc_map_at_3_max": 0.343118, + "nauc_map_at_3_std": -0.116036, + "nauc_map_at_3_diff1": 0.459745, + "nauc_map_at_5_max": 0.336273, + "nauc_map_at_5_std": -0.115412, + "nauc_map_at_5_diff1": 0.450294, + "nauc_map_at_10_max": 0.336052, + "nauc_map_at_10_std": -0.110078, + "nauc_map_at_10_diff1": 0.447567, + "nauc_map_at_20_max": 0.336451, + "nauc_map_at_20_std": -0.109284, + "nauc_map_at_20_diff1": 0.446973, + "nauc_map_at_100_max": 0.337065, + "nauc_map_at_100_std": -0.107549, + "nauc_map_at_100_diff1": 0.446446, + "nauc_map_at_1000_max": 0.337689, + "nauc_map_at_1000_std": -0.107221, + "nauc_map_at_1000_diff1": 0.446689, + "nauc_recall_at_1_max": 0.302251, + "nauc_recall_at_1_std": -0.11661, + "nauc_recall_at_1_diff1": 0.465799, + "nauc_recall_at_3_max": 0.329357, + "nauc_recall_at_3_std": -0.129923, + "nauc_recall_at_3_diff1": 0.434107, + "nauc_recall_at_5_max": 0.30504, + "nauc_recall_at_5_std": -0.129511, + "nauc_recall_at_5_diff1": 0.399376, + "nauc_recall_at_10_max": 0.294168, + "nauc_recall_at_10_std": -0.105244, + "nauc_recall_at_10_diff1": 0.374346, + "nauc_recall_at_20_max": 0.292694, + "nauc_recall_at_20_std": -0.090804, + "nauc_recall_at_20_diff1": 0.355771, + "nauc_recall_at_100_max": 0.29189, + "nauc_recall_at_100_std": -0.029686, + "nauc_recall_at_100_diff1": 0.316989, + "nauc_recall_at_1000_max": 0.250769, + "nauc_recall_at_1000_std": 0.046059, + "nauc_recall_at_1000_diff1": 0.19538, + "nauc_precision_at_1_max": 0.302251, + "nauc_precision_at_1_std": -0.11661, + "nauc_precision_at_1_diff1": 0.465799, + "nauc_precision_at_3_max": 0.329357, + "nauc_precision_at_3_std": -0.129923, + "nauc_precision_at_3_diff1": 0.434107, + "nauc_precision_at_5_max": 0.30504, + "nauc_precision_at_5_std": -0.129511, + "nauc_precision_at_5_diff1": 0.399376, + "nauc_precision_at_10_max": 0.294168, + "nauc_precision_at_10_std": -0.105244, + "nauc_precision_at_10_diff1": 0.374346, + "nauc_precision_at_20_max": 0.292694, + "nauc_precision_at_20_std": -0.090804, + "nauc_precision_at_20_diff1": 0.355771, + "nauc_precision_at_100_max": 0.29189, + "nauc_precision_at_100_std": -0.029686, + "nauc_precision_at_100_diff1": 0.316989, + "nauc_precision_at_1000_max": 0.250769, + "nauc_precision_at_1000_std": 0.046059, + "nauc_precision_at_1000_diff1": 0.19538, + "nauc_mrr_at_1_max": 0.302251, + "nauc_mrr_at_1_std": -0.11661, + "nauc_mrr_at_1_diff1": 0.465799, + "nauc_mrr_at_3_max": 0.302329, + "nauc_mrr_at_3_std": -0.130703, + "nauc_mrr_at_3_diff1": 0.45098, + "nauc_mrr_at_5_max": 0.29573, + "nauc_mrr_at_5_std": -0.131795, + "nauc_mrr_at_5_diff1": 0.442042, + "nauc_mrr_at_10_max": 0.290011, + "nauc_mrr_at_10_std": -0.133588, + "nauc_mrr_at_10_diff1": 0.436386, + "nauc_mrr_at_20_max": 0.29058, + "nauc_mrr_at_20_std": -0.132475, + "nauc_mrr_at_20_diff1": 0.43622, + "nauc_mrr_at_100_max": 0.290954, + "nauc_mrr_at_100_std": -0.131084, + "nauc_mrr_at_100_diff1": 0.436194, + "nauc_mrr_at_1000_max": 0.291207, + "nauc_mrr_at_1000_std": -0.131169, + "nauc_mrr_at_1000_diff1": 0.43659, + "hit_rate_at_1": 0.27539, + "hit_rate_at_3": 0.37891, + "hit_rate_at_5": 0.43555, + "hit_rate_at_10": 0.5127, + "hit_rate_at_20": 0.59277, + "hit_rate_at_100": 0.7998, + "hit_rate_at_1000": 0.98633, + "main_score": 0.25756, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 39.90969204902649, + "kg_co2_emissions": null, + "date": 1776146426.144978 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NordicLangClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NordicLangClassification.json new file mode 100644 index 0000000000..c710bcfa66 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NordicLangClassification.json @@ -0,0 +1,145 @@ +{ + "dataset_revision": "425d7e9de276902b73d5359ecd4811f3b80cc7c0", + "task_name": "NordicLangClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.899667, + "f1": 0.899845, + "f1_weighted": 0.900989, + "precision": 0.903233, + "precision_weighted": 0.905817, + "recall": 0.900098, + "recall_weighted": 0.899667, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.909333, + "f1": 0.908798, + "f1_weighted": 0.909151, + "precision": 0.911697, + "precision_weighted": 0.91208, + "recall": 0.908936, + "recall_weighted": 0.909333, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.912667, + "f1": 0.912189, + "f1_weighted": 0.91294, + "precision": 0.914312, + "precision_weighted": 0.915083, + "recall": 0.911902, + "recall_weighted": 0.912667, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.914, + "f1": 0.913494, + "f1_weighted": 0.914457, + "precision": 0.915343, + "precision_weighted": 0.916761, + "recall": 0.913532, + "recall_weighted": 0.914, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.916667, + "f1": 0.916385, + "f1_weighted": 0.917119, + "precision": 0.918553, + "precision_weighted": 0.919285, + "recall": 0.915893, + "recall_weighted": 0.916667, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.919667, + "f1": 0.919398, + "f1_weighted": 0.919919, + "precision": 0.922057, + "precision_weighted": 0.922104, + "recall": 0.918604, + "recall_weighted": 0.919667, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.918, + "f1": 0.917606, + "f1_weighted": 0.918458, + "precision": 0.920053, + "precision_weighted": 0.920817, + "recall": 0.917043, + "recall_weighted": 0.918, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.902333, + "f1": 0.901878, + "f1_weighted": 0.902405, + "precision": 0.905601, + "precision_weighted": 0.906769, + "recall": 0.902361, + "recall_weighted": 0.902333, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.914333, + "f1": 0.914054, + "f1_weighted": 0.914321, + "precision": 0.916835, + "precision_weighted": 0.916606, + "recall": 0.913477, + "recall_weighted": 0.914333, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.904333, + "f1": 0.904039, + "f1_weighted": 0.904638, + "precision": 0.906681, + "precision_weighted": 0.907943, + "recall": 0.904358, + "recall_weighted": 0.904333, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.9111, + "f1": 0.910769, + "f1_weighted": 0.91144, + "precision": 0.913437, + "precision_weighted": 0.914326, + "recall": 0.910621, + "recall_weighted": 0.9111, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.9111, + "hf_subset": "default", + "languages": [ + "nob-Latn", + "nno-Latn", + "dan-Latn", + "swe-Latn", + "isl-Latn", + "fao-Latn" + ] + } + ] + }, + "evaluation_time": 41.60946607589722, + "kg_co2_emissions": null, + "date": 1776145888.397882 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorwegianCourtsBitextMining.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorwegianCourtsBitextMining.json new file mode 100644 index 0000000000..b82911d57f --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorwegianCourtsBitextMining.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "ddf68e49e8d393964288ad5b498f1d61cca7b23f", + "task_name": "NorwegianCourtsBitextMining", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "precision": 0.928363, + "recall": 0.942982, + "f1": 0.932749, + "accuracy": 0.942982, + "main_score": 0.932749, + "hf_subset": "default", + "languages": [ + "nob-Latn", + "nno-Latn" + ] + } + ] + }, + "evaluation_time": 3.1199300289154053, + "kg_co2_emissions": null, + "date": 1776145281.001391 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorwegianParliamentClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorwegianParliamentClassification.json new file mode 100644 index 0000000000..34915794d8 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/NorwegianParliamentClassification.json @@ -0,0 +1,270 @@ +{ + "dataset_revision": "3047317d2586abb183293f92b1b7d66d1c9ec81a", + "task_name": "NorwegianParliamentClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.536667, + "f1": 0.534275, + "f1_weighted": 0.534275, + "precision": 0.537436, + "precision_weighted": 0.537436, + "recall": 0.536667, + "recall_weighted": 0.536667, + "ap": 0.519903, + "ap_weighted": 0.519903 + }, + { + "accuracy": 0.529167, + "f1": 0.515296, + "f1_weighted": 0.515296, + "precision": 0.532937, + "precision_weighted": 0.532937, + "recall": 0.529167, + "recall_weighted": 0.529167, + "ap": 0.515219, + "ap_weighted": 0.515219 + }, + { + "accuracy": 0.6075, + "f1": 0.605613, + "f1_weighted": 0.605613, + "precision": 0.609597, + "precision_weighted": 0.609597, + "recall": 0.6075, + "recall_weighted": 0.6075, + "ap": 0.563902, + "ap_weighted": 0.563902 + }, + { + "accuracy": 0.593333, + "f1": 0.592381, + "f1_weighted": 0.592381, + "precision": 0.594214, + "precision_weighted": 0.594214, + "recall": 0.593333, + "recall_weighted": 0.593333, + "ap": 0.55631, + "ap_weighted": 0.55631 + }, + { + "accuracy": 0.6075, + "f1": 0.605329, + "f1_weighted": 0.605329, + "precision": 0.609919, + "precision_weighted": 0.609919, + "recall": 0.6075, + "recall_weighted": 0.6075, + "ap": 0.567319, + "ap_weighted": 0.567319 + }, + { + "accuracy": 0.5375, + "f1": 0.522745, + "f1_weighted": 0.522745, + "precision": 0.542792, + "precision_weighted": 0.542792, + "recall": 0.5375, + "recall_weighted": 0.5375, + "ap": 0.51979, + "ap_weighted": 0.51979 + }, + { + "accuracy": 0.5075, + "f1": 0.506224, + "f1_weighted": 0.506224, + "precision": 0.507578, + "precision_weighted": 0.507578, + "recall": 0.5075, + "recall_weighted": 0.5075, + "ap": 0.503813, + "ap_weighted": 0.503813 + }, + { + "accuracy": 0.563333, + "f1": 0.563256, + "f1_weighted": 0.563256, + "precision": 0.563378, + "precision_weighted": 0.563378, + "recall": 0.563333, + "recall_weighted": 0.563333, + "ap": 0.535574, + "ap_weighted": 0.535574 + }, + { + "accuracy": 0.624167, + "f1": 0.624091, + "f1_weighted": 0.624091, + "precision": 0.624266, + "precision_weighted": 0.624266, + "recall": 0.624167, + "recall_weighted": 0.624167, + "ap": 0.577076, + "ap_weighted": 0.577076 + }, + { + "accuracy": 0.63, + "f1": 0.629407, + "f1_weighted": 0.629407, + "precision": 0.630837, + "precision_weighted": 0.630837, + "recall": 0.63, + "recall_weighted": 0.63, + "ap": 0.58337, + "ap_weighted": 0.58337 + } + ], + "accuracy": 0.573667, + "f1": 0.569862, + "f1_weighted": 0.569862, + "precision": 0.575295, + "precision_weighted": 0.575295, + "recall": 0.573667, + "recall_weighted": 0.573667, + "ap": 0.544227, + "ap_weighted": 0.544227, + "main_score": 0.573667, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ], + "validation": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.545, + "f1": 0.541945, + "f1_weighted": 0.541945, + "precision": 0.546233, + "precision_weighted": 0.546233, + "recall": 0.545, + "recall_weighted": 0.545, + "ap": 0.52492, + "ap_weighted": 0.52492 + }, + { + "accuracy": 0.479167, + "f1": 0.467041, + "f1_weighted": 0.467041, + "precision": 0.477081, + "precision_weighted": 0.477081, + "recall": 0.479167, + "recall_weighted": 0.479167, + "ap": 0.489917, + "ap_weighted": 0.489917 + }, + { + "accuracy": 0.609167, + "f1": 0.60871, + "f1_weighted": 0.60871, + "precision": 0.609679, + "precision_weighted": 0.609679, + "recall": 0.609167, + "recall_weighted": 0.609167, + "ap": 0.565738, + "ap_weighted": 0.565738 + }, + { + "accuracy": 0.600833, + "f1": 0.599866, + "f1_weighted": 0.599866, + "precision": 0.601818, + "precision_weighted": 0.601818, + "recall": 0.600833, + "recall_weighted": 0.600833, + "ap": 0.561693, + "ap_weighted": 0.561693 + }, + { + "accuracy": 0.624167, + "f1": 0.621896, + "f1_weighted": 0.621896, + "precision": 0.627223, + "precision_weighted": 0.627223, + "recall": 0.624167, + "recall_weighted": 0.624167, + "ap": 0.580329, + "ap_weighted": 0.580329 + }, + { + "accuracy": 0.51, + "f1": 0.494214, + "f1_weighted": 0.494214, + "precision": 0.511427, + "precision_weighted": 0.511427, + "recall": 0.51, + "recall_weighted": 0.51, + "ap": 0.505074, + "ap_weighted": 0.505074 + }, + { + "accuracy": 0.521667, + "f1": 0.520767, + "f1_weighted": 0.520767, + "precision": 0.521831, + "precision_weighted": 0.521831, + "recall": 0.521667, + "recall_weighted": 0.521667, + "ap": 0.511347, + "ap_weighted": 0.511347 + }, + { + "accuracy": 0.555833, + "f1": 0.555826, + "f1_weighted": 0.555826, + "precision": 0.555837, + "precision_weighted": 0.555837, + "recall": 0.555833, + "recall_weighted": 0.555833, + "ap": 0.531008, + "ap_weighted": 0.531008 + }, + { + "accuracy": 0.623333, + "f1": 0.623307, + "f1_weighted": 0.623307, + "precision": 0.623368, + "precision_weighted": 0.623368, + "recall": 0.623333, + "recall_weighted": 0.623333, + "ap": 0.576628, + "ap_weighted": 0.576628 + }, + { + "accuracy": 0.576667, + "f1": 0.576242, + "f1_weighted": 0.576242, + "precision": 0.576975, + "precision_weighted": 0.576975, + "recall": 0.576667, + "recall_weighted": 0.576667, + "ap": 0.544609, + "ap_weighted": 0.544609 + } + ], + "accuracy": 0.564583, + "f1": 0.560981, + "f1_weighted": 0.560981, + "precision": 0.565147, + "precision_weighted": 0.565147, + "recall": 0.564583, + "recall_weighted": 0.564583, + "ap": 0.539126, + "ap_weighted": 0.539126, + "main_score": 0.564583, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 97.72072911262512, + "kg_co2_emissions": null, + "date": 1776146009.718022 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLHierarchicalClusteringP2P.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLHierarchicalClusteringP2P.json new file mode 100644 index 0000000000..e866c90742 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLHierarchicalClusteringP2P.json @@ -0,0 +1,83 @@ +{ + "dataset_revision": "693a321c42fb13ffe76bb9043f8d2aaa8f0a9499", + "task_name": "SNLHierarchicalClusteringP2P", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.38436, + 0.373799, + 0.404655, + 0.415824, + 0.4289, + 0.411, + 0.430465, + 0.393346, + 0.416445, + 0.375884 + ], + "Level 1": [ + 0.62489, + 0.621453, + 0.632997, + 0.616762, + 0.616218, + 0.626938, + 0.641427, + 0.635772, + 0.614863, + 0.631034 + ], + "Level 2": [ + 0.706878, + 0.700094, + 0.72194, + 0.704904, + 0.697797, + 0.701024, + 0.705964, + 0.698219, + 0.701402, + 0.699967 + ], + "Level 3": [ + 0.712914, + 0.718745, + 0.718645, + 0.721116, + 0.726674, + 0.720469, + 0.718429, + 0.728854, + 0.728507, + 0.725982 + ], + "Level 4": [ + 0.398422, + 0.38612, + 0.386842, + 0.379842, + 0.379875, + 0.392307, + 0.381878, + 0.387887, + 0.381484, + 0.390243 + ] + }, + "v_measure": 0.568409, + "v_measure_std": 0.1457, + "main_score": 0.568409, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 79.553875207901, + "kg_co2_emissions": null, + "date": 1776146826.809719 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLHierarchicalClusteringS2S.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLHierarchicalClusteringS2S.json new file mode 100644 index 0000000000..d17324cdfe --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLHierarchicalClusteringS2S.json @@ -0,0 +1,83 @@ +{ + "dataset_revision": "b505e4ce65f255228e49dd07b6f8148731c5dc64", + "task_name": "SNLHierarchicalClusteringS2S", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.423592, + 0.436733, + 0.413065, + 0.416752, + 0.421976, + 0.433317, + 0.422008, + 0.455472, + 0.421292, + 0.43091 + ], + "Level 1": [ + 0.647068, + 0.637344, + 0.651841, + 0.649307, + 0.641834, + 0.628788, + 0.636214, + 0.643345, + 0.620661, + 0.642661 + ], + "Level 2": [ + 0.710458, + 0.687764, + 0.69801, + 0.690244, + 0.694195, + 0.704643, + 0.701644, + 0.688064, + 0.69456, + 0.692362 + ], + "Level 3": [ + 0.704112, + 0.709037, + 0.717161, + 0.709475, + 0.712676, + 0.711545, + 0.712853, + 0.71721, + 0.71241, + 0.70739 + ], + "Level 4": [ + 0.381004, + 0.384718, + 0.376991, + 0.369927, + 0.370087, + 0.385248, + 0.377653, + 0.378991, + 0.383425, + 0.395398 + ] + }, + "v_measure": 0.571069, + "v_measure_std": 0.139573, + "main_score": 0.571069, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 52.47382378578186, + "kg_co2_emissions": null, + "date": 1776146745.076325 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLRetrieval.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLRetrieval.json new file mode 100644 index 0000000000..a933284da6 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SNLRetrieval.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "22c474c88fb4678052f9099bb917ad8f9e155f9f", + "task_name": "SNLRetrieval", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.96154, + "ndcg_at_3": 0.97365, + "ndcg_at_5": 0.97458, + "ndcg_at_10": 0.97588, + "ndcg_at_20": 0.97667, + "ndcg_at_100": 0.97793, + "ndcg_at_1000": 0.97822, + "map_at_1": 0.96154, + "map_at_3": 0.9709, + "map_at_5": 0.9714, + "map_at_10": 0.97197, + "map_at_20": 0.97219, + "map_at_100": 0.97236, + "map_at_1000": 0.97237, + "recall_at_1": 0.96154, + "recall_at_3": 0.98154, + "recall_at_5": 0.98385, + "recall_at_10": 0.98769, + "recall_at_20": 0.99077, + "recall_at_100": 0.99769, + "recall_at_1000": 1.0, + "accuracy": 0.96154, + "precision_at_1": 0.96154, + "precision_at_3": 0.32718, + "precision_at_5": 0.19677, + "precision_at_10": 0.09877, + "precision_at_20": 0.04954, + "precision_at_100": 0.00998, + "precision_at_1000": 0.001, + "mrr_at_1": 0.961538, + "mrr_at_3": 0.970897, + "mrr_at_5": 0.971397, + "mrr_at_10": 0.971969, + "mrr_at_20": 0.97219, + "mrr_at_100": 0.972355, + "mrr_at_1000": 0.972366, + "nauc_ndcg_at_1_max": 0.80774, + "nauc_ndcg_at_1_std": 0.141457, + "nauc_ndcg_at_1_diff1": 0.970261, + "nauc_ndcg_at_3_max": 0.873031, + "nauc_ndcg_at_3_std": 0.199267, + "nauc_ndcg_at_3_diff1": 0.965719, + "nauc_ndcg_at_5_max": 0.871637, + "nauc_ndcg_at_5_std": 0.225919, + "nauc_ndcg_at_5_diff1": 0.96809, + "nauc_ndcg_at_10_max": 0.864715, + "nauc_ndcg_at_10_std": 0.209527, + "nauc_ndcg_at_10_diff1": 0.966369, + "nauc_ndcg_at_20_max": 0.860161, + "nauc_ndcg_at_20_std": 0.199713, + "nauc_ndcg_at_20_diff1": 0.965237, + "nauc_ndcg_at_100_max": 0.859467, + "nauc_ndcg_at_100_std": 0.188026, + "nauc_ndcg_at_100_diff1": 0.967382, + "nauc_ndcg_at_1000_max": 0.857592, + "nauc_ndcg_at_1000_std": 0.193891, + "nauc_ndcg_at_1000_diff1": 0.967436, + "nauc_map_at_1_max": 0.80774, + "nauc_map_at_1_std": 0.141457, + "nauc_map_at_1_diff1": 0.970261, + "nauc_map_at_3_max": 0.85618, + "nauc_map_at_3_std": 0.185596, + "nauc_map_at_3_diff1": 0.967032, + "nauc_map_at_5_max": 0.855248, + "nauc_map_at_5_std": 0.198163, + "nauc_map_at_5_diff1": 0.968324, + "nauc_map_at_10_max": 0.852297, + "nauc_map_at_10_std": 0.191926, + "nauc_map_at_10_diff1": 0.967678, + "nauc_map_at_20_max": 0.851123, + "nauc_map_at_20_std": 0.189664, + "nauc_map_at_20_diff1": 0.967421, + "nauc_map_at_100_max": 0.85127, + "nauc_map_at_100_std": 0.188436, + "nauc_map_at_100_diff1": 0.967853, + "nauc_map_at_1000_max": 0.851213, + "nauc_map_at_1000_std": 0.188633, + "nauc_map_at_1000_diff1": 0.967846, + "nauc_recall_at_1_max": 0.80774, + "nauc_recall_at_1_std": 0.141457, + "nauc_recall_at_1_diff1": 0.970261, + "nauc_recall_at_3_max": 0.947109, + "nauc_recall_at_3_std": 0.258364, + "nauc_recall_at_3_diff1": 0.959831, + "nauc_recall_at_5_max": 0.952003, + "nauc_recall_at_5_std": 0.371104, + "nauc_recall_at_5_diff1": 0.96732, + "nauc_recall_at_10_max": 0.937004, + "nauc_recall_at_10_std": 0.315184, + "nauc_recall_at_10_diff1": 0.957108, + "nauc_recall_at_20_max": 0.916005, + "nauc_recall_at_20_std": 0.248911, + "nauc_recall_at_20_diff1": 0.94281, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": -0.219888, + "nauc_recall_at_100_diff1": 0.956427, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.80774, + "nauc_precision_at_1_std": 0.141457, + "nauc_precision_at_1_diff1": 0.970261, + "nauc_precision_at_3_max": 0.947109, + "nauc_precision_at_3_std": 0.258364, + "nauc_precision_at_3_diff1": 0.959831, + "nauc_precision_at_5_max": 0.952003, + "nauc_precision_at_5_std": 0.371104, + "nauc_precision_at_5_diff1": 0.96732, + "nauc_precision_at_10_max": 0.937004, + "nauc_precision_at_10_std": 0.315184, + "nauc_precision_at_10_diff1": 0.957108, + "nauc_precision_at_20_max": 0.916005, + "nauc_precision_at_20_std": 0.248911, + "nauc_precision_at_20_diff1": 0.94281, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": -0.219888, + "nauc_precision_at_100_diff1": 0.956427, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.80774, + "nauc_mrr_at_1_std": 0.141457, + "nauc_mrr_at_1_diff1": 0.970261, + "nauc_mrr_at_3_max": 0.85618, + "nauc_mrr_at_3_std": 0.185596, + "nauc_mrr_at_3_diff1": 0.967032, + "nauc_mrr_at_5_max": 0.855248, + "nauc_mrr_at_5_std": 0.198163, + "nauc_mrr_at_5_diff1": 0.968324, + "nauc_mrr_at_10_max": 0.852297, + "nauc_mrr_at_10_std": 0.191926, + "nauc_mrr_at_10_diff1": 0.967678, + "nauc_mrr_at_20_max": 0.851123, + "nauc_mrr_at_20_std": 0.189664, + "nauc_mrr_at_20_diff1": 0.967421, + "nauc_mrr_at_100_max": 0.85127, + "nauc_mrr_at_100_std": 0.188436, + "nauc_mrr_at_100_diff1": 0.967853, + "nauc_mrr_at_1000_max": 0.851213, + "nauc_mrr_at_1000_std": 0.188633, + "nauc_mrr_at_1000_diff1": 0.967846, + "hit_rate_at_1": 0.96154, + "hit_rate_at_3": 0.98154, + "hit_rate_at_5": 0.98385, + "hit_rate_at_10": 0.98769, + "hit_rate_at_20": 0.99077, + "hit_rate_at_100": 0.99769, + "hit_rate_at_1000": 1.0, + "main_score": 0.97588, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 49.022589921951294, + "kg_co2_emissions": null, + "date": 1776146477.279985 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/ScalaClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/ScalaClassification.json new file mode 100644 index 0000000000..020236094a --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/ScalaClassification.json @@ -0,0 +1,524 @@ +{ + "dataset_revision": "ec85bb6c69679ed15ac66c0bf6e180bf563eb137", + "task_name": "ScalaClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.541016, + "f1": 0.539783, + "f1_weighted": 0.539783, + "precision": 0.54146, + "precision_weighted": 0.54146, + "recall": 0.541016, + "recall_weighted": 0.541016, + "ap": 0.522384, + "ap_weighted": 0.522384 + }, + { + "accuracy": 0.520996, + "f1": 0.52084, + "f1_weighted": 0.52084, + "precision": 0.521024, + "precision_weighted": 0.521024, + "recall": 0.520996, + "recall_weighted": 0.520996, + "ap": 0.510924, + "ap_weighted": 0.510924 + }, + { + "accuracy": 0.516113, + "f1": 0.5159, + "f1_weighted": 0.5159, + "precision": 0.516142, + "precision_weighted": 0.516142, + "recall": 0.516113, + "recall_weighted": 0.516113, + "ap": 0.508328, + "ap_weighted": 0.508328 + }, + { + "accuracy": 0.52832, + "f1": 0.526956, + "f1_weighted": 0.526956, + "precision": 0.528651, + "precision_weighted": 0.528651, + "recall": 0.52832, + "recall_weighted": 0.52832, + "ap": 0.514884, + "ap_weighted": 0.514884 + }, + { + "accuracy": 0.521973, + "f1": 0.520407, + "f1_weighted": 0.520407, + "precision": 0.522263, + "precision_weighted": 0.522263, + "recall": 0.521973, + "recall_weighted": 0.521973, + "ap": 0.511531, + "ap_weighted": 0.511531 + }, + { + "accuracy": 0.513184, + "f1": 0.512779, + "f1_weighted": 0.512779, + "precision": 0.513228, + "precision_weighted": 0.513228, + "recall": 0.513184, + "recall_weighted": 0.513184, + "ap": 0.506756, + "ap_weighted": 0.506756 + }, + { + "accuracy": 0.527832, + "f1": 0.527583, + "f1_weighted": 0.527583, + "precision": 0.527891, + "precision_weighted": 0.527891, + "recall": 0.527832, + "recall_weighted": 0.527832, + "ap": 0.514728, + "ap_weighted": 0.514728 + }, + { + "accuracy": 0.529297, + "f1": 0.528648, + "f1_weighted": 0.528648, + "precision": 0.529459, + "precision_weighted": 0.529459, + "recall": 0.529297, + "recall_weighted": 0.529297, + "ap": 0.515576, + "ap_weighted": 0.515576 + }, + { + "accuracy": 0.51416, + "f1": 0.514063, + "f1_weighted": 0.514063, + "precision": 0.514172, + "precision_weighted": 0.514172, + "recall": 0.51416, + "recall_weighted": 0.51416, + "ap": 0.507286, + "ap_weighted": 0.507286 + }, + { + "accuracy": 0.524902, + "f1": 0.524121, + "f1_weighted": 0.524121, + "precision": 0.525067, + "precision_weighted": 0.525067, + "recall": 0.524902, + "recall_weighted": 0.524902, + "ap": 0.513126, + "ap_weighted": 0.513126 + } + ], + "accuracy": 0.523779, + "f1": 0.523108, + "f1_weighted": 0.523108, + "precision": 0.523936, + "precision_weighted": 0.523936, + "recall": 0.523779, + "recall_weighted": 0.523779, + "ap": 0.512552, + "ap_weighted": 0.512552, + "main_score": 0.523779, + "hf_subset": "Danish", + "languages": [ + "dan-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.52002, + "f1": 0.519698, + "f1_weighted": 0.519698, + "precision": 0.520073, + "precision_weighted": 0.520073, + "recall": 0.52002, + "recall_weighted": 0.52002, + "ap": 0.510391, + "ap_weighted": 0.510391 + }, + { + "accuracy": 0.51709, + "f1": 0.500337, + "f1_weighted": 0.500337, + "precision": 0.519737, + "precision_weighted": 0.519737, + "recall": 0.51709, + "recall_weighted": 0.51709, + "ap": 0.509006, + "ap_weighted": 0.509006 + }, + { + "accuracy": 0.524902, + "f1": 0.524852, + "f1_weighted": 0.524852, + "precision": 0.524913, + "precision_weighted": 0.524913, + "recall": 0.524902, + "recall_weighted": 0.524902, + "ap": 0.513084, + "ap_weighted": 0.513084 + }, + { + "accuracy": 0.516602, + "f1": 0.512754, + "f1_weighted": 0.512754, + "precision": 0.517143, + "precision_weighted": 0.517143, + "recall": 0.516602, + "recall_weighted": 0.516602, + "ap": 0.508535, + "ap_weighted": 0.508535 + }, + { + "accuracy": 0.525391, + "f1": 0.524897, + "f1_weighted": 0.524897, + "precision": 0.525497, + "precision_weighted": 0.525497, + "recall": 0.525391, + "recall_weighted": 0.525391, + "ap": 0.513384, + "ap_weighted": 0.513384 + }, + { + "accuracy": 0.521484, + "f1": 0.518904, + "f1_weighted": 0.518904, + "precision": 0.521955, + "precision_weighted": 0.521955, + "recall": 0.521484, + "recall_weighted": 0.521484, + "ap": 0.511145, + "ap_weighted": 0.511145 + }, + { + "accuracy": 0.542969, + "f1": 0.542305, + "f1_weighted": 0.542305, + "precision": 0.54322, + "precision_weighted": 0.54322, + "recall": 0.542969, + "recall_weighted": 0.542969, + "ap": 0.5232, + "ap_weighted": 0.5232 + }, + { + "accuracy": 0.525391, + "f1": 0.52127, + "f1_weighted": 0.52127, + "precision": 0.526296, + "precision_weighted": 0.526296, + "recall": 0.525391, + "recall_weighted": 0.525391, + "ap": 0.513239, + "ap_weighted": 0.513239 + }, + { + "accuracy": 0.513672, + "f1": 0.513426, + "f1_weighted": 0.513426, + "precision": 0.5137, + "precision_weighted": 0.5137, + "recall": 0.513672, + "recall_weighted": 0.513672, + "ap": 0.507015, + "ap_weighted": 0.507015 + }, + { + "accuracy": 0.519531, + "f1": 0.518797, + "f1_weighted": 0.518797, + "precision": 0.519651, + "precision_weighted": 0.519651, + "recall": 0.519531, + "recall_weighted": 0.519531, + "ap": 0.510119, + "ap_weighted": 0.510119 + } + ], + "accuracy": 0.522705, + "f1": 0.519724, + "f1_weighted": 0.519724, + "precision": 0.523218, + "precision_weighted": 0.523218, + "recall": 0.522705, + "recall_weighted": 0.522705, + "ap": 0.511912, + "ap_weighted": 0.511912, + "main_score": 0.522705, + "hf_subset": "Norwegian_b", + "languages": [ + "nob-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.497559, + "f1": 0.496024, + "f1_weighted": 0.496024, + "precision": 0.497528, + "precision_weighted": 0.497528, + "recall": 0.497559, + "recall_weighted": 0.497559, + "ap": 0.498786, + "ap_weighted": 0.498786 + }, + { + "accuracy": 0.521484, + "f1": 0.520716, + "f1_weighted": 0.520716, + "precision": 0.521623, + "precision_weighted": 0.521623, + "recall": 0.521484, + "recall_weighted": 0.521484, + "ap": 0.51117, + "ap_weighted": 0.51117 + }, + { + "accuracy": 0.505859, + "f1": 0.505651, + "f1_weighted": 0.505651, + "precision": 0.505869, + "precision_weighted": 0.505869, + "recall": 0.505859, + "recall_weighted": 0.505859, + "ap": 0.502963, + "ap_weighted": 0.502963 + }, + { + "accuracy": 0.504883, + "f1": 0.5037, + "f1_weighted": 0.5037, + "precision": 0.50493, + "precision_weighted": 0.50493, + "recall": 0.504883, + "recall_weighted": 0.504883, + "ap": 0.502463, + "ap_weighted": 0.502463 + }, + { + "accuracy": 0.509766, + "f1": 0.509372, + "f1_weighted": 0.509372, + "precision": 0.509797, + "precision_weighted": 0.509797, + "recall": 0.509766, + "recall_weighted": 0.509766, + "ap": 0.504973, + "ap_weighted": 0.504973 + }, + { + "accuracy": 0.512207, + "f1": 0.503383, + "f1_weighted": 0.503383, + "precision": 0.513141, + "precision_weighted": 0.513141, + "recall": 0.512207, + "recall_weighted": 0.512207, + "ap": 0.506307, + "ap_weighted": 0.506307 + }, + { + "accuracy": 0.514648, + "f1": 0.514558, + "f1_weighted": 0.514558, + "precision": 0.514659, + "precision_weighted": 0.514659, + "recall": 0.514648, + "recall_weighted": 0.514648, + "ap": 0.507533, + "ap_weighted": 0.507533 + }, + { + "accuracy": 0.522949, + "f1": 0.522087, + "f1_weighted": 0.522087, + "precision": 0.523116, + "precision_weighted": 0.523116, + "recall": 0.522949, + "recall_weighted": 0.522949, + "ap": 0.51205, + "ap_weighted": 0.51205 + }, + { + "accuracy": 0.510742, + "f1": 0.505839, + "f1_weighted": 0.505839, + "precision": 0.511186, + "precision_weighted": 0.511186, + "recall": 0.510742, + "recall_weighted": 0.510742, + "ap": 0.505467, + "ap_weighted": 0.505467 + }, + { + "accuracy": 0.506348, + "f1": 0.505909, + "f1_weighted": 0.505909, + "precision": 0.50637, + "precision_weighted": 0.50637, + "recall": 0.506348, + "recall_weighted": 0.506348, + "ap": 0.503212, + "ap_weighted": 0.503212 + } + ], + "accuracy": 0.510645, + "f1": 0.508724, + "f1_weighted": 0.508724, + "precision": 0.510822, + "precision_weighted": 0.510822, + "recall": 0.510645, + "recall_weighted": 0.510645, + "ap": 0.505492, + "ap_weighted": 0.505492, + "main_score": 0.510645, + "hf_subset": "Norwegian_n", + "languages": [ + "nno-Latn" + ] + }, + { + "scores_per_experiment": [ + { + "accuracy": 0.540039, + "f1": 0.53812, + "f1_weighted": 0.53812, + "precision": 0.540716, + "precision_weighted": 0.540716, + "recall": 0.540039, + "recall_weighted": 0.540039, + "ap": 0.52186, + "ap_weighted": 0.52186 + }, + { + "accuracy": 0.514648, + "f1": 0.51358, + "f1_weighted": 0.51358, + "precision": 0.514778, + "precision_weighted": 0.514778, + "recall": 0.514648, + "recall_weighted": 0.514648, + "ap": 0.507561, + "ap_weighted": 0.507561 + }, + { + "accuracy": 0.540527, + "f1": 0.540435, + "f1_weighted": 0.540435, + "precision": 0.54056, + "precision_weighted": 0.54056, + "recall": 0.540527, + "recall_weighted": 0.540527, + "ap": 0.521954, + "ap_weighted": 0.521954 + }, + { + "accuracy": 0.522949, + "f1": 0.522309, + "f1_weighted": 0.522309, + "precision": 0.523073, + "precision_weighted": 0.523073, + "recall": 0.522949, + "recall_weighted": 0.522949, + "ap": 0.511965, + "ap_weighted": 0.511965 + }, + { + "accuracy": 0.523438, + "f1": 0.520728, + "f1_weighted": 0.520728, + "precision": 0.52398, + "precision_weighted": 0.52398, + "recall": 0.523438, + "recall_weighted": 0.523438, + "ap": 0.512196, + "ap_weighted": 0.512196 + }, + { + "accuracy": 0.519531, + "f1": 0.517465, + "f1_weighted": 0.517465, + "precision": 0.519872, + "precision_weighted": 0.519872, + "recall": 0.519531, + "recall_weighted": 0.519531, + "ap": 0.510205, + "ap_weighted": 0.510205 + }, + { + "accuracy": 0.532227, + "f1": 0.529281, + "f1_weighted": 0.529281, + "precision": 0.533054, + "precision_weighted": 0.533054, + "recall": 0.532227, + "recall_weighted": 0.532227, + "ap": 0.517347, + "ap_weighted": 0.517347 + }, + { + "accuracy": 0.521973, + "f1": 0.520461, + "f1_weighted": 0.520461, + "precision": 0.522253, + "precision_weighted": 0.522253, + "recall": 0.521973, + "recall_weighted": 0.521973, + "ap": 0.51142, + "ap_weighted": 0.51142 + }, + { + "accuracy": 0.527344, + "f1": 0.527228, + "f1_weighted": 0.527228, + "precision": 0.52737, + "precision_weighted": 0.52737, + "recall": 0.527344, + "recall_weighted": 0.527344, + "ap": 0.514444, + "ap_weighted": 0.514444 + }, + { + "accuracy": 0.534668, + "f1": 0.530142, + "f1_weighted": 0.530142, + "precision": 0.536057, + "precision_weighted": 0.536057, + "recall": 0.534668, + "recall_weighted": 0.534668, + "ap": 0.518339, + "ap_weighted": 0.518339 + } + ], + "accuracy": 0.527734, + "f1": 0.525975, + "f1_weighted": 0.525975, + "precision": 0.528171, + "precision_weighted": 0.528171, + "recall": 0.527734, + "recall_weighted": 0.527734, + "ap": 0.514729, + "ap_weighted": 0.514729, + "main_score": 0.527734, + "hf_subset": "Swedish", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 86.41133546829224, + "kg_co2_emissions": null, + "date": 1776146102.488614 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SweFaqRetrieval.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SweFaqRetrieval.json new file mode 100644 index 0000000000..502a42e319 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SweFaqRetrieval.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "208cb812631068a4bd8b93c8b9291370b47f282c", + "task_name": "SweFaqRetrieval", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.72904, + "ndcg_at_3": 0.82695, + "ndcg_at_5": 0.83828, + "ndcg_at_10": 0.85061, + "ndcg_at_20": 0.8552, + "ndcg_at_100": 0.85937, + "ndcg_at_1000": 0.85965, + "map_at_1": 0.72904, + "map_at_3": 0.80344, + "map_at_5": 0.80978, + "map_at_10": 0.81508, + "map_at_20": 0.81642, + "map_at_100": 0.81708, + "map_at_1000": 0.81709, + "recall_at_1": 0.72904, + "recall_at_3": 0.89474, + "recall_at_5": 0.92203, + "recall_at_10": 0.95906, + "recall_at_20": 0.97661, + "recall_at_100": 0.99805, + "recall_at_1000": 1.0, + "accuracy": 0.72904, + "precision_at_1": 0.72904, + "precision_at_3": 0.29825, + "precision_at_5": 0.18441, + "precision_at_10": 0.09591, + "precision_at_20": 0.04883, + "precision_at_100": 0.00998, + "precision_at_1000": 0.001, + "mrr_at_1": 0.729045, + "mrr_at_3": 0.803444, + "mrr_at_5": 0.809779, + "mrr_at_10": 0.815075, + "mrr_at_20": 0.816421, + "mrr_at_100": 0.817077, + "mrr_at_1000": 0.817092, + "nauc_ndcg_at_1_max": 0.445075, + "nauc_ndcg_at_1_std": -0.019588, + "nauc_ndcg_at_1_diff1": 0.751262, + "nauc_ndcg_at_3_max": 0.461394, + "nauc_ndcg_at_3_std": -0.053791, + "nauc_ndcg_at_3_diff1": 0.692311, + "nauc_ndcg_at_5_max": 0.45318, + "nauc_ndcg_at_5_std": -0.050855, + "nauc_ndcg_at_5_diff1": 0.7108, + "nauc_ndcg_at_10_max": 0.457361, + "nauc_ndcg_at_10_std": -0.054164, + "nauc_ndcg_at_10_diff1": 0.719484, + "nauc_ndcg_at_20_max": 0.456982, + "nauc_ndcg_at_20_std": -0.050443, + "nauc_ndcg_at_20_diff1": 0.720395, + "nauc_ndcg_at_100_max": 0.457777, + "nauc_ndcg_at_100_std": -0.046713, + "nauc_ndcg_at_100_diff1": 0.721397, + "nauc_ndcg_at_1000_max": 0.456875, + "nauc_ndcg_at_1000_std": -0.045581, + "nauc_ndcg_at_1000_diff1": 0.721073, + "nauc_map_at_1_max": 0.445075, + "nauc_map_at_1_std": -0.019588, + "nauc_map_at_1_diff1": 0.751262, + "nauc_map_at_3_max": 0.457168, + "nauc_map_at_3_std": -0.04463, + "nauc_map_at_3_diff1": 0.709119, + "nauc_map_at_5_max": 0.452921, + "nauc_map_at_5_std": -0.042611, + "nauc_map_at_5_diff1": 0.718794, + "nauc_map_at_10_max": 0.454813, + "nauc_map_at_10_std": -0.043202, + "nauc_map_at_10_diff1": 0.722371, + "nauc_map_at_20_max": 0.454815, + "nauc_map_at_20_std": -0.042027, + "nauc_map_at_20_diff1": 0.722607, + "nauc_map_at_100_max": 0.4551, + "nauc_map_at_100_std": -0.041767, + "nauc_map_at_100_diff1": 0.722869, + "nauc_map_at_1000_max": 0.455061, + "nauc_map_at_1000_std": -0.041718, + "nauc_map_at_1000_diff1": 0.722855, + "nauc_recall_at_1_max": 0.445075, + "nauc_recall_at_1_std": -0.019588, + "nauc_recall_at_1_diff1": 0.751262, + "nauc_recall_at_3_max": 0.481528, + "nauc_recall_at_3_std": -0.097707, + "nauc_recall_at_3_diff1": 0.610878, + "nauc_recall_at_5_max": 0.451929, + "nauc_recall_at_5_std": -0.101717, + "nauc_recall_at_5_diff1": 0.666531, + "nauc_recall_at_10_max": 0.483488, + "nauc_recall_at_10_std": -0.187253, + "nauc_recall_at_10_diff1": 0.707373, + "nauc_recall_at_20_max": 0.489068, + "nauc_recall_at_20_std": -0.219239, + "nauc_recall_at_20_diff1": 0.717837, + "nauc_recall_at_100_max": 0.869366, + "nauc_recall_at_100_std": -0.563045, + "nauc_recall_at_100_diff1": 0.869366, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.445075, + "nauc_precision_at_1_std": -0.019588, + "nauc_precision_at_1_diff1": 0.751262, + "nauc_precision_at_3_max": 0.481528, + "nauc_precision_at_3_std": -0.097707, + "nauc_precision_at_3_diff1": 0.610878, + "nauc_precision_at_5_max": 0.451929, + "nauc_precision_at_5_std": -0.101717, + "nauc_precision_at_5_diff1": 0.666531, + "nauc_precision_at_10_max": 0.483488, + "nauc_precision_at_10_std": -0.187253, + "nauc_precision_at_10_diff1": 0.707373, + "nauc_precision_at_20_max": 0.489068, + "nauc_precision_at_20_std": -0.219239, + "nauc_precision_at_20_diff1": 0.717837, + "nauc_precision_at_100_max": 0.869366, + "nauc_precision_at_100_std": -0.563045, + "nauc_precision_at_100_diff1": 0.869366, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.445075, + "nauc_mrr_at_1_std": -0.019588, + "nauc_mrr_at_1_diff1": 0.751262, + "nauc_mrr_at_3_max": 0.457168, + "nauc_mrr_at_3_std": -0.04463, + "nauc_mrr_at_3_diff1": 0.709119, + "nauc_mrr_at_5_max": 0.452921, + "nauc_mrr_at_5_std": -0.042611, + "nauc_mrr_at_5_diff1": 0.718794, + "nauc_mrr_at_10_max": 0.454813, + "nauc_mrr_at_10_std": -0.043202, + "nauc_mrr_at_10_diff1": 0.722371, + "nauc_mrr_at_20_max": 0.454815, + "nauc_mrr_at_20_std": -0.042027, + "nauc_mrr_at_20_diff1": 0.722607, + "nauc_mrr_at_100_max": 0.4551, + "nauc_mrr_at_100_std": -0.041767, + "nauc_mrr_at_100_diff1": 0.722869, + "nauc_mrr_at_1000_max": 0.455061, + "nauc_mrr_at_1000_std": -0.041718, + "nauc_mrr_at_1000_diff1": 0.722855, + "hit_rate_at_1": 0.72904, + "hit_rate_at_3": 0.89474, + "hit_rate_at_5": 0.92203, + "hit_rate_at_10": 0.95906, + "hit_rate_at_20": 0.97661, + "hit_rate_at_100": 0.99805, + "hit_rate_at_1000": 1.0, + "main_score": 0.85061, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 12.2958664894104, + "kg_co2_emissions": null, + "date": 1776146585.912684 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SweRecClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SweRecClassification.json new file mode 100644 index 0000000000..517e5d607c --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SweRecClassification.json @@ -0,0 +1,140 @@ +{ + "dataset_revision": "b07c6ce548f6a7ac8d546e1bbe197a0086409190", + "task_name": "SweRecClassification", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.858887, + "f1": 0.77709, + "f1_weighted": 0.861089, + "precision": 0.77394, + "precision_weighted": 0.864035, + "recall": 0.781169, + "recall_weighted": 0.858887, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.806152, + "f1": 0.750308, + "f1_weighted": 0.827406, + "precision": 0.753792, + "precision_weighted": 0.870368, + "recall": 0.789874, + "recall_weighted": 0.806152, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.837891, + "f1": 0.760153, + "f1_weighted": 0.846014, + "precision": 0.75533, + "precision_weighted": 0.861049, + "recall": 0.77463, + "recall_weighted": 0.837891, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.848633, + "f1": 0.776158, + "f1_weighted": 0.855432, + "precision": 0.76924, + "precision_weighted": 0.865227, + "recall": 0.789255, + "recall_weighted": 0.848633, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.824219, + "f1": 0.733321, + "f1_weighted": 0.830153, + "precision": 0.730806, + "precision_weighted": 0.837578, + "recall": 0.738798, + "recall_weighted": 0.824219, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.854004, + "f1": 0.750473, + "f1_weighted": 0.848514, + "precision": 0.759916, + "precision_weighted": 0.844711, + "recall": 0.744442, + "recall_weighted": 0.854004, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.832031, + "f1": 0.749223, + "f1_weighted": 0.836215, + "precision": 0.746142, + "precision_weighted": 0.842922, + "recall": 0.755055, + "recall_weighted": 0.832031, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.84082, + "f1": 0.776858, + "f1_weighted": 0.852417, + "precision": 0.769834, + "precision_weighted": 0.872648, + "recall": 0.801372, + "recall_weighted": 0.84082, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.772461, + "f1": 0.702272, + "f1_weighted": 0.791547, + "precision": 0.714438, + "precision_weighted": 0.842324, + "recall": 0.730901, + "recall_weighted": 0.772461, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.854004, + "f1": 0.773645, + "f1_weighted": 0.856909, + "precision": 0.769909, + "precision_weighted": 0.860319, + "recall": 0.778513, + "recall_weighted": 0.854004, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.83291, + "f1": 0.75495, + "f1_weighted": 0.84057, + "precision": 0.754335, + "precision_weighted": 0.856118, + "recall": 0.768401, + "recall_weighted": 0.83291, + "ap": NaN, + "ap_weighted": NaN, + "main_score": 0.83291, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 65.34958505630493, + "kg_co2_emissions": null, + "date": 1776146302.539216 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwedishSentimentClassification.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwedishSentimentClassification.json new file mode 100644 index 0000000000..ec47e89522 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwedishSentimentClassification.json @@ -0,0 +1,270 @@ +{ + "dataset_revision": "39e35f55d58338ebd602f8d83b52cfe027f5146a", + "task_name": "SwedishSentimentClassification", + "mteb_version": "2.12.16", + "scores": { + "validation": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.969238, + "f1": 0.969225, + "f1_weighted": 0.969227, + "precision": 0.969898, + "precision_weighted": 0.969849, + "recall": 0.969185, + "recall_weighted": 0.969238, + "ap": 0.947292, + "ap_weighted": 0.947292 + }, + { + "accuracy": 0.969238, + "f1": 0.969231, + "f1_weighted": 0.969233, + "precision": 0.969549, + "precision_weighted": 0.969516, + "recall": 0.969202, + "recall_weighted": 0.969238, + "ap": 0.949726, + "ap_weighted": 0.949726 + }, + { + "accuracy": 0.973145, + "f1": 0.973138, + "f1_weighted": 0.97314, + "precision": 0.973458, + "precision_weighted": 0.973425, + "recall": 0.973109, + "recall_weighted": 0.973145, + "ap": 0.955251, + "ap_weighted": 0.955251 + }, + { + "accuracy": 0.958008, + "f1": 0.957988, + "f1_weighted": 0.957991, + "precision": 0.958684, + "precision_weighted": 0.958635, + "recall": 0.957953, + "recall_weighted": 0.958008, + "ap": 0.931494, + "ap_weighted": 0.931494 + }, + { + "accuracy": 0.96875, + "f1": 0.968747, + "f1_weighted": 0.968748, + "precision": 0.968855, + "precision_weighted": 0.968836, + "recall": 0.96873, + "recall_weighted": 0.96875, + "ap": 0.951312, + "ap_weighted": 0.951312 + }, + { + "accuracy": 0.974121, + "f1": 0.974116, + "f1_weighted": 0.974117, + "precision": 0.97439, + "precision_weighted": 0.974359, + "recall": 0.974088, + "recall_weighted": 0.974121, + "ap": 0.957057, + "ap_weighted": 0.957057 + }, + { + "accuracy": 0.964355, + "f1": 0.96434, + "f1_weighted": 0.964342, + "precision": 0.965008, + "precision_weighted": 0.964959, + "recall": 0.964302, + "recall_weighted": 0.964355, + "ap": 0.940477, + "ap_weighted": 0.940477 + }, + { + "accuracy": 0.969727, + "f1": 0.969717, + "f1_weighted": 0.969719, + "precision": 0.970168, + "precision_weighted": 0.970128, + "recall": 0.969683, + "recall_weighted": 0.969727, + "ap": 0.949392, + "ap_weighted": 0.949392 + }, + { + "accuracy": 0.966309, + "f1": 0.966296, + "f1_weighted": 0.966298, + "precision": 0.966834, + "precision_weighted": 0.96679, + "recall": 0.966261, + "recall_weighted": 0.966309, + "ap": 0.943992, + "ap_weighted": 0.943992 + }, + { + "accuracy": 0.928223, + "f1": 0.927861, + "f1_weighted": 0.927876, + "precision": 0.936412, + "precision_weighted": 0.936239, + "recall": 0.928023, + "recall_weighted": 0.928223, + "ap": 0.875894, + "ap_weighted": 0.875894 + } + ], + "accuracy": 0.964111, + "f1": 0.964066, + "f1_weighted": 0.964069, + "precision": 0.965326, + "precision_weighted": 0.965273, + "recall": 0.964054, + "recall_weighted": 0.964111, + "ap": 0.940189, + "ap_weighted": 0.940189, + "main_score": 0.964111, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ], + "test": [ + { + "scores_per_experiment": [ + { + "accuracy": 0.972168, + "f1": 0.972159, + "f1_weighted": 0.97216, + "precision": 0.972687, + "precision_weighted": 0.972657, + "recall": 0.972136, + "recall_weighted": 0.972168, + "ap": 0.952168, + "ap_weighted": 0.952168 + }, + { + "accuracy": 0.965332, + "f1": 0.965328, + "f1_weighted": 0.965329, + "precision": 0.965474, + "precision_weighted": 0.965459, + "recall": 0.965316, + "recall_weighted": 0.965332, + "ap": 0.945792, + "ap_weighted": 0.945792 + }, + { + "accuracy": 0.974609, + "f1": 0.974609, + "f1_weighted": 0.974609, + "precision": 0.974645, + "precision_weighted": 0.974638, + "recall": 0.974602, + "recall_weighted": 0.974609, + "ap": 0.960908, + "ap_weighted": 0.960908 + }, + { + "accuracy": 0.95166, + "f1": 0.951652, + "f1_weighted": 0.951653, + "precision": 0.951906, + "precision_weighted": 0.951886, + "recall": 0.951638, + "recall_weighted": 0.95166, + "ap": 0.925521, + "ap_weighted": 0.925521 + }, + { + "accuracy": 0.969727, + "f1": 0.969726, + "f1_weighted": 0.969727, + "precision": 0.969726, + "precision_weighted": 0.969727, + "recall": 0.969726, + "recall_weighted": 0.969727, + "ap": 0.955621, + "ap_weighted": 0.955621 + }, + { + "accuracy": 0.976562, + "f1": 0.97656, + "f1_weighted": 0.976561, + "precision": 0.976693, + "precision_weighted": 0.976678, + "recall": 0.976547, + "recall_weighted": 0.976562, + "ap": 0.961987, + "ap_weighted": 0.961987 + }, + { + "accuracy": 0.969238, + "f1": 0.969227, + "f1_weighted": 0.969228, + "precision": 0.969816, + "precision_weighted": 0.969785, + "recall": 0.969205, + "recall_weighted": 0.969238, + "ap": 0.947638, + "ap_weighted": 0.947638 + }, + { + "accuracy": 0.971191, + "f1": 0.971189, + "f1_weighted": 0.97119, + "precision": 0.971278, + "precision_weighted": 0.971267, + "recall": 0.971179, + "recall_weighted": 0.971191, + "ap": 0.954941, + "ap_weighted": 0.954941 + }, + { + "accuracy": 0.96875, + "f1": 0.968748, + "f1_weighted": 0.968749, + "precision": 0.968824, + "precision_weighted": 0.968814, + "recall": 0.968738, + "recall_weighted": 0.96875, + "ap": 0.951673, + "ap_weighted": 0.951673 + }, + { + "accuracy": 0.932129, + "f1": 0.93185, + "f1_weighted": 0.931859, + "precision": 0.938972, + "precision_weighted": 0.938866, + "recall": 0.932008, + "recall_weighted": 0.932129, + "ap": 0.882405, + "ap_weighted": 0.882405 + } + ], + "accuracy": 0.965137, + "f1": 0.965105, + "f1_weighted": 0.965106, + "precision": 0.966002, + "precision_weighted": 0.965978, + "recall": 0.96511, + "recall_weighted": 0.965137, + "ap": 0.943865, + "ap_weighted": 0.943865, + "main_score": 0.965137, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 130.33137035369873, + "kg_co2_emissions": null, + "date": 1776146235.23792 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednClusteringP2P.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednClusteringP2P.json new file mode 100644 index 0000000000..968fa9560d --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednClusteringP2P.json @@ -0,0 +1,60 @@ +{ + "dataset_revision": "f8dbf10ec231cc25e9f63454d5cd2d90af95e5f8", + "task_name": "SwednClusteringP2P", + "mteb_version": "2.12.16", + "scores": { + "summaries": [ + { + "v_measures": { + "Level 0": [ + 0.388848, + 0.202132, + 0.411863, + 0.246308, + 0.351216, + 0.527238, + 0.299333, + 0.316714, + 0.480735, + 0.458231 + ] + }, + "v_measure": 0.368262, + "v_measure_std": 0.099254, + "main_score": 0.368262, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ], + "articles": [ + { + "v_measures": { + "Level 0": [ + 0.421149, + 0.454146, + 0.533303, + 0.4393, + 0.438641, + 0.514701, + 0.426535, + 0.565142, + 0.336769, + 0.279367 + ] + }, + "v_measure": 0.440905, + "v_measure_std": 0.081937, + "main_score": 0.440905, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 93.95308685302734, + "kg_co2_emissions": null, + "date": 1776146923.665985 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednClusteringS2S.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednClusteringS2S.json new file mode 100644 index 0000000000..8a945c1893 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednClusteringS2S.json @@ -0,0 +1,35 @@ +{ + "dataset_revision": "4dc1f92a8d5c4fe4be7995baa8009384f46d98d6", + "task_name": "SwednClusteringS2S", + "mteb_version": "2.12.16", + "scores": { + "headlines": [ + { + "v_measures": { + "Level 0": [ + 0.181848, + 0.1571, + 0.298969, + 0.253123, + 0.333963, + 0.266178, + 0.17488, + 0.214341, + 0.20778, + 0.20406 + ] + }, + "v_measure": 0.229224, + "v_measure_std": 0.0543, + "main_score": 0.229224, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 8.552327156066895, + "kg_co2_emissions": null, + "date": 1776146934.871101 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednRetrieval.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednRetrieval.json new file mode 100644 index 0000000000..801b4b7e31 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/SwednRetrieval.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "3920fa0e9ae78f6af3d8a20b33b39e1f9acdfdf9", + "task_name": "SwednRetrieval", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.8125, + "ndcg_at_3": 0.79514, + "ndcg_at_5": 0.82744, + "ndcg_at_10": 0.84569, + "ndcg_at_20": 0.85243, + "ndcg_at_100": 0.8604, + "ndcg_at_1000": 0.865, + "map_at_1": 0.40625, + "map_at_3": 0.75936, + "map_at_5": 0.7869, + "map_at_10": 0.79793, + "map_at_20": 0.80065, + "map_at_100": 0.80219, + "map_at_1000": 0.80242, + "recall_at_1": 0.40625, + "recall_at_3": 0.80273, + "recall_at_5": 0.86572, + "recall_at_10": 0.91162, + "recall_at_20": 0.93311, + "recall_at_100": 0.96826, + "recall_at_1000": 0.99805, + "accuracy": 0.40625, + "precision_at_1": 0.8125, + "precision_at_3": 0.53516, + "precision_at_5": 0.34629, + "precision_at_10": 0.18232, + "precision_at_20": 0.09331, + "precision_at_100": 0.01937, + "precision_at_1000": 0.002, + "mrr_at_1": 0.8125, + "mrr_at_3": 0.858561, + "mrr_at_5": 0.862858, + "mrr_at_10": 0.865457, + "mrr_at_20": 0.866205, + "mrr_at_100": 0.866735, + "mrr_at_1000": 0.866815, + "nauc_ndcg_at_1_max": 0.688198, + "nauc_ndcg_at_1_std": 0.192758, + "nauc_ndcg_at_1_diff1": 0.716849, + "nauc_ndcg_at_3_max": 0.646646, + "nauc_ndcg_at_3_std": 0.16457, + "nauc_ndcg_at_3_diff1": 0.470393, + "nauc_ndcg_at_5_max": 0.687169, + "nauc_ndcg_at_5_std": 0.207738, + "nauc_ndcg_at_5_diff1": 0.490083, + "nauc_ndcg_at_10_max": 0.69908, + "nauc_ndcg_at_10_std": 0.231027, + "nauc_ndcg_at_10_diff1": 0.519858, + "nauc_ndcg_at_20_max": 0.698457, + "nauc_ndcg_at_20_std": 0.244801, + "nauc_ndcg_at_20_diff1": 0.525387, + "nauc_ndcg_at_100_max": 0.694189, + "nauc_ndcg_at_100_std": 0.238429, + "nauc_ndcg_at_100_diff1": 0.530496, + "nauc_ndcg_at_1000_max": 0.684625, + "nauc_ndcg_at_1000_std": 0.216731, + "nauc_ndcg_at_1000_diff1": 0.526002, + "nauc_map_at_1_max": 0.688198, + "nauc_map_at_1_std": 0.192758, + "nauc_map_at_1_diff1": 0.716849, + "nauc_map_at_3_max": 0.609833, + "nauc_map_at_3_std": 0.111389, + "nauc_map_at_3_diff1": 0.432859, + "nauc_map_at_5_max": 0.639924, + "nauc_map_at_5_std": 0.141, + "nauc_map_at_5_diff1": 0.445588, + "nauc_map_at_10_max": 0.645288, + "nauc_map_at_10_std": 0.151122, + "nauc_map_at_10_diff1": 0.460596, + "nauc_map_at_20_max": 0.644533, + "nauc_map_at_20_std": 0.154784, + "nauc_map_at_20_diff1": 0.462473, + "nauc_map_at_100_max": 0.643874, + "nauc_map_at_100_std": 0.153668, + "nauc_map_at_100_diff1": 0.463496, + "nauc_map_at_1000_max": 0.643522, + "nauc_map_at_1000_std": 0.152912, + "nauc_map_at_1000_diff1": 0.463274, + "nauc_recall_at_1_max": 0.688198, + "nauc_recall_at_1_std": 0.192758, + "nauc_recall_at_1_diff1": 0.716849, + "nauc_recall_at_3_max": 0.647611, + "nauc_recall_at_3_std": 0.1844, + "nauc_recall_at_3_diff1": 0.403029, + "nauc_recall_at_5_max": 0.729307, + "nauc_recall_at_5_std": 0.288917, + "nauc_recall_at_5_diff1": 0.419395, + "nauc_recall_at_10_max": 0.809387, + "nauc_recall_at_10_std": 0.424903, + "nauc_recall_at_10_diff1": 0.514338, + "nauc_recall_at_20_max": 0.836259, + "nauc_recall_at_20_std": 0.573973, + "nauc_recall_at_20_diff1": 0.542415, + "nauc_recall_at_100_max": 0.947653, + "nauc_recall_at_100_std": 0.831687, + "nauc_recall_at_100_diff1": 0.65905, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.86523, + "nauc_precision_at_1_max": 0.688198, + "nauc_precision_at_1_std": 0.192758, + "nauc_precision_at_1_diff1": 0.716849, + "nauc_precision_at_3_max": 0.647611, + "nauc_precision_at_3_std": 0.1844, + "nauc_precision_at_3_diff1": 0.403029, + "nauc_precision_at_5_max": 0.729307, + "nauc_precision_at_5_std": 0.288917, + "nauc_precision_at_5_diff1": 0.419395, + "nauc_precision_at_10_max": 0.809387, + "nauc_precision_at_10_std": 0.424903, + "nauc_precision_at_10_diff1": 0.514338, + "nauc_precision_at_20_max": 0.836259, + "nauc_precision_at_20_std": 0.573973, + "nauc_precision_at_20_diff1": 0.542415, + "nauc_precision_at_100_max": 0.947653, + "nauc_precision_at_100_std": 0.831687, + "nauc_precision_at_100_diff1": 0.65905, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 0.86523, + "nauc_mrr_at_1_max": 0.688198, + "nauc_mrr_at_1_std": 0.192758, + "nauc_mrr_at_1_diff1": 0.716849, + "nauc_mrr_at_3_max": 0.751377, + "nauc_mrr_at_3_std": 0.296049, + "nauc_mrr_at_3_diff1": 0.703008, + "nauc_mrr_at_5_max": 0.749286, + "nauc_mrr_at_5_std": 0.295092, + "nauc_mrr_at_5_diff1": 0.702869, + "nauc_mrr_at_10_max": 0.746473, + "nauc_mrr_at_10_std": 0.293707, + "nauc_mrr_at_10_diff1": 0.703493, + "nauc_mrr_at_20_max": 0.745647, + "nauc_mrr_at_20_std": 0.292487, + "nauc_mrr_at_20_diff1": 0.703257, + "nauc_mrr_at_100_max": 0.745025, + "nauc_mrr_at_100_std": 0.291419, + "nauc_mrr_at_100_diff1": 0.703706, + "nauc_mrr_at_1000_max": 0.744864, + "nauc_mrr_at_1000_std": 0.291004, + "nauc_mrr_at_1000_diff1": 0.70369, + "hit_rate_at_1": 0.8125, + "hit_rate_at_3": 0.91504, + "hit_rate_at_5": 0.93262, + "hit_rate_at_10": 0.95215, + "hit_rate_at_20": 0.96191, + "hit_rate_at_100": 0.98438, + "hit_rate_at_1000": 1.0, + "main_score": 0.84569, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 77.1168360710144, + "kg_co2_emissions": null, + "date": 1776146563.955035 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/TV2Nordretrieval.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/TV2Nordretrieval.json new file mode 100644 index 0000000000..3b58030985 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/TV2Nordretrieval.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "80cdb115ec2ef46d4e926b252f2b59af62d6c070", + "task_name": "TV2Nordretrieval", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.9248, + "ndcg_at_3": 0.95236, + "ndcg_at_5": 0.95641, + "ndcg_at_10": 0.95894, + "ndcg_at_20": 0.96057, + "ndcg_at_100": 0.96142, + "ndcg_at_1000": 0.96153, + "map_at_1": 0.9248, + "map_at_3": 0.94596, + "map_at_5": 0.94823, + "map_at_10": 0.94928, + "map_at_20": 0.94974, + "map_at_100": 0.94987, + "map_at_1000": 0.94987, + "recall_at_1": 0.9248, + "recall_at_3": 0.9707, + "recall_at_5": 0.98047, + "recall_at_10": 0.98828, + "recall_at_20": 0.99463, + "recall_at_100": 0.99902, + "recall_at_1000": 1.0, + "accuracy": 0.9248, + "precision_at_1": 0.9248, + "precision_at_3": 0.32357, + "precision_at_5": 0.19609, + "precision_at_10": 0.09883, + "precision_at_20": 0.04973, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "mrr_at_1": 0.924805, + "mrr_at_3": 0.945964, + "mrr_at_5": 0.948234, + "mrr_at_10": 0.949278, + "mrr_at_20": 0.949743, + "mrr_at_100": 0.949869, + "mrr_at_1000": 0.949872, + "nauc_ndcg_at_1_max": 0.701461, + "nauc_ndcg_at_1_std": -0.366559, + "nauc_ndcg_at_1_diff1": 0.945749, + "nauc_ndcg_at_3_max": 0.785057, + "nauc_ndcg_at_3_std": -0.329485, + "nauc_ndcg_at_3_diff1": 0.949438, + "nauc_ndcg_at_5_max": 0.779567, + "nauc_ndcg_at_5_std": -0.329175, + "nauc_ndcg_at_5_diff1": 0.951357, + "nauc_ndcg_at_10_max": 0.768074, + "nauc_ndcg_at_10_std": -0.316978, + "nauc_ndcg_at_10_diff1": 0.949732, + "nauc_ndcg_at_20_max": 0.762087, + "nauc_ndcg_at_20_std": -0.319406, + "nauc_ndcg_at_20_diff1": 0.950134, + "nauc_ndcg_at_100_max": 0.757195, + "nauc_ndcg_at_100_std": -0.330995, + "nauc_ndcg_at_100_diff1": 0.949046, + "nauc_ndcg_at_1000_max": 0.756677, + "nauc_ndcg_at_1000_std": -0.334279, + "nauc_ndcg_at_1000_diff1": 0.948892, + "nauc_map_at_1_max": 0.701461, + "nauc_map_at_1_std": -0.366559, + "nauc_map_at_1_diff1": 0.945749, + "nauc_map_at_3_max": 0.75985, + "nauc_map_at_3_std": -0.343716, + "nauc_map_at_3_diff1": 0.948148, + "nauc_map_at_5_max": 0.75619, + "nauc_map_at_5_std": -0.344207, + "nauc_map_at_5_diff1": 0.948926, + "nauc_map_at_10_max": 0.75189, + "nauc_map_at_10_std": -0.34082, + "nauc_map_at_10_diff1": 0.948266, + "nauc_map_at_20_max": 0.750467, + "nauc_map_at_20_std": -0.341629, + "nauc_map_at_20_diff1": 0.948348, + "nauc_map_at_100_max": 0.749868, + "nauc_map_at_100_std": -0.34299, + "nauc_map_at_100_diff1": 0.948218, + "nauc_map_at_1000_max": 0.749859, + "nauc_map_at_1000_std": -0.343053, + "nauc_map_at_1000_diff1": 0.948215, + "nauc_recall_at_1_max": 0.701461, + "nauc_recall_at_1_std": -0.366559, + "nauc_recall_at_1_diff1": 0.945749, + "nauc_recall_at_3_max": 0.917058, + "nauc_recall_at_3_std": -0.252375, + "nauc_recall_at_3_diff1": 0.956346, + "nauc_recall_at_5_max": 0.953196, + "nauc_recall_at_5_std": -0.21201, + "nauc_recall_at_5_diff1": 0.970879, + "nauc_recall_at_10_max": 0.943783, + "nauc_recall_at_10_std": 0.007038, + "nauc_recall_at_10_diff1": 0.967807, + "nauc_recall_at_20_max": 0.97623, + "nauc_recall_at_20_std": 0.323682, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": 0.934633, + "nauc_recall_at_100_std": 0.777076, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.701461, + "nauc_precision_at_1_std": -0.366559, + "nauc_precision_at_1_diff1": 0.945749, + "nauc_precision_at_3_max": 0.917058, + "nauc_precision_at_3_std": -0.252375, + "nauc_precision_at_3_diff1": 0.956346, + "nauc_precision_at_5_max": 0.953196, + "nauc_precision_at_5_std": -0.21201, + "nauc_precision_at_5_diff1": 0.970879, + "nauc_precision_at_10_max": 0.943783, + "nauc_precision_at_10_std": 0.007038, + "nauc_precision_at_10_diff1": 0.967807, + "nauc_precision_at_20_max": 0.97623, + "nauc_precision_at_20_std": 0.323682, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 0.934633, + "nauc_precision_at_100_std": 0.777076, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.701461, + "nauc_mrr_at_1_std": -0.366559, + "nauc_mrr_at_1_diff1": 0.945749, + "nauc_mrr_at_3_max": 0.75985, + "nauc_mrr_at_3_std": -0.343716, + "nauc_mrr_at_3_diff1": 0.948148, + "nauc_mrr_at_5_max": 0.75619, + "nauc_mrr_at_5_std": -0.344207, + "nauc_mrr_at_5_diff1": 0.948926, + "nauc_mrr_at_10_max": 0.75189, + "nauc_mrr_at_10_std": -0.34082, + "nauc_mrr_at_10_diff1": 0.948266, + "nauc_mrr_at_20_max": 0.750467, + "nauc_mrr_at_20_std": -0.341629, + "nauc_mrr_at_20_diff1": 0.948348, + "nauc_mrr_at_100_max": 0.749868, + "nauc_mrr_at_100_std": -0.34299, + "nauc_mrr_at_100_diff1": 0.948218, + "nauc_mrr_at_1000_max": 0.749859, + "nauc_mrr_at_1000_std": -0.343053, + "nauc_mrr_at_1000_diff1": 0.948215, + "hit_rate_at_1": 0.9248, + "hit_rate_at_3": 0.9707, + "hit_rate_at_5": 0.98047, + "hit_rate_at_10": 0.98828, + "hit_rate_at_20": 0.99463, + "hit_rate_at_100": 0.99902, + "hit_rate_at_1000": 1.0, + "main_score": 0.95894, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 87.76099896430969, + "kg_co2_emissions": null, + "date": 1776146675.928014 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/TwitterHjerneRetrieval.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/TwitterHjerneRetrieval.json new file mode 100644 index 0000000000..91c44f1107 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/TwitterHjerneRetrieval.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "97ad55673cf9746f8e4b3aaa92b1bb92d82e52db", + "task_name": "TwitterHjerneRetrieval", + "mteb_version": "2.12.16", + "scores": { + "train": [ + { + "ndcg_at_1": 0.85714, + "ndcg_at_3": 0.74852, + "ndcg_at_5": 0.73574, + "ndcg_at_10": 0.7816, + "ndcg_at_20": 0.81789, + "ndcg_at_100": 0.83826, + "ndcg_at_1000": 0.8424, + "map_at_1": 0.27554, + "map_at_3": 0.56825, + "map_at_5": 0.64837, + "map_at_10": 0.69616, + "map_at_20": 0.71611, + "map_at_100": 0.72273, + "map_at_1000": 0.72319, + "recall_at_1": 0.27554, + "recall_at_3": 0.59134, + "recall_at_5": 0.69805, + "recall_at_10": 0.80346, + "recall_at_20": 0.90498, + "recall_at_100": 0.97792, + "recall_at_1000": 1.0, + "accuracy": 0.27554, + "precision_at_1": 0.85714, + "precision_at_3": 0.65801, + "precision_at_5": 0.48312, + "precision_at_10": 0.27922, + "precision_at_20": 0.15584, + "precision_at_100": 0.03351, + "precision_at_1000": 0.0034, + "mrr_at_1": 0.857143, + "mrr_at_3": 0.887446, + "mrr_at_5": 0.893939, + "mrr_at_10": 0.896104, + "mrr_at_20": 0.897908, + "mrr_at_100": 0.898341, + "mrr_at_1000": 0.898341, + "nauc_ndcg_at_1_max": 0.591502, + "nauc_ndcg_at_1_std": 0.070574, + "nauc_ndcg_at_1_diff1": 0.493405, + "nauc_ndcg_at_3_max": 0.577306, + "nauc_ndcg_at_3_std": 0.128907, + "nauc_ndcg_at_3_diff1": 0.284428, + "nauc_ndcg_at_5_max": 0.530427, + "nauc_ndcg_at_5_std": 0.092383, + "nauc_ndcg_at_5_diff1": 0.265513, + "nauc_ndcg_at_10_max": 0.538588, + "nauc_ndcg_at_10_std": 0.127659, + "nauc_ndcg_at_10_diff1": 0.246931, + "nauc_ndcg_at_20_max": 0.570051, + "nauc_ndcg_at_20_std": 0.170451, + "nauc_ndcg_at_20_diff1": 0.243403, + "nauc_ndcg_at_100_max": 0.603944, + "nauc_ndcg_at_100_std": 0.177508, + "nauc_ndcg_at_100_diff1": 0.272616, + "nauc_ndcg_at_1000_max": 0.594373, + "nauc_ndcg_at_1000_std": 0.160938, + "nauc_ndcg_at_1000_diff1": 0.278147, + "nauc_map_at_1_max": 0.013078, + "nauc_map_at_1_std": -0.218876, + "nauc_map_at_1_diff1": 0.421531, + "nauc_map_at_3_max": 0.373122, + "nauc_map_at_3_std": -0.033077, + "nauc_map_at_3_diff1": 0.346395, + "nauc_map_at_5_max": 0.490997, + "nauc_map_at_5_std": 0.033526, + "nauc_map_at_5_diff1": 0.277575, + "nauc_map_at_10_max": 0.50414, + "nauc_map_at_10_std": 0.073781, + "nauc_map_at_10_diff1": 0.248648, + "nauc_map_at_20_max": 0.530359, + "nauc_map_at_20_std": 0.094087, + "nauc_map_at_20_diff1": 0.253406, + "nauc_map_at_100_max": 0.540995, + "nauc_map_at_100_std": 0.098911, + "nauc_map_at_100_diff1": 0.25973, + "nauc_map_at_1000_max": 0.540177, + "nauc_map_at_1000_std": 0.097502, + "nauc_map_at_1000_diff1": 0.260022, + "nauc_recall_at_1_max": 0.013078, + "nauc_recall_at_1_std": -0.218876, + "nauc_recall_at_1_diff1": 0.421531, + "nauc_recall_at_3_max": 0.366505, + "nauc_recall_at_3_std": -0.026302, + "nauc_recall_at_3_diff1": 0.330458, + "nauc_recall_at_5_max": 0.477713, + "nauc_recall_at_5_std": 0.047084, + "nauc_recall_at_5_diff1": 0.212165, + "nauc_recall_at_10_max": 0.470885, + "nauc_recall_at_10_std": 0.146102, + "nauc_recall_at_10_diff1": 0.145962, + "nauc_recall_at_20_max": 0.459329, + "nauc_recall_at_20_std": 0.284369, + "nauc_recall_at_20_diff1": 0.016937, + "nauc_recall_at_100_max": 0.864353, + "nauc_recall_at_100_std": 0.647977, + "nauc_recall_at_100_diff1": 0.091014, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.591502, + "nauc_precision_at_1_std": 0.070574, + "nauc_precision_at_1_diff1": 0.493405, + "nauc_precision_at_3_max": 0.580805, + "nauc_precision_at_3_std": 0.232572, + "nauc_precision_at_3_diff1": 0.063239, + "nauc_precision_at_5_max": 0.503544, + "nauc_precision_at_5_std": 0.242714, + "nauc_precision_at_5_diff1": -0.05249, + "nauc_precision_at_10_max": 0.415796, + "nauc_precision_at_10_std": 0.273809, + "nauc_precision_at_10_diff1": -0.101688, + "nauc_precision_at_20_max": 0.381555, + "nauc_precision_at_20_std": 0.341414, + "nauc_precision_at_20_diff1": -0.168909, + "nauc_precision_at_100_max": 0.353934, + "nauc_precision_at_100_std": 0.348723, + "nauc_precision_at_100_diff1": -0.187112, + "nauc_precision_at_1000_max": 0.31634, + "nauc_precision_at_1000_std": 0.319788, + "nauc_precision_at_1000_diff1": -0.196854, + "nauc_mrr_at_1_max": 0.591502, + "nauc_mrr_at_1_std": 0.070574, + "nauc_mrr_at_1_diff1": 0.493405, + "nauc_mrr_at_3_max": 0.670991, + "nauc_mrr_at_3_std": 0.164383, + "nauc_mrr_at_3_diff1": 0.452884, + "nauc_mrr_at_5_max": 0.670492, + "nauc_mrr_at_5_std": 0.206314, + "nauc_mrr_at_5_diff1": 0.422678, + "nauc_mrr_at_10_max": 0.669379, + "nauc_mrr_at_10_std": 0.195286, + "nauc_mrr_at_10_diff1": 0.456576, + "nauc_mrr_at_20_max": 0.663387, + "nauc_mrr_at_20_std": 0.185358, + "nauc_mrr_at_20_diff1": 0.457187, + "nauc_mrr_at_100_max": 0.663128, + "nauc_mrr_at_100_std": 0.183011, + "nauc_mrr_at_100_diff1": 0.45676, + "nauc_mrr_at_1000_max": 0.663128, + "nauc_mrr_at_1000_std": 0.183011, + "nauc_mrr_at_1000_diff1": 0.45676, + "hit_rate_at_1": 0.85714, + "hit_rate_at_3": 0.92208, + "hit_rate_at_5": 0.94805, + "hit_rate_at_10": 0.96104, + "hit_rate_at_20": 0.98701, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.7816, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 4.3471150398254395, + "kg_co2_emissions": null, + "date": 1776146690.515156 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/VGHierarchicalClusteringP2P.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/VGHierarchicalClusteringP2P.json new file mode 100644 index 0000000000..61fedf4c76 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/VGHierarchicalClusteringP2P.json @@ -0,0 +1,47 @@ +{ + "dataset_revision": "d4c5a8ba10ae71224752c727094ac4c46947fa29", + "task_name": "VGHierarchicalClusteringP2P", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.457297, + 0.447199, + 0.462536, + 0.44769, + 0.445518, + 0.457842, + 0.449208, + 0.472493, + 0.463656, + 0.458271 + ], + "Level 1": [ + 0.468284, + 0.474966, + 0.455524, + 0.464757, + 0.463972, + 0.468347, + 0.457285, + 0.448552, + 0.453689, + 0.459809 + ] + }, + "v_measure": 0.458845, + "v_measure_std": 0.008381, + "main_score": 0.458845, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 75.99258279800415, + "kg_co2_emissions": null, + "date": 1776147037.207831 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/VGHierarchicalClusteringS2S.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/VGHierarchicalClusteringS2S.json new file mode 100644 index 0000000000..3d21034528 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/VGHierarchicalClusteringS2S.json @@ -0,0 +1,47 @@ +{ + "dataset_revision": "d4c5a8ba10ae71224752c727094ac4c46947fa29", + "task_name": "VGHierarchicalClusteringS2S", + "mteb_version": "2.12.16", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.427412, + 0.436669, + 0.430907, + 0.430456, + 0.416344, + 0.429033, + 0.433412, + 0.430251, + 0.445752, + 0.426133 + ], + "Level 1": [ + 0.412318, + 0.41582, + 0.405194, + 0.406072, + 0.404179, + 0.415427, + 0.421731, + 0.409169, + 0.415054, + 0.414465 + ] + }, + "v_measure": 0.42129, + "v_measure_std": 0.01128, + "main_score": 0.42129, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 22.587974071502686, + "kg_co2_emissions": null, + "date": 1776146959.594695 +} \ No newline at end of file diff --git a/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/model_meta.json b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/model_meta.json new file mode 100644 index 0000000000..de500336e2 --- /dev/null +++ b/results/nicher92__saga-embed_v1/3be07ac3d7c3e00e4402ae9285b23fcf8fda6735/model_meta.json @@ -0,0 +1,47 @@ +{ + "loader_kwargs": {}, + "name": "nicher92/saga-embed_v1", + "revision": "3be07ac3d7c3e00e4402ae9285b23fcf8fda6735", + "release_date": "2025-01-09", + "languages": [ + "dan-Latn", + "nor-Latn", + "swe-Latn" + ], + "n_parameters": 404219904, + "n_active_parameters_override": null, + "n_embedding_parameters": 51576832, + "memory_usage_mb": 2167.0, + "max_tokens": 512.0, + "embed_dim": 1024, + "license": "mit", + "open_weights": true, + "public_training_code": null, + "public_training_data": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/nicher92/saga-embed_v1", + "similarity_fn_name": "cosine", + "use_instructions": true, + "training_datasets": { + "RedditClustering": [], + "StackExchangeClustering": [], + "NQ": [] + }, + "adapted_from": "answerdotai/ModernBERT-base", + "superseded_by": null, + "modalities": [ + "text" + ], + "model_type": [ + "dense" + ], + "citation": null, + "contacts": null, + "experiment_kwargs": null, + "output_dtypes": null, + "loader": "SentenceTransformerEncoderWrapper", + "is_cross_encoder": false +}