From 40fc56f2d78e4cbf5e4ec3944a608dcd84a6e30d Mon Sep 17 00:00:00 2001 From: masader-bot Date: Wed, 8 Jul 2026 10:56:48 +0300 Subject: [PATCH 1/3] Creating datasets/absher.json --- datasets/absher.json | 96 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 datasets/absher.json diff --git a/datasets/absher.json b/datasets/absher.json new file mode 100644 index 00000000..53482473 --- /dev/null +++ b/datasets/absher.json @@ -0,0 +1,96 @@ +{ + "Name": "Absher", + "Dialect Subsets": [ + { + "Name": "Central", + "Volume": 5652.0, + "Unit": "sentences", + "Dialect": "Saudi Arabia" + }, + { + "Name": "Western", + "Volume": 5262.0, + "Unit": "sentences", + "Dialect": "Saudi Arabia" + }, + { + "Name": "Southern", + "Volume": 2682.0, + "Unit": "sentences", + "Dialect": "Saudi Arabia" + }, + { + "Name": "Northern", + "Volume": 414.0, + "Unit": "sentences", + "Dialect": "Saudi Arabia" + }, + { + "Name": "Eastern", + "Volume": 216.0, + "Unit": "sentences", + "Dialect": "Saudi Arabia" + }, + { + "Name": "General", + "Volume": 4338.0, + "Unit": "sentences", + "Dialect": "Saudi Arabia" + } + ], + "HF Link": "https://huggingface.co/datasets/Renad10/Absher-Benchmark", + "Link": "https://github.com/renad-01/Absher-Benchmark", + "License": "CC BY 4.0", + "Year": 2025, + "Language": "ar", + "Dialect": "Saudi Arabia", + "Source": [ + "web pages", + "manual construction" + ], + "Domain": [ + "culture", + "general" + ], + "Form": "text", + "Annotation Style": [ + "human validation", + "LLM annotation" + ], + "Description": "Absher is a culturally grounded benchmark designed to evaluate the ability of Large Language Models (LLMs) to understand Saudi Arabic dialects and their associated cultural knowledge", + "Volume": 18564.0, + "Unit": "sentences", + "Ethical Risks": "Low", + "Provider": [ + "King Khalid University" + ], + "Derived From": [], + "Paper Title": "From Words to Proverbs: Evaluating LLMs\u2019 Linguistic and Cultural Competence in Saudi Dialects with Absher", + "Paper Link": "https://arxiv.org/pdf/2507.10216.pdf", + "Script": "Arab", + "Tokenized": false, + "Host": "GitHub", + "Access": "Free", + "Cost": "", + "Has Splits": false, + "Partial": false, + "Tasks": [ + "multiple choice question answering" + ], + "Venue Title": "arXiv", + "Venue Type": "preprint", + "Venue Name": "arXiv", + "Authors": [ + "Renad Al-Monef", + "Hassan Alhuzali", + "Nora Alturayeif", + "Ashwag Alasmari" + ], + "Affiliations": [ + "King Khalid University", + "Umm Al-Qura University", + "Imam Abdulrahman Bin Faisal University" + ], + "Abstract": "As large language models (LLMs) become increasingly central to Arabic NLP applications, their effectiveness in linguistically diverse settings, particularly regions with rich dialectal variation such as Saudi Arabia, remains underexplored. Existing evaluation paradigms tend to prioritize high-resource languages or Modern Standard Arabic (MSA), overlooking regional linguistic and cultural specificities. This leads to performance limitations and cultural biases in real-world deployments. To address this gap, we introduce Absher, the first comprehensive and fine-grained benchmark designed to assess the understanding of LLMs regarding Saudi dialects and their embedded cultural nuances. Absher consists of over 18,000 multiple choice questions derived from a curated dataset of dialectal words, phrases, and proverbs sourced from five major Saudi regions. The benchmark spans six task categories: Meaning, True/False, Fill-in-the-Blank, Contextual Usage, Cultural Interpretation, and Location Recognition, enabling multifaceted evaluation across both linguistic and cultural dimensions. We perform zero-shot evaluations on six state-of-the-art open LLMs: ALLaM, LLaMA, Jais, Mistral, Qwen, and AceGPT. Our results reveal substantial performance variability across dialects and question types. Qwen achieved the highest overall accuracy, excelling in word-level questions (63%), while ALLaM outperformed others in the interpretation of proverbs (48% accuracy). All models struggled with content from underrepresented dialects, particularly Southern and Eastern variants, and with context-free True/False questions, highlighting weaknesses in dialect grounding and binary reasoning. These findings demonstrate the need for dialect-aware training and culturally aligned evaluation. We position Absher as a critical step towards more equitable and effective LLM development for real-world Arabic applications.", + "Added By": "Zaid Alyafeai" +} \ No newline at end of file From 5c26996dadc19306fa1ef706cd098efc95fd7860 Mon Sep 17 00:00:00 2001 From: Zaid Alyafeai Date: Wed, 8 Jul 2026 10:58:10 +0300 Subject: [PATCH 2/3] Update absher.json --- datasets/absher.json | 41 ++--------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/datasets/absher.json b/datasets/absher.json index 53482473..1afa7f4e 100644 --- a/datasets/absher.json +++ b/datasets/absher.json @@ -1,43 +1,6 @@ { "Name": "Absher", - "Dialect Subsets": [ - { - "Name": "Central", - "Volume": 5652.0, - "Unit": "sentences", - "Dialect": "Saudi Arabia" - }, - { - "Name": "Western", - "Volume": 5262.0, - "Unit": "sentences", - "Dialect": "Saudi Arabia" - }, - { - "Name": "Southern", - "Volume": 2682.0, - "Unit": "sentences", - "Dialect": "Saudi Arabia" - }, - { - "Name": "Northern", - "Volume": 414.0, - "Unit": "sentences", - "Dialect": "Saudi Arabia" - }, - { - "Name": "Eastern", - "Volume": 216.0, - "Unit": "sentences", - "Dialect": "Saudi Arabia" - }, - { - "Name": "General", - "Volume": 4338.0, - "Unit": "sentences", - "Dialect": "Saudi Arabia" - } - ], + "Dialect Subsets": [], "HF Link": "https://huggingface.co/datasets/Renad10/Absher-Benchmark", "Link": "https://github.com/renad-01/Absher-Benchmark", "License": "CC BY 4.0", @@ -93,4 +56,4 @@ ], "Abstract": "As large language models (LLMs) become increasingly central to Arabic NLP applications, their effectiveness in linguistically diverse settings, particularly regions with rich dialectal variation such as Saudi Arabia, remains underexplored. Existing evaluation paradigms tend to prioritize high-resource languages or Modern Standard Arabic (MSA), overlooking regional linguistic and cultural specificities. This leads to performance limitations and cultural biases in real-world deployments. To address this gap, we introduce Absher, the first comprehensive and fine-grained benchmark designed to assess the understanding of LLMs regarding Saudi dialects and their embedded cultural nuances. Absher consists of over 18,000 multiple choice questions derived from a curated dataset of dialectal words, phrases, and proverbs sourced from five major Saudi regions. The benchmark spans six task categories: Meaning, True/False, Fill-in-the-Blank, Contextual Usage, Cultural Interpretation, and Location Recognition, enabling multifaceted evaluation across both linguistic and cultural dimensions. We perform zero-shot evaluations on six state-of-the-art open LLMs: ALLaM, LLaMA, Jais, Mistral, Qwen, and AceGPT. Our results reveal substantial performance variability across dialects and question types. Qwen achieved the highest overall accuracy, excelling in word-level questions (63%), while ALLaM outperformed others in the interpretation of proverbs (48% accuracy). All models struggled with content from underrepresented dialects, particularly Southern and Eastern variants, and with context-free True/False questions, highlighting weaknesses in dialect grounding and binary reasoning. These findings demonstrate the need for dialect-aware training and culturally aligned evaluation. We position Absher as a critical step towards more equitable and effective LLM development for real-world Arabic applications.", "Added By": "Zaid Alyafeai" -} \ No newline at end of file +} From 75b1dbb6fdb7e5d20f554c3c981a7ac21a905e88 Mon Sep 17 00:00:00 2001 From: Zaid Alyafeai Date: Wed, 8 Jul 2026 10:58:48 +0300 Subject: [PATCH 3/3] Update absher.json --- datasets/absher.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasets/absher.json b/datasets/absher.json index 1afa7f4e..dd2cd957 100644 --- a/datasets/absher.json +++ b/datasets/absher.json @@ -40,9 +40,9 @@ "Tasks": [ "multiple choice question answering" ], - "Venue Title": "arXiv", - "Venue Type": "preprint", - "Venue Name": "arXiv", + "Venue Title": "AEJ", + "Venue Type": "journal", + "Venue Name": "Alexandria Engineering Journal", "Authors": [ "Renad Al-Monef", "Hassan Alhuzali",