@inproceedings{qandos-etal-2026-safir,
title = "{SAFIR}: A Sub-Dialectal Benchmark for Saudi {A}rabic{--}{E}nglish Machine Translation",
author = "Qandos, Nour and
Nacar, Omer and
Hamod, Aya and
Ahmad, Samar and
Al Hallay, Rahaf Saeed and
Al Rabghi, Ahmad",
editor = "Abdul-Mageed, Muhammad and
Elsayed, Tamer and
Bouamor, Houda and
El-Beltagy, Samhaa and
Al-Khalifa, Hend and
Abdelali, Ahmed and
Elmadany, AbdelRahim and
El Mekki, Abdellah and
Darwish, Kareem and
Alkhereyf, Sakhar and
Luqman, Hamzah and
Khalifa, Salam and
Magdy, Samar and
Alyafeai, Zaid and
Hamed, Injy and
Zaghouani, Wajdi and
Zitouni, Imed",
booktitle = "Proceedings of The Fourth {A}rabic Natural Language Processing Conference",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.arabicnlp-main.20/",
pages = "289--301",
ISBN = "979-8-89176-473-6",
abstract = "Arabic machine translation benchmarks provide limited coverage of variation within Saudi Arabic. We introduce SAFIR, a Saudi-centric benchmark of 610 unique sentence pairs organized into 12 creator specific evaluation splits across five regional groups: Najdi, Hijazi, Southern, Northern, and Eastern. Native speakers authored the Arabic sentences and their English translations; 235 pairs received independent double review for dialect authenticity, translation adequacy, and fluency. We evaluate twelve systems, comprising losed-weight LLMs, open-weight LLMs, and dedicated MT models, in the zero-shot Saudi$\rightarrow$English setting using reference-based metrics and GPT-4o judgments. Closed-weight LLMs obtain the strongest aggregate results. Gemini 2.5 Pro achieves the highest average BLEU (34.41), while GPT-5 achieves the highest BERTScore (83.60). Rankings vary across splits, and Southern varieties present particular challenges in judged adequacy. Differences between reference overlap, adequacy, and fluency support reporting performance by sampled variety and evaluation dimension. SAFIR and its leaderboard support more detailed assessment of Saudi Arabic translation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="qandos-etal-2026-safir">
<titleInfo>
<title>SAFIR: A Sub-Dialectal Benchmark for Saudi Arabic–English Machine Translation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nour</namePart>
<namePart type="family">Qandos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Nacar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aya</namePart>
<namePart type="family">Hamod</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samar</namePart>
<namePart type="family">Ahmad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rahaf</namePart>
<namePart type="given">Saeed</namePart>
<namePart type="family">Al Hallay</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ahmad</namePart>
<namePart type="family">Al Rabghi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of The Fourth Arabic Natural Language Processing Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="family">Abdul-Mageed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tamer</namePart>
<namePart type="family">Elsayed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Houda</namePart>
<namePart type="family">Bouamor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samhaa</namePart>
<namePart type="family">El-Beltagy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hend</namePart>
<namePart type="family">Al-Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ahmed</namePart>
<namePart type="family">Abdelali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">AbdelRahim</namePart>
<namePart type="family">Elmadany</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abdellah</namePart>
<namePart type="family">El Mekki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kareem</namePart>
<namePart type="family">Darwish</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sakhar</namePart>
<namePart type="family">Alkhereyf</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hamzah</namePart>
<namePart type="family">Luqman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salam</namePart>
<namePart type="family">Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samar</namePart>
<namePart type="family">Magdy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zaid</namePart>
<namePart type="family">Alyafeai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Injy</namePart>
<namePart type="family">Hamed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wajdi</namePart>
<namePart type="family">Zaghouani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Imed</namePart>
<namePart type="family">Zitouni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-473-6</identifier>
</relatedItem>
<abstract>Arabic machine translation benchmarks provide limited coverage of variation within Saudi Arabic. We introduce SAFIR, a Saudi-centric benchmark of 610 unique sentence pairs organized into 12 creator specific evaluation splits across five regional groups: Najdi, Hijazi, Southern, Northern, and Eastern. Native speakers authored the Arabic sentences and their English translations; 235 pairs received independent double review for dialect authenticity, translation adequacy, and fluency. We evaluate twelve systems, comprising losed-weight LLMs, open-weight LLMs, and dedicated MT models, in the zero-shot Saudi\rightarrowEnglish setting using reference-based metrics and GPT-4o judgments. Closed-weight LLMs obtain the strongest aggregate results. Gemini 2.5 Pro achieves the highest average BLEU (34.41), while GPT-5 achieves the highest BERTScore (83.60). Rankings vary across splits, and Southern varieties present particular challenges in judged adequacy. Differences between reference overlap, adequacy, and fluency support reporting performance by sampled variety and evaluation dimension. SAFIR and its leaderboard support more detailed assessment of Saudi Arabic translation.</abstract>
<identifier type="citekey">qandos-etal-2026-safir</identifier>
<location>
<url>https://aclanthology.org/2026.arabicnlp-main.20/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>289</start>
<end>301</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SAFIR: A Sub-Dialectal Benchmark for Saudi Arabic–English Machine Translation
%A Qandos, Nour
%A Nacar, Omer
%A Hamod, Aya
%A Ahmad, Samar
%A Al Hallay, Rahaf Saeed
%A Al Rabghi, Ahmad
%Y Abdul-Mageed, Muhammad
%Y Elsayed, Tamer
%Y Bouamor, Houda
%Y El-Beltagy, Samhaa
%Y Al-Khalifa, Hend
%Y Abdelali, Ahmed
%Y Elmadany, AbdelRahim
%Y El Mekki, Abdellah
%Y Darwish, Kareem
%Y Alkhereyf, Sakhar
%Y Luqman, Hamzah
%Y Khalifa, Salam
%Y Magdy, Samar
%Y Alyafeai, Zaid
%Y Hamed, Injy
%Y Zaghouani, Wajdi
%Y Zitouni, Imed
%S Proceedings of The Fourth Arabic Natural Language Processing Conference
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-473-6
%F qandos-etal-2026-safir
%X Arabic machine translation benchmarks provide limited coverage of variation within Saudi Arabic. We introduce SAFIR, a Saudi-centric benchmark of 610 unique sentence pairs organized into 12 creator specific evaluation splits across five regional groups: Najdi, Hijazi, Southern, Northern, and Eastern. Native speakers authored the Arabic sentences and their English translations; 235 pairs received independent double review for dialect authenticity, translation adequacy, and fluency. We evaluate twelve systems, comprising losed-weight LLMs, open-weight LLMs, and dedicated MT models, in the zero-shot Saudi\rightarrowEnglish setting using reference-based metrics and GPT-4o judgments. Closed-weight LLMs obtain the strongest aggregate results. Gemini 2.5 Pro achieves the highest average BLEU (34.41), while GPT-5 achieves the highest BERTScore (83.60). Rankings vary across splits, and Southern varieties present particular challenges in judged adequacy. Differences between reference overlap, adequacy, and fluency support reporting performance by sampled variety and evaluation dimension. SAFIR and its leaderboard support more detailed assessment of Saudi Arabic translation.
%U https://aclanthology.org/2026.arabicnlp-main.20/
%P 289-301
Markdown (Informal)
[SAFIR: A Sub-Dialectal Benchmark for Saudi Arabic–English Machine Translation](https://aclanthology.org/2026.arabicnlp-main.20/) (Qandos et al., ArabicNLP 2026)
ACL