@inproceedings{zaghouani-etal-2026-qatarguard,
title = "{Q}atar{G}uard: A Culturally-Aware Safety Benchmark for {A}rabic and Multilingual {LLM}s",
author = "Zaghouani, Wajdi and
Aldous, Kholoud Khalil and
Biswas, Md. Rafiul and
Abdelkreem, Mohammed",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1710/",
pages = "36539--36550",
ISBN = "979-8-89176-462-0",
abstract = "We introduce QatarGuard, an Arabic safety evaluation dataset containing 3,289 human curated prompts across eleven categories: Self-Harm, Violence, Child Exploitation, Vulgar Language, Radicalization, Sexual Language, Racist Content, Regulated Goods and Illegal Activities, Education, Family, and Health. We evaluate how safely five large language models respond to prompts: 299 from the Self-Harm category and 450 from three additional categories: Radicalization, Sexual Language, and Violence. The models tested include four commercial models (Claude 4.5 Sonnet, GPT-4.1, GPT-3.5 Turbo, and Google Gemini 2.5 Flash Lite) and one open-source Arabic-centric model (Fanar-2-27B-Instruct). We use GPT-5 as an automated judge, supported by manual review, to identify failure modes specific to Qatari Arabic dialect. For Self-Harm, unsafe response rates ranged from 2.34{\%} (Gemini 2.5 Flash Lite) to 6.02{\%} (GPT-3.5 Turbo), with Fanar-2-27B-Instruct at 4.35{\%}. Across the other three categories, results varied more widely, from 2.91{\%} to 13.48{\%}. Sexual Language was the most challenging category, where even the best-performing model gave unsafe responses to 5.3{\%} of prompts. Our findings show that Arabic dialect influences model behavior in safety-sensitive contexts, and that overall safety scores can hide weaknesses in specific categories. This highlights the need for safety evaluations that account for both dialect and category. The dataset is available for non-commercial research upon request, and evaluation scripts will be publicly released upon acceptance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zaghouani-etal-2026-qatarguard">
<titleInfo>
<title>QatarGuard: A Culturally-Aware Safety Benchmark for Arabic and Multilingual LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wajdi</namePart>
<namePart type="family">Zaghouani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kholoud</namePart>
<namePart type="given">Khalil</namePart>
<namePart type="family">Aldous</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Md.</namePart>
<namePart type="given">Rafiul</namePart>
<namePart type="family">Biswas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohammed</namePart>
<namePart type="family">Abdelkreem</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>We introduce QatarGuard, an Arabic safety evaluation dataset containing 3,289 human curated prompts across eleven categories: Self-Harm, Violence, Child Exploitation, Vulgar Language, Radicalization, Sexual Language, Racist Content, Regulated Goods and Illegal Activities, Education, Family, and Health. We evaluate how safely five large language models respond to prompts: 299 from the Self-Harm category and 450 from three additional categories: Radicalization, Sexual Language, and Violence. The models tested include four commercial models (Claude 4.5 Sonnet, GPT-4.1, GPT-3.5 Turbo, and Google Gemini 2.5 Flash Lite) and one open-source Arabic-centric model (Fanar-2-27B-Instruct). We use GPT-5 as an automated judge, supported by manual review, to identify failure modes specific to Qatari Arabic dialect. For Self-Harm, unsafe response rates ranged from 2.34% (Gemini 2.5 Flash Lite) to 6.02% (GPT-3.5 Turbo), with Fanar-2-27B-Instruct at 4.35%. Across the other three categories, results varied more widely, from 2.91% to 13.48%. Sexual Language was the most challenging category, where even the best-performing model gave unsafe responses to 5.3% of prompts. Our findings show that Arabic dialect influences model behavior in safety-sensitive contexts, and that overall safety scores can hide weaknesses in specific categories. This highlights the need for safety evaluations that account for both dialect and category. The dataset is available for non-commercial research upon request, and evaluation scripts will be publicly released upon acceptance.</abstract>
<identifier type="citekey">zaghouani-etal-2026-qatarguard</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1710/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>36539</start>
<end>36550</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T QatarGuard: A Culturally-Aware Safety Benchmark for Arabic and Multilingual LLMs
%A Zaghouani, Wajdi
%A Aldous, Kholoud Khalil
%A Biswas, Md. Rafiul
%A Abdelkreem, Mohammed
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zaghouani-etal-2026-qatarguard
%X We introduce QatarGuard, an Arabic safety evaluation dataset containing 3,289 human curated prompts across eleven categories: Self-Harm, Violence, Child Exploitation, Vulgar Language, Radicalization, Sexual Language, Racist Content, Regulated Goods and Illegal Activities, Education, Family, and Health. We evaluate how safely five large language models respond to prompts: 299 from the Self-Harm category and 450 from three additional categories: Radicalization, Sexual Language, and Violence. The models tested include four commercial models (Claude 4.5 Sonnet, GPT-4.1, GPT-3.5 Turbo, and Google Gemini 2.5 Flash Lite) and one open-source Arabic-centric model (Fanar-2-27B-Instruct). We use GPT-5 as an automated judge, supported by manual review, to identify failure modes specific to Qatari Arabic dialect. For Self-Harm, unsafe response rates ranged from 2.34% (Gemini 2.5 Flash Lite) to 6.02% (GPT-3.5 Turbo), with Fanar-2-27B-Instruct at 4.35%. Across the other three categories, results varied more widely, from 2.91% to 13.48%. Sexual Language was the most challenging category, where even the best-performing model gave unsafe responses to 5.3% of prompts. Our findings show that Arabic dialect influences model behavior in safety-sensitive contexts, and that overall safety scores can hide weaknesses in specific categories. This highlights the need for safety evaluations that account for both dialect and category. The dataset is available for non-commercial research upon request, and evaluation scripts will be publicly released upon acceptance.
%U https://aclanthology.org/2026.findings-emnlp.1710/
%P 36539-36550
Markdown (Informal)
[QatarGuard: A Culturally-Aware Safety Benchmark for Arabic and Multilingual LLMs](https://aclanthology.org/2026.findings-emnlp.1710/) (Zaghouani et al., Findings 2026)
ACL