@inproceedings{issa-rahmouni-2026-cultural,
title = "Cultural Understanding in Large Language Models: A Comparative Study with Human Judgments",
author = "Issa, Elsayed and
Rahmouni, Kamilia",
editor = "Abdul-Mageed, Muhammad and
Elsayed, Tamer and
Bouamor, Houda and
El-Beltagy, Samhaa and
Al-Khalifa, Hend and
Abdelali, Ahmed and
Elmadany, AbdelRahim and
El Mekki, Abdellah and
Darwish, Kareem and
Alkhereyf, Sakhar and
Luqman, Hamzah and
Khalifa, Salam and
Magdy, Samar and
Alyafeai, Zaid and
Hamed, Injy and
Zaghouani, Wajdi and
Zitouni, Imed",
booktitle = "Proceedings of The Fourth {A}rabic Natural Language Processing Conference",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.arabicnlp-main.15/",
pages = "200--218",
ISBN = "979-8-89176-473-6",
abstract = "This paper evaluates whether large language models (LLMs) can demonstrate cultural competence in Egyptian Arabic{---}not just factual knowledge about culture, but the ability to choose culturally appropriate idiomatic and figurative expressions in everyday social situations. 67 Egyptian participants completed a multiple-choice questionnaire of 90 everyday scenarios to provide human gold labels. Using the same scenarios, each pairing a culturally appropriate Egyptian idiom against a plausible distractor, we evaluated 11 open-source and 3 proprietary models under five prompts and two answer orderings. The results indicated that humans performed near ceiling (98.2{\%}), while LLMs averaged 68.2{\%}. The results also showed that a single accuracy figure conflates three separable behaviors: cultural competence, an answer-position heuristic, and abstention. These behaviors are distributed so unevenly that the model ranking reorders depending on which one is measured. One model{'}s apparent mid-tier accuracy is almost entirely a ``pick the first option'' heuristic (a 0.75 normal{--}reversed gap), and another falls from 51.8{\%} to below chance once refusals to answer are scored as errors. We argue that cultural-competence evaluation should report this decomposition rather than a headline accuracy."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="issa-rahmouni-2026-cultural">
<titleInfo>
<title>Cultural Understanding in Large Language Models: A Comparative Study with Human Judgments</title>
</titleInfo>
<name type="personal">
<namePart type="given">Elsayed</namePart>
<namePart type="family">Issa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kamilia</namePart>
<namePart type="family">Rahmouni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of The Fourth Arabic Natural Language Processing Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="family">Abdul-Mageed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tamer</namePart>
<namePart type="family">Elsayed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Houda</namePart>
<namePart type="family">Bouamor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samhaa</namePart>
<namePart type="family">El-Beltagy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hend</namePart>
<namePart type="family">Al-Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ahmed</namePart>
<namePart type="family">Abdelali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">AbdelRahim</namePart>
<namePart type="family">Elmadany</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abdellah</namePart>
<namePart type="family">El Mekki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kareem</namePart>
<namePart type="family">Darwish</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sakhar</namePart>
<namePart type="family">Alkhereyf</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hamzah</namePart>
<namePart type="family">Luqman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salam</namePart>
<namePart type="family">Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samar</namePart>
<namePart type="family">Magdy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zaid</namePart>
<namePart type="family">Alyafeai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Injy</namePart>
<namePart type="family">Hamed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wajdi</namePart>
<namePart type="family">Zaghouani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Imed</namePart>
<namePart type="family">Zitouni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-473-6</identifier>
</relatedItem>
<abstract>This paper evaluates whether large language models (LLMs) can demonstrate cultural competence in Egyptian Arabic—not just factual knowledge about culture, but the ability to choose culturally appropriate idiomatic and figurative expressions in everyday social situations. 67 Egyptian participants completed a multiple-choice questionnaire of 90 everyday scenarios to provide human gold labels. Using the same scenarios, each pairing a culturally appropriate Egyptian idiom against a plausible distractor, we evaluated 11 open-source and 3 proprietary models under five prompts and two answer orderings. The results indicated that humans performed near ceiling (98.2%), while LLMs averaged 68.2%. The results also showed that a single accuracy figure conflates three separable behaviors: cultural competence, an answer-position heuristic, and abstention. These behaviors are distributed so unevenly that the model ranking reorders depending on which one is measured. One model’s apparent mid-tier accuracy is almost entirely a “pick the first option” heuristic (a 0.75 normal–reversed gap), and another falls from 51.8% to below chance once refusals to answer are scored as errors. We argue that cultural-competence evaluation should report this decomposition rather than a headline accuracy.</abstract>
<identifier type="citekey">issa-rahmouni-2026-cultural</identifier>
<location>
<url>https://aclanthology.org/2026.arabicnlp-main.15/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>200</start>
<end>218</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Cultural Understanding in Large Language Models: A Comparative Study with Human Judgments
%A Issa, Elsayed
%A Rahmouni, Kamilia
%Y Abdul-Mageed, Muhammad
%Y Elsayed, Tamer
%Y Bouamor, Houda
%Y El-Beltagy, Samhaa
%Y Al-Khalifa, Hend
%Y Abdelali, Ahmed
%Y Elmadany, AbdelRahim
%Y El Mekki, Abdellah
%Y Darwish, Kareem
%Y Alkhereyf, Sakhar
%Y Luqman, Hamzah
%Y Khalifa, Salam
%Y Magdy, Samar
%Y Alyafeai, Zaid
%Y Hamed, Injy
%Y Zaghouani, Wajdi
%Y Zitouni, Imed
%S Proceedings of The Fourth Arabic Natural Language Processing Conference
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-473-6
%F issa-rahmouni-2026-cultural
%X This paper evaluates whether large language models (LLMs) can demonstrate cultural competence in Egyptian Arabic—not just factual knowledge about culture, but the ability to choose culturally appropriate idiomatic and figurative expressions in everyday social situations. 67 Egyptian participants completed a multiple-choice questionnaire of 90 everyday scenarios to provide human gold labels. Using the same scenarios, each pairing a culturally appropriate Egyptian idiom against a plausible distractor, we evaluated 11 open-source and 3 proprietary models under five prompts and two answer orderings. The results indicated that humans performed near ceiling (98.2%), while LLMs averaged 68.2%. The results also showed that a single accuracy figure conflates three separable behaviors: cultural competence, an answer-position heuristic, and abstention. These behaviors are distributed so unevenly that the model ranking reorders depending on which one is measured. One model’s apparent mid-tier accuracy is almost entirely a “pick the first option” heuristic (a 0.75 normal–reversed gap), and another falls from 51.8% to below chance once refusals to answer are scored as errors. We argue that cultural-competence evaluation should report this decomposition rather than a headline accuracy.
%U https://aclanthology.org/2026.arabicnlp-main.15/
%P 200-218
Markdown (Informal)
[Cultural Understanding in Large Language Models: A Comparative Study with Human Judgments](https://aclanthology.org/2026.arabicnlp-main.15/) (Issa & Rahmouni, ArabicNLP 2026)
ACL