@inproceedings{alsagheer-etal-2026-legal,
title = "Same Legal Facts, Different Verdicts: Evaluating Cross-Lingual and Dialectal Robustness in {LLM} Legal Reasoning",
author = "Alsagheer, Dana R and
Kamal, Mohammad and
Kamal, Abdulrahman and
Lu, Yang and
Diallo, Nour and
Shi, Weidong and
Kamal, Omar",
editor = "Abdul-Mageed, Muhammad and
Elsayed, Tamer and
Bouamor, Houda and
El-Beltagy, Samhaa and
Al-Khalifa, Hend and
Abdelali, Ahmed and
Elmadany, AbdelRahim and
El Mekki, Abdellah and
Darwish, Kareem and
Alkhereyf, Sakhar and
Luqman, Hamzah and
Khalifa, Salam and
Magdy, Samar and
Alyafeai, Zaid and
Hamed, Injy and
Zaghouani, Wajdi and
Zitouni, Imed",
booktitle = "Proceedings of The Fourth {A}rabic Natural Language Processing Conference",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.arabicnlp-main.1/",
pages = "1--10",
ISBN = "979-8-89176-473-6",
abstract = "Large language models (LLMs) are increasingly evaluated on legal reasoning tasks, but most benchmarks remain English-centered and test each legal problem in only one linguistic form. We examine whether LLMs preserve legal reasoning performance when the same legal question is expressed across English, formal Arabic, simplified Arabic, and Egyptian Arabic. We construct a controlled benchmark of 200 bar-exam-style multiple-choice questions, preserving the legal facts, answer choices, and correct answer across all language versions. We evaluate GPT-4o, Gemini 2.5 Pro, Grok, and Qwen against a shared human-verified answer key. All models achieve their highest accuracy in English, but Arabic performance varies substantially. Qwen scores 97.0{\%} in English and 79.0{\%} in each Arabic condition, while GPT-4o is the most stable model, with 90.0{\%} in English and 80.0{--}82.0{\%} across Arabic variants. Gemini 2.5 Pro drops from 80.5{\%} in English to 17.5{--}21.0{\%} in Arabic, and Grok drops from 45.0{\%} to 16.5{--}26.5{\%}. These results show that strong English legal accuracy does not guarantee robust Arabic legal reasoning. We argue that multilingual legal evaluation should measure not only accuracy, but also English{--}Arabic gaps and cross-language variation across semantically aligned legal questions."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="alsagheer-etal-2026-legal">
<titleInfo>
<title>Same Legal Facts, Different Verdicts: Evaluating Cross-Lingual and Dialectal Robustness in LLM Legal Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="given">R</namePart>
<namePart type="family">Alsagheer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohammad</namePart>
<namePart type="family">Kamal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abdulrahman</namePart>
<namePart type="family">Kamal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yang</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nour</namePart>
<namePart type="family">Diallo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weidong</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omar</namePart>
<namePart type="family">Kamal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of The Fourth Arabic Natural Language Processing Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="family">Abdul-Mageed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tamer</namePart>
<namePart type="family">Elsayed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Houda</namePart>
<namePart type="family">Bouamor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samhaa</namePart>
<namePart type="family">El-Beltagy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hend</namePart>
<namePart type="family">Al-Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ahmed</namePart>
<namePart type="family">Abdelali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">AbdelRahim</namePart>
<namePart type="family">Elmadany</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abdellah</namePart>
<namePart type="family">El Mekki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kareem</namePart>
<namePart type="family">Darwish</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sakhar</namePart>
<namePart type="family">Alkhereyf</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hamzah</namePart>
<namePart type="family">Luqman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salam</namePart>
<namePart type="family">Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samar</namePart>
<namePart type="family">Magdy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zaid</namePart>
<namePart type="family">Alyafeai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Injy</namePart>
<namePart type="family">Hamed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wajdi</namePart>
<namePart type="family">Zaghouani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Imed</namePart>
<namePart type="family">Zitouni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-473-6</identifier>
</relatedItem>
<abstract>Large language models (LLMs) are increasingly evaluated on legal reasoning tasks, but most benchmarks remain English-centered and test each legal problem in only one linguistic form. We examine whether LLMs preserve legal reasoning performance when the same legal question is expressed across English, formal Arabic, simplified Arabic, and Egyptian Arabic. We construct a controlled benchmark of 200 bar-exam-style multiple-choice questions, preserving the legal facts, answer choices, and correct answer across all language versions. We evaluate GPT-4o, Gemini 2.5 Pro, Grok, and Qwen against a shared human-verified answer key. All models achieve their highest accuracy in English, but Arabic performance varies substantially. Qwen scores 97.0% in English and 79.0% in each Arabic condition, while GPT-4o is the most stable model, with 90.0% in English and 80.0–82.0% across Arabic variants. Gemini 2.5 Pro drops from 80.5% in English to 17.5–21.0% in Arabic, and Grok drops from 45.0% to 16.5–26.5%. These results show that strong English legal accuracy does not guarantee robust Arabic legal reasoning. We argue that multilingual legal evaluation should measure not only accuracy, but also English–Arabic gaps and cross-language variation across semantically aligned legal questions.</abstract>
<identifier type="citekey">alsagheer-etal-2026-legal</identifier>
<location>
<url>https://aclanthology.org/2026.arabicnlp-main.1/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1</start>
<end>10</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Same Legal Facts, Different Verdicts: Evaluating Cross-Lingual and Dialectal Robustness in LLM Legal Reasoning
%A Alsagheer, Dana R.
%A Kamal, Mohammad
%A Kamal, Abdulrahman
%A Lu, Yang
%A Diallo, Nour
%A Shi, Weidong
%A Kamal, Omar
%Y Abdul-Mageed, Muhammad
%Y Elsayed, Tamer
%Y Bouamor, Houda
%Y El-Beltagy, Samhaa
%Y Al-Khalifa, Hend
%Y Abdelali, Ahmed
%Y Elmadany, AbdelRahim
%Y El Mekki, Abdellah
%Y Darwish, Kareem
%Y Alkhereyf, Sakhar
%Y Luqman, Hamzah
%Y Khalifa, Salam
%Y Magdy, Samar
%Y Alyafeai, Zaid
%Y Hamed, Injy
%Y Zaghouani, Wajdi
%Y Zitouni, Imed
%S Proceedings of The Fourth Arabic Natural Language Processing Conference
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-473-6
%F alsagheer-etal-2026-legal
%X Large language models (LLMs) are increasingly evaluated on legal reasoning tasks, but most benchmarks remain English-centered and test each legal problem in only one linguistic form. We examine whether LLMs preserve legal reasoning performance when the same legal question is expressed across English, formal Arabic, simplified Arabic, and Egyptian Arabic. We construct a controlled benchmark of 200 bar-exam-style multiple-choice questions, preserving the legal facts, answer choices, and correct answer across all language versions. We evaluate GPT-4o, Gemini 2.5 Pro, Grok, and Qwen against a shared human-verified answer key. All models achieve their highest accuracy in English, but Arabic performance varies substantially. Qwen scores 97.0% in English and 79.0% in each Arabic condition, while GPT-4o is the most stable model, with 90.0% in English and 80.0–82.0% across Arabic variants. Gemini 2.5 Pro drops from 80.5% in English to 17.5–21.0% in Arabic, and Grok drops from 45.0% to 16.5–26.5%. These results show that strong English legal accuracy does not guarantee robust Arabic legal reasoning. We argue that multilingual legal evaluation should measure not only accuracy, but also English–Arabic gaps and cross-language variation across semantically aligned legal questions.
%U https://aclanthology.org/2026.arabicnlp-main.1/
%P 1-10
Markdown (Informal)
[Same Legal Facts, Different Verdicts: Evaluating Cross-Lingual and Dialectal Robustness in LLM Legal Reasoning](https://aclanthology.org/2026.arabicnlp-main.1/) (Alsagheer et al., ArabicNLP 2026)
ACL
- Dana R Alsagheer, Mohammad Kamal, Abdulrahman Kamal, Yang Lu, Nour Diallo, Weidong Shi, and Omar Kamal. 2026. Same Legal Facts, Different Verdicts: Evaluating Cross-Lingual and Dialectal Robustness in LLM Legal Reasoning. In Proceedings of The Fourth Arabic Natural Language Processing Conference, pages 1–10, Budapest, Hungary. Association for Computational Linguistics.