@inproceedings{jlali-suppa-2026-tunisianmmlu,
title = "Is {T}unisian{MMLU} Actually {T}unisian? A Native-Speaker Audit of a Machine-Translated Dialect Benchmark",
author = "Jlali, Fatma Ezzahra and
{\v{S}}uppa, Marek",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.20/",
pages = "257--269",
ISBN = "979-8-89176-469-9",
abstract = "TunisianMMLU, the only MMLU-style benchmark for Tunisian Arabic (Derja), was machine-translated with no native-speaker validation. Scanning all 21{,}500 items with native-validated patterns, we flag 63.4{\%} of the corpus; the largest signal, in 55.1{\%} of items, is the Moroccan present-tense prefix, a form Tunisian does not have. The dominant defect is not translation noise but a different dialect. We then present the first native-speaker audit: 400 items stratified over all 44 subjects, labelled independently by two native speakers on six criteria. The non-author annotator marks 64.5{\%} defective, both agree on 62.0{\%}, and author adjudication yields 86.0{\%}. Nearly all defects originate in translation, not in the source benchmarks. Six models corroborate the labels, scoring highest on items marked clean and lowest on unrepairable ones. Corrected text raises every above-chance model by 2.3 to 4.9 points (one gain survives multiple-test correction), leaves a chance-level Derja model unchanged, and narrows the gap between a dialect-adapted model and its base without changing rank order. We release TunisianMMLU-Verified: a corrected subset, corpus-wide flags, per-item labels, the annotation guideline, and all evaluation code."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jlali-suppa-2026-tunisianmmlu">
<titleInfo>
<title>Is TunisianMMLU Actually Tunisian? A Native-Speaker Audit of a Machine-Translated Dialect Benchmark</title>
</titleInfo>
<name type="personal">
<namePart type="given">Fatma</namePart>
<namePart type="given">Ezzahra</namePart>
<namePart type="family">Jlali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marek</namePart>
<namePart type="family">Šuppa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>TunisianMMLU, the only MMLU-style benchmark for Tunisian Arabic (Derja), was machine-translated with no native-speaker validation. Scanning all 21,500 items with native-validated patterns, we flag 63.4% of the corpus; the largest signal, in 55.1% of items, is the Moroccan present-tense prefix, a form Tunisian does not have. The dominant defect is not translation noise but a different dialect. We then present the first native-speaker audit: 400 items stratified over all 44 subjects, labelled independently by two native speakers on six criteria. The non-author annotator marks 64.5% defective, both agree on 62.0%, and author adjudication yields 86.0%. Nearly all defects originate in translation, not in the source benchmarks. Six models corroborate the labels, scoring highest on items marked clean and lowest on unrepairable ones. Corrected text raises every above-chance model by 2.3 to 4.9 points (one gain survives multiple-test correction), leaves a chance-level Derja model unchanged, and narrows the gap between a dialect-adapted model and its base without changing rank order. We release TunisianMMLU-Verified: a corrected subset, corpus-wide flags, per-item labels, the annotation guideline, and all evaluation code.</abstract>
<identifier type="citekey">jlali-suppa-2026-tunisianmmlu</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.20/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>257</start>
<end>269</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Is TunisianMMLU Actually Tunisian? A Native-Speaker Audit of a Machine-Translated Dialect Benchmark
%A Jlali, Fatma Ezzahra
%A Šuppa, Marek
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F jlali-suppa-2026-tunisianmmlu
%X TunisianMMLU, the only MMLU-style benchmark for Tunisian Arabic (Derja), was machine-translated with no native-speaker validation. Scanning all 21,500 items with native-validated patterns, we flag 63.4% of the corpus; the largest signal, in 55.1% of items, is the Moroccan present-tense prefix, a form Tunisian does not have. The dominant defect is not translation noise but a different dialect. We then present the first native-speaker audit: 400 items stratified over all 44 subjects, labelled independently by two native speakers on six criteria. The non-author annotator marks 64.5% defective, both agree on 62.0%, and author adjudication yields 86.0%. Nearly all defects originate in translation, not in the source benchmarks. Six models corroborate the labels, scoring highest on items marked clean and lowest on unrepairable ones. Corrected text raises every above-chance model by 2.3 to 4.9 points (one gain survives multiple-test correction), leaves a chance-level Derja model unchanged, and narrows the gap between a dialect-adapted model and its base without changing rank order. We release TunisianMMLU-Verified: a corrected subset, corpus-wide flags, per-item labels, the annotation guideline, and all evaluation code.
%U https://aclanthology.org/2026.mrl-1.20/
%P 257-269
Markdown (Informal)
[Is TunisianMMLU Actually Tunisian? A Native-Speaker Audit of a Machine-Translated Dialect Benchmark](https://aclanthology.org/2026.mrl-1.20/) (Jlali & Šuppa, MRL 2026)
ACL