@inproceedings{uppadhyay-etal-2026-depart,
title = "{DEPART}: {DE}composing {PAR}i{T}y across Multilingual {LLM}s",
author = "Uppadhyay, Manan and
Kodali, Prashant and
Chitale, Pranjal A and
Ramaprasad, Reshma and
Beniwal, Himanshu and
Sitaram, Sunayana",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2328/",
pages = "50003--50026",
ISBN = "979-8-89176-462-0",
abstract = "The current evaluation landscape for multilingual Large Language Models (LLMs) broadly assesses raw performance through static leaderboards, without offering a principled account of why performance gaps emerge. Gauging raw accuracy alone provides no explainable or actionable outcomes, masking systemic biases while failing to capture linguistic parity, a critical prerequisite for equitable deployment. To address this limitation, we first statistically demonstrate the widespread existence of severe performance disparities across multilingual benchmarks and then introduce a two-step hierarchical evaluation approach to break down their root causes. In the first step, we fit a Bayesian hierarchical mixed-effects model to isolate and analyze the performance variance attributable strictly to language identity, independent of task type. We demonstrate that observable language features (e.g., script, family, and typological distance) explain 79{\%} of this language-based variance on understanding tasks and 92{\%} on reasoning tasks - a quantity we formalize as $R^2_{ling}$, the proportional reduction in the language random-intercept variance attributable to observable features - with a model{'}s internal representational similarity to English emerging as the dominant predictor across both task buckets. In the second step, we broaden our analysis across the wider variance axes of the experimental design to unpack the systemic drivers of benchmark score variation. We find that natural language understanding (NLU) and reasoning tasks possess fundamentally divergent variance profiles: model identity heavily dominates understanding tasks (66.7{\%} of total variance), whereas the benchmark-by-model interaction accounts for the largest share of variance in reasoning (46.3{\%}). Together, these results transition multilingual evaluation from passive performance mapping into an explainable, diagnostic framework that yields actionable levers to intentionally target the root drivers of language disparity."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="uppadhyay-etal-2026-depart">
<titleInfo>
<title>DEPART: DEcomposing PARiTy across Multilingual LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Manan</namePart>
<namePart type="family">Uppadhyay</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Prashant</namePart>
<namePart type="family">Kodali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pranjal</namePart>
<namePart type="given">A</namePart>
<namePart type="family">Chitale</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Reshma</namePart>
<namePart type="family">Ramaprasad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Himanshu</namePart>
<namePart type="family">Beniwal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunayana</namePart>
<namePart type="family">Sitaram</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>The current evaluation landscape for multilingual Large Language Models (LLMs) broadly assesses raw performance through static leaderboards, without offering a principled account of why performance gaps emerge. Gauging raw accuracy alone provides no explainable or actionable outcomes, masking systemic biases while failing to capture linguistic parity, a critical prerequisite for equitable deployment. To address this limitation, we first statistically demonstrate the widespread existence of severe performance disparities across multilingual benchmarks and then introduce a two-step hierarchical evaluation approach to break down their root causes. In the first step, we fit a Bayesian hierarchical mixed-effects model to isolate and analyze the performance variance attributable strictly to language identity, independent of task type. We demonstrate that observable language features (e.g., script, family, and typological distance) explain 79% of this language-based variance on understanding tasks and 92% on reasoning tasks - a quantity we formalize as R²_ling, the proportional reduction in the language random-intercept variance attributable to observable features - with a model’s internal representational similarity to English emerging as the dominant predictor across both task buckets. In the second step, we broaden our analysis across the wider variance axes of the experimental design to unpack the systemic drivers of benchmark score variation. We find that natural language understanding (NLU) and reasoning tasks possess fundamentally divergent variance profiles: model identity heavily dominates understanding tasks (66.7% of total variance), whereas the benchmark-by-model interaction accounts for the largest share of variance in reasoning (46.3%). Together, these results transition multilingual evaluation from passive performance mapping into an explainable, diagnostic framework that yields actionable levers to intentionally target the root drivers of language disparity.</abstract>
<identifier type="citekey">uppadhyay-etal-2026-depart</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2328/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>50003</start>
<end>50026</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DEPART: DEcomposing PARiTy across Multilingual LLMs
%A Uppadhyay, Manan
%A Kodali, Prashant
%A Chitale, Pranjal A.
%A Ramaprasad, Reshma
%A Beniwal, Himanshu
%A Sitaram, Sunayana
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F uppadhyay-etal-2026-depart
%X The current evaluation landscape for multilingual Large Language Models (LLMs) broadly assesses raw performance through static leaderboards, without offering a principled account of why performance gaps emerge. Gauging raw accuracy alone provides no explainable or actionable outcomes, masking systemic biases while failing to capture linguistic parity, a critical prerequisite for equitable deployment. To address this limitation, we first statistically demonstrate the widespread existence of severe performance disparities across multilingual benchmarks and then introduce a two-step hierarchical evaluation approach to break down their root causes. In the first step, we fit a Bayesian hierarchical mixed-effects model to isolate and analyze the performance variance attributable strictly to language identity, independent of task type. We demonstrate that observable language features (e.g., script, family, and typological distance) explain 79% of this language-based variance on understanding tasks and 92% on reasoning tasks - a quantity we formalize as R²_ling, the proportional reduction in the language random-intercept variance attributable to observable features - with a model’s internal representational similarity to English emerging as the dominant predictor across both task buckets. In the second step, we broaden our analysis across the wider variance axes of the experimental design to unpack the systemic drivers of benchmark score variation. We find that natural language understanding (NLU) and reasoning tasks possess fundamentally divergent variance profiles: model identity heavily dominates understanding tasks (66.7% of total variance), whereas the benchmark-by-model interaction accounts for the largest share of variance in reasoning (46.3%). Together, these results transition multilingual evaluation from passive performance mapping into an explainable, diagnostic framework that yields actionable levers to intentionally target the root drivers of language disparity.
%U https://aclanthology.org/2026.findings-emnlp.2328/
%P 50003-50026
Markdown (Informal)
[DEPART: DEcomposing PARiTy across Multilingual LLMs](https://aclanthology.org/2026.findings-emnlp.2328/) (Uppadhyay et al., Findings 2026)
ACL
- Manan Uppadhyay, Prashant Kodali, Pranjal A Chitale, Reshma Ramaprasad, Himanshu Beniwal, and Sunayana Sitaram. 2026. DEPART: DEcomposing PARiTy across Multilingual LLMs. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 50003–50026, Budapest, Hungary. Association for Computational Linguistics.