@inproceedings{rastogi-2026-reliability,
title = "A Reliability Audit of Aggregated {MTEB} Scores: Coverage Asymmetry and Language-Specific Rank Inversions Across 17 Languages",
author = "Rastogi, Archit",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 2: Short Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-short.28/",
pages = "398--429",
ISBN = "979-8-89176-440-8",
abstract = "Practitioners select multilingual embedding models from aggregated MTEB leaderboard scores, but the same global score can correspond to very different per-language behaviours. We audit MTEB(Multilingual) retrieval rankings against language-specific rankings on 17 languages across three nested model rosters (restricted $n{=}7$; deployment-relevant curated $n{=}25$; broad MMTEB-eligible $n{=}28$-57), using Kendall $\tau$ and pairwise inversion rate. At deployment-realistic roster size, median pairwise inversion rate is 33{\%} (per-language Kendall $\tau$ reliable after Benjamini-Hochberg FDR in 11/17 languages); the extended tier shows higher agreement ($\tau{=}0.69$), reflecting its multilingual-first composition. Per-language task coverage is asymmetric, and inversion rate decreases with the number of MTEB retrieval tasks available for a language (Spearman $\rho{=}{-}0.59$ curated). A language-balanced aggregator (per-language mean first, then macro-average) drops the median inversion rate from 33{\%} to 14.5{\%} (leave-one-out, $\tau{=}0.71$), the lowest among the held-out aggregators we test and our most actionable mitigation. A cluster of hidden-failure models, headed by granite-311m (16/17 languages) and harrier-0.6b (10/17), ranks in the global top half but collapses to the local bottom quartile, with a distinctively shallow English-multilingual NDCG gap. We release the three-tier audit data, a multilingual RAG benchmark (IT/JA/HI $\times$ finance/law), and the analysis pipeline."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rastogi-2026-reliability">
<titleInfo>
<title>A Reliability Audit of Aggregated MTEB Scores: Coverage Asymmetry and Language-Specific Rank Inversions Across 17 Languages</title>
</titleInfo>
<name type="personal">
<namePart type="given">Archit</namePart>
<namePart type="family">Rastogi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-440-8</identifier>
</relatedItem>
<abstract>Practitioners select multilingual embedding models from aggregated MTEB leaderboard scores, but the same global score can correspond to very different per-language behaviours. We audit MTEB(Multilingual) retrieval rankings against language-specific rankings on 17 languages across three nested model rosters (restricted n=7; deployment-relevant curated n=25; broad MMTEB-eligible n=28-57), using Kendall τ and pairwise inversion rate. At deployment-realistic roster size, median pairwise inversion rate is 33% (per-language Kendall τ reliable after Benjamini-Hochberg FDR in 11/17 languages); the extended tier shows higher agreement (τ=0.69), reflecting its multilingual-first composition. Per-language task coverage is asymmetric, and inversion rate decreases with the number of MTEB retrieval tasks available for a language (Spearman ρ=-0.59 curated). A language-balanced aggregator (per-language mean first, then macro-average) drops the median inversion rate from 33% to 14.5% (leave-one-out, τ=0.71), the lowest among the held-out aggregators we test and our most actionable mitigation. A cluster of hidden-failure models, headed by granite-311m (16/17 languages) and harrier-0.6b (10/17), ranks in the global top half but collapses to the local bottom quartile, with a distinctively shallow English-multilingual NDCG gap. We release the three-tier audit data, a multilingual RAG benchmark (IT/JA/HI \times finance/law), and the analysis pipeline.</abstract>
<identifier type="citekey">rastogi-2026-reliability</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-short.28/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>398</start>
<end>429</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Reliability Audit of Aggregated MTEB Scores: Coverage Asymmetry and Language-Specific Rank Inversions Across 17 Languages
%A Rastogi, Archit
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-440-8
%F rastogi-2026-reliability
%X Practitioners select multilingual embedding models from aggregated MTEB leaderboard scores, but the same global score can correspond to very different per-language behaviours. We audit MTEB(Multilingual) retrieval rankings against language-specific rankings on 17 languages across three nested model rosters (restricted n=7; deployment-relevant curated n=25; broad MMTEB-eligible n=28-57), using Kendall τ and pairwise inversion rate. At deployment-realistic roster size, median pairwise inversion rate is 33% (per-language Kendall τ reliable after Benjamini-Hochberg FDR in 11/17 languages); the extended tier shows higher agreement (τ=0.69), reflecting its multilingual-first composition. Per-language task coverage is asymmetric, and inversion rate decreases with the number of MTEB retrieval tasks available for a language (Spearman ρ=-0.59 curated). A language-balanced aggregator (per-language mean first, then macro-average) drops the median inversion rate from 33% to 14.5% (leave-one-out, τ=0.71), the lowest among the held-out aggregators we test and our most actionable mitigation. A cluster of hidden-failure models, headed by granite-311m (16/17 languages) and harrier-0.6b (10/17), ranks in the global top half but collapses to the local bottom quartile, with a distinctively shallow English-multilingual NDCG gap. We release the three-tier audit data, a multilingual RAG benchmark (IT/JA/HI \times finance/law), and the analysis pipeline.
%U https://aclanthology.org/2026.aacl-short.28/
%P 398-429
Markdown (Informal)
[A Reliability Audit of Aggregated MTEB Scores: Coverage Asymmetry and Language-Specific Rank Inversions Across 17 Languages](https://aclanthology.org/2026.aacl-short.28/) (Rastogi, AACL-IJCNLP 2026)
ACL