@inproceedings{uzunoglu-etal-2026-flaw,
title = "The Flaw of Averages: Measuring Benchmark-Level Distributional Robustness",
author = "Uzunoglu, Arda and
Li, Tianjian and
Khashabi, Daniel",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1895/",
pages = "43186--43226",
ISBN = "979-8-89176-459-0",
abstract = "Benchmarks are central to measuring progress in language models, but aggregate scores can obscure substantial variation across subdomains, making models appear broadly competent despite concentrated strengths and weaknesses. We study this issue as \textit{benchmark-level distributional robustness}: whether aggregate scores faithfully reflect performance across benchmark subdomains. We operationalize this notion with benchmark harmony, an entropy-based measure of how uniformly model performance is distributed across subdomains. Measuring harmony on 19 language model benchmarks across five model families, we find substantial variation in benchmark-level distributional robustness. Low-harmony benchmarks are more likely to yield aggregate scores that overstate broad competence, whereas high-harmony benchmarks provide more representative summaries of model capability. Rebalancing benchmarks by pruning overrepresented subdomains to increase harmony substantially shifts aggregate scores for low-harmony benchmarks, but leaves high-harmony benchmarks comparatively stable. For example, while BoolQ remains comparatively stable as harmony increases, PubMedQA, which evaluates performance in a medically consequential domain, exhibits substantial, often statistically significant, shifts in aggregate accuracy. Together, these findings show that aggregate scores can misrepresent broad competence when performance is unevenly distributed. We therefore recommend reporting benchmark harmony alongside aggregate accuracy as a diagnostic of benchmark representativeness when interpreting claims about broad model competence."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="uzunoglu-etal-2026-flaw">
<titleInfo>
<title>The Flaw of Averages: Measuring Benchmark-Level Distributional Robustness</title>
</titleInfo>
<name type="personal">
<namePart type="given">Arda</namePart>
<namePart type="family">Uzunoglu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianjian</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Khashabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Benchmarks are central to measuring progress in language models, but aggregate scores can obscure substantial variation across subdomains, making models appear broadly competent despite concentrated strengths and weaknesses. We study this issue as benchmark-level distributional robustness: whether aggregate scores faithfully reflect performance across benchmark subdomains. We operationalize this notion with benchmark harmony, an entropy-based measure of how uniformly model performance is distributed across subdomains. Measuring harmony on 19 language model benchmarks across five model families, we find substantial variation in benchmark-level distributional robustness. Low-harmony benchmarks are more likely to yield aggregate scores that overstate broad competence, whereas high-harmony benchmarks provide more representative summaries of model capability. Rebalancing benchmarks by pruning overrepresented subdomains to increase harmony substantially shifts aggregate scores for low-harmony benchmarks, but leaves high-harmony benchmarks comparatively stable. For example, while BoolQ remains comparatively stable as harmony increases, PubMedQA, which evaluates performance in a medically consequential domain, exhibits substantial, often statistically significant, shifts in aggregate accuracy. Together, these findings show that aggregate scores can misrepresent broad competence when performance is unevenly distributed. We therefore recommend reporting benchmark harmony alongside aggregate accuracy as a diagnostic of benchmark representativeness when interpreting claims about broad model competence.</abstract>
<identifier type="citekey">uzunoglu-etal-2026-flaw</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1895/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>43186</start>
<end>43226</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Flaw of Averages: Measuring Benchmark-Level Distributional Robustness
%A Uzunoglu, Arda
%A Li, Tianjian
%A Khashabi, Daniel
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F uzunoglu-etal-2026-flaw
%X Benchmarks are central to measuring progress in language models, but aggregate scores can obscure substantial variation across subdomains, making models appear broadly competent despite concentrated strengths and weaknesses. We study this issue as benchmark-level distributional robustness: whether aggregate scores faithfully reflect performance across benchmark subdomains. We operationalize this notion with benchmark harmony, an entropy-based measure of how uniformly model performance is distributed across subdomains. Measuring harmony on 19 language model benchmarks across five model families, we find substantial variation in benchmark-level distributional robustness. Low-harmony benchmarks are more likely to yield aggregate scores that overstate broad competence, whereas high-harmony benchmarks provide more representative summaries of model capability. Rebalancing benchmarks by pruning overrepresented subdomains to increase harmony substantially shifts aggregate scores for low-harmony benchmarks, but leaves high-harmony benchmarks comparatively stable. For example, while BoolQ remains comparatively stable as harmony increases, PubMedQA, which evaluates performance in a medically consequential domain, exhibits substantial, often statistically significant, shifts in aggregate accuracy. Together, these findings show that aggregate scores can misrepresent broad competence when performance is unevenly distributed. We therefore recommend reporting benchmark harmony alongside aggregate accuracy as a diagnostic of benchmark representativeness when interpreting claims about broad model competence.
%U https://aclanthology.org/2026.emnlp-main.1895/
%P 43186-43226
Markdown (Informal)
[The Flaw of Averages: Measuring Benchmark-Level Distributional Robustness](https://aclanthology.org/2026.emnlp-main.1895/) (Uzunoglu et al., EMNLP 2026)
ACL