@inproceedings{firke-kannan-2026-correlation,
title = "The Correlation Mirage: Benchmark Dependence Collapses for Top-Performing {LLM}s",
author = "Firke, Raj and
Kannan, Rajeswari",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2046/",
pages = "46629--46644",
ISBN = "979-8-89176-459-0",
abstract = "Standard approaches to benchmark redundancy rely on Spearman correlation, which implicitly treats dependence as uniform across the performance spectrum. Using copula-based dependence modeling on 4,497 models (779 deduplicated models; 714 unique base-model families) from the Open LLM Leaderboard v2, we show this assumption fails dramatically: benchmark pairs correlating at $\rho_S > 0.73$ show near-zero or reversed ranking agreement among top-performing models, with conditional Spearman reductions of up to 97{\%}{---}findings that hold after cluster-robust inference at the base-model-family level ($\geq$96{\%} bootstrap stability for two of three headline pairs, $\Delta$BIC CIs entirely above zero). The mechanism is non-Gaussian dependence structure: 47{\%} of benchmark pairs follow copula families with zero asymptotic tail dependence despite strong bulk association ($\Delta$BIC up to 585), confirmed by nonparametric validation requiring no parametric assumptions. We validate across BenchPress (316 benchmarks; signal strengthens with sample size) and kensho/WILD (27 benchmarks, 65 independent-organization models; 38{\%} Frank prevalence), and demonstrate that no scalar dependence measure{---}Spearman, distance correlation, or mutual information{---}can detect the tail decomposition. In plain terms: a model ranked first on one benchmark but last on another is not unusual among frontier models{---}precisely the ones practitioners are deploying."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="firke-kannan-2026-correlation">
<titleInfo>
<title>The Correlation Mirage: Benchmark Dependence Collapses for Top-Performing LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="family">Firke</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rajeswari</namePart>
<namePart type="family">Kannan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Standard approaches to benchmark redundancy rely on Spearman correlation, which implicitly treats dependence as uniform across the performance spectrum. Using copula-based dependence modeling on 4,497 models (779 deduplicated models; 714 unique base-model families) from the Open LLM Leaderboard v2, we show this assumption fails dramatically: benchmark pairs correlating at ρ_S > 0.73 show near-zero or reversed ranking agreement among top-performing models, with conditional Spearman reductions of up to 97%—findings that hold after cluster-robust inference at the base-model-family level (\geq96% bootstrap stability for two of three headline pairs, ΔBIC CIs entirely above zero). The mechanism is non-Gaussian dependence structure: 47% of benchmark pairs follow copula families with zero asymptotic tail dependence despite strong bulk association (ΔBIC up to 585), confirmed by nonparametric validation requiring no parametric assumptions. We validate across BenchPress (316 benchmarks; signal strengthens with sample size) and kensho/WILD (27 benchmarks, 65 independent-organization models; 38% Frank prevalence), and demonstrate that no scalar dependence measure—Spearman, distance correlation, or mutual information—can detect the tail decomposition. In plain terms: a model ranked first on one benchmark but last on another is not unusual among frontier models—precisely the ones practitioners are deploying.</abstract>
<identifier type="citekey">firke-kannan-2026-correlation</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2046/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>46629</start>
<end>46644</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Correlation Mirage: Benchmark Dependence Collapses for Top-Performing LLMs
%A Firke, Raj
%A Kannan, Rajeswari
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F firke-kannan-2026-correlation
%X Standard approaches to benchmark redundancy rely on Spearman correlation, which implicitly treats dependence as uniform across the performance spectrum. Using copula-based dependence modeling on 4,497 models (779 deduplicated models; 714 unique base-model families) from the Open LLM Leaderboard v2, we show this assumption fails dramatically: benchmark pairs correlating at ρ_S > 0.73 show near-zero or reversed ranking agreement among top-performing models, with conditional Spearman reductions of up to 97%—findings that hold after cluster-robust inference at the base-model-family level (\geq96% bootstrap stability for two of three headline pairs, ΔBIC CIs entirely above zero). The mechanism is non-Gaussian dependence structure: 47% of benchmark pairs follow copula families with zero asymptotic tail dependence despite strong bulk association (ΔBIC up to 585), confirmed by nonparametric validation requiring no parametric assumptions. We validate across BenchPress (316 benchmarks; signal strengthens with sample size) and kensho/WILD (27 benchmarks, 65 independent-organization models; 38% Frank prevalence), and demonstrate that no scalar dependence measure—Spearman, distance correlation, or mutual information—can detect the tail decomposition. In plain terms: a model ranked first on one benchmark but last on another is not unusual among frontier models—precisely the ones practitioners are deploying.
%U https://aclanthology.org/2026.emnlp-main.2046/
%P 46629-46644
Markdown (Informal)
[The Correlation Mirage: Benchmark Dependence Collapses for Top-Performing LLMs](https://aclanthology.org/2026.emnlp-main.2046/) (Firke & Kannan, EMNLP 2026)
ACL