@inproceedings{rammouz-etal-2026-absolute,
title = "Absolute Error Is Not Enough: Ranking Model Performance Without Labels to Estimate Disparities at Scale",
author = "Rammouz, Veronica and
Gonzalez, Aaron O. and
Cruzportillo, Carlos and
Tan, Adrian and
Beebe, Nicole Lang and
Yang, Ke and
Rios, Anthony",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.104/",
pages = "2290--2307",
ISBN = "979-8-89176-439-2",
abstract = "Estimating model performance without labeled data is important for deploying NLP systems across diverse domains. Prior work has proposed proxy measures based on distribution shift or predicted correctness, but it remains unclear whether these methods reliably preserve the \textit{ranking} of domains, which is critical for model selection and risk triage. We study this problem from an ordinal perspective using a two-step framework that pairs a base classifier with an error predictor to estimate instance-level failures and disparities and aggregate them into label-free performance estimates. Across multiple sample sizes from the \textit{GeoOLID} and \textit{Amazon Reviews} datasets, large language model{--}based error predictors substantially outperform drift-based baselines in $\rho$ and $\tau$ rank correlations. We further show that lower absolute estimation error ($RMSE$) does not necessarily imply better ranking, highlighting the importance of evaluating ordinal consistency directly.\par \textbf{WARNING:} This paper contains offensive and hateful speech in the appendix."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rammouz-etal-2026-absolute">
<titleInfo>
<title>Absolute Error Is Not Enough: Ranking Model Performance Without Labels to Estimate Disparities at Scale</title>
</titleInfo>
<name type="personal">
<namePart type="given">Veronica</namePart>
<namePart type="family">Rammouz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="given">O</namePart>
<namePart type="family">Gonzalez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Carlos</namePart>
<namePart type="family">Cruzportillo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Adrian</namePart>
<namePart type="family">Tan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicole</namePart>
<namePart type="given">Lang</namePart>
<namePart type="family">Beebe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ke</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anthony</namePart>
<namePart type="family">Rios</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Estimating model performance without labeled data is important for deploying NLP systems across diverse domains. Prior work has proposed proxy measures based on distribution shift or predicted correctness, but it remains unclear whether these methods reliably preserve the ranking of domains, which is critical for model selection and risk triage. We study this problem from an ordinal perspective using a two-step framework that pairs a base classifier with an error predictor to estimate instance-level failures and disparities and aggregate them into label-free performance estimates. Across multiple sample sizes from the GeoOLID and Amazon Reviews datasets, large language model–based error predictors substantially outperform drift-based baselines in ρ and τ rank correlations. We further show that lower absolute estimation error (RMSE) does not necessarily imply better ranking, highlighting the importance of evaluating ordinal consistency directly.\par WARNING: This paper contains offensive and hateful speech in the appendix.</abstract>
<identifier type="citekey">rammouz-etal-2026-absolute</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.104/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>2290</start>
<end>2307</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Absolute Error Is Not Enough: Ranking Model Performance Without Labels to Estimate Disparities at Scale
%A Rammouz, Veronica
%A Gonzalez, Aaron O.
%A Cruzportillo, Carlos
%A Tan, Adrian
%A Beebe, Nicole Lang
%A Yang, Ke
%A Rios, Anthony
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F rammouz-etal-2026-absolute
%X Estimating model performance without labeled data is important for deploying NLP systems across diverse domains. Prior work has proposed proxy measures based on distribution shift or predicted correctness, but it remains unclear whether these methods reliably preserve the ranking of domains, which is critical for model selection and risk triage. We study this problem from an ordinal perspective using a two-step framework that pairs a base classifier with an error predictor to estimate instance-level failures and disparities and aggregate them into label-free performance estimates. Across multiple sample sizes from the GeoOLID and Amazon Reviews datasets, large language model–based error predictors substantially outperform drift-based baselines in ρ and τ rank correlations. We further show that lower absolute estimation error (RMSE) does not necessarily imply better ranking, highlighting the importance of evaluating ordinal consistency directly.\par WARNING: This paper contains offensive and hateful speech in the appendix.
%U https://aclanthology.org/2026.aacl-long.104/
%P 2290-2307
Markdown (Informal)
[Absolute Error Is Not Enough: Ranking Model Performance Without Labels to Estimate Disparities at Scale](https://aclanthology.org/2026.aacl-long.104/) (Rammouz et al., AACL-IJCNLP 2026)
ACL
- Veronica Rammouz, Aaron O. Gonzalez, Carlos Cruzportillo, Adrian Tan, Nicole Lang Beebe, Ke Yang, and Anthony Rios. 2026. Absolute Error Is Not Enough: Ranking Model Performance Without Labels to Estimate Disparities at Scale. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 2290–2307, Hengqin, China. Association for Computational Linguistics.