@inproceedings{pang-etal-2026-fair,
title = "{FAIR}-Eval: Reliable and Efficient Adaptive Benchmarking for Large Language Models",
author = "Pang, Ziliang and
Wang, Tianhe and
Yu, Tianshu and
Zhang, Ziyi and
Ren, Shuo and
Zhang, Jiajun",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1980/",
pages = "42196--42220",
ISBN = "979-8-89176-462-0",
abstract = "Evaluating large language models typically requires thousands of benchmark items, making the process expensive and increasingly impractical. Existing protocols rely on average accuracy over fixed item sets, treating all items as equally informative{---}yet many items are too easy or too hard to distinguish between models, making evaluation both wasteful and imprecise. Item Response Theory (IRT) with Computerized Adaptive Testing (CAT) offers a principled alternative{---}separately estimating model ability and item difficulty, then adaptively selecting the most informative items. However, practical deployment is bottlenecked by costly item calibration, and standard greedy selection causes severe over-exposure of frequently picked items while leaving the rest of the bank under-utilized, which risks data contamination over time. We propose \textbf{FAIR-Eval} with two components: \textbf{Feature-Driven Calibration (FDC)}, which jointly learns item parameters from text features and sparse responses, enabling low-cost bank expansion requiring only minimal model responses; and \textbf{Maximum Fisher Information Plus (MFI+)}, which counteracts greedy selection by incorporating exposure control, while simultaneously enforcing difficulty coverage and domain diversity to systematically activate under-utilized items. On a 6,796-item bank (193 calibration models, 50 held-out), FAIR-Eval achieves Pearson $r = 0.9882$ and Spearman $\rho = 0.9828$ against full evaluation using around 3{\%} of items. Crucially, it broadens difficulty coverage from 60.0{\%} to 68.7{\%}, achieving a more comprehensive and robust assessment of model capabilities with minimal test length."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="pang-etal-2026-fair">
<titleInfo>
<title>FAIR-Eval: Reliable and Efficient Adaptive Benchmarking for Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ziliang</namePart>
<namePart type="family">Pang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianhe</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianshu</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziyi</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuo</namePart>
<namePart type="family">Ren</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiajun</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Evaluating large language models typically requires thousands of benchmark items, making the process expensive and increasingly impractical. Existing protocols rely on average accuracy over fixed item sets, treating all items as equally informative—yet many items are too easy or too hard to distinguish between models, making evaluation both wasteful and imprecise. Item Response Theory (IRT) with Computerized Adaptive Testing (CAT) offers a principled alternative—separately estimating model ability and item difficulty, then adaptively selecting the most informative items. However, practical deployment is bottlenecked by costly item calibration, and standard greedy selection causes severe over-exposure of frequently picked items while leaving the rest of the bank under-utilized, which risks data contamination over time. We propose FAIR-Eval with two components: Feature-Driven Calibration (FDC), which jointly learns item parameters from text features and sparse responses, enabling low-cost bank expansion requiring only minimal model responses; and Maximum Fisher Information Plus (MFI+), which counteracts greedy selection by incorporating exposure control, while simultaneously enforcing difficulty coverage and domain diversity to systematically activate under-utilized items. On a 6,796-item bank (193 calibration models, 50 held-out), FAIR-Eval achieves Pearson r = 0.9882 and Spearman ρ = 0.9828 against full evaluation using around 3% of items. Crucially, it broadens difficulty coverage from 60.0% to 68.7%, achieving a more comprehensive and robust assessment of model capabilities with minimal test length.</abstract>
<identifier type="citekey">pang-etal-2026-fair</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1980/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>42196</start>
<end>42220</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T FAIR-Eval: Reliable and Efficient Adaptive Benchmarking for Large Language Models
%A Pang, Ziliang
%A Wang, Tianhe
%A Yu, Tianshu
%A Zhang, Ziyi
%A Ren, Shuo
%A Zhang, Jiajun
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F pang-etal-2026-fair
%X Evaluating large language models typically requires thousands of benchmark items, making the process expensive and increasingly impractical. Existing protocols rely on average accuracy over fixed item sets, treating all items as equally informative—yet many items are too easy or too hard to distinguish between models, making evaluation both wasteful and imprecise. Item Response Theory (IRT) with Computerized Adaptive Testing (CAT) offers a principled alternative—separately estimating model ability and item difficulty, then adaptively selecting the most informative items. However, practical deployment is bottlenecked by costly item calibration, and standard greedy selection causes severe over-exposure of frequently picked items while leaving the rest of the bank under-utilized, which risks data contamination over time. We propose FAIR-Eval with two components: Feature-Driven Calibration (FDC), which jointly learns item parameters from text features and sparse responses, enabling low-cost bank expansion requiring only minimal model responses; and Maximum Fisher Information Plus (MFI+), which counteracts greedy selection by incorporating exposure control, while simultaneously enforcing difficulty coverage and domain diversity to systematically activate under-utilized items. On a 6,796-item bank (193 calibration models, 50 held-out), FAIR-Eval achieves Pearson r = 0.9882 and Spearman ρ = 0.9828 against full evaluation using around 3% of items. Crucially, it broadens difficulty coverage from 60.0% to 68.7%, achieving a more comprehensive and robust assessment of model capabilities with minimal test length.
%U https://aclanthology.org/2026.findings-emnlp.1980/
%P 42196-42220
Markdown (Informal)
[FAIR-Eval: Reliable and Efficient Adaptive Benchmarking for Large Language Models](https://aclanthology.org/2026.findings-emnlp.1980/) (Pang et al., Findings 2026)
ACL