@inproceedings{deng-etal-2026-one,
title = "One Example Is Enough to Pass Fairness Benchmarks: Rethinking Fairness Evaluation for Aligned {LLM}s",
author = "Deng, Naihao and
Arif, Samee and
Chang, Shuaichen and
Chen, Yulong and
Mihalcea, Rada",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2125/",
pages = "48329--48351",
ISBN = "979-8-89176-459-0",
abstract = "Fairness benchmarks such as BBQ have become the de facto standard for fairness evaluation across major model families. We argue that these benchmarks are too easy to saturate to support that role: training Qwen 2.5 7B Base with Group Relative Policy Optimization (GRPO) on a \textit{single} BBQ example lifts mean BBQ accuracy from 79.9 to 92.0 across all 11 social-bias categories, closing of the gap to the large-scale RLHF counterpart, and the effect generalizes across model families. A cross-conditioning analysis shows the improvement is carried by the reasoning traces generated by the model, not by the output head; one example suffices to elicit a category-agnostic ``missing evidence'' reasoning pattern. Strikingly, placing that single example in context as a one-shot demonstration{---}with no weight update{---}likewise reaches comparable BBQ accuracy and produces reasoning a fairness reward model rates as equally fair, confirming that the benchmark is saturable without gradient-based alignment. We argue that BBQ-style multiple-choice abstention benchmarks measure a single structural cue, and a model that solves them does not thereby become fair. We call for evaluation suites that cover a broader spectrum of fairness alignment."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="deng-etal-2026-one">
<titleInfo>
<title>One Example Is Enough to Pass Fairness Benchmarks: Rethinking Fairness Evaluation for Aligned LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Naihao</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samee</namePart>
<namePart type="family">Arif</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuaichen</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yulong</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rada</namePart>
<namePart type="family">Mihalcea</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Fairness benchmarks such as BBQ have become the de facto standard for fairness evaluation across major model families. We argue that these benchmarks are too easy to saturate to support that role: training Qwen 2.5 7B Base with Group Relative Policy Optimization (GRPO) on a single BBQ example lifts mean BBQ accuracy from 79.9 to 92.0 across all 11 social-bias categories, closing of the gap to the large-scale RLHF counterpart, and the effect generalizes across model families. A cross-conditioning analysis shows the improvement is carried by the reasoning traces generated by the model, not by the output head; one example suffices to elicit a category-agnostic “missing evidence” reasoning pattern. Strikingly, placing that single example in context as a one-shot demonstration—with no weight update—likewise reaches comparable BBQ accuracy and produces reasoning a fairness reward model rates as equally fair, confirming that the benchmark is saturable without gradient-based alignment. We argue that BBQ-style multiple-choice abstention benchmarks measure a single structural cue, and a model that solves them does not thereby become fair. We call for evaluation suites that cover a broader spectrum of fairness alignment.</abstract>
<identifier type="citekey">deng-etal-2026-one</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2125/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>48329</start>
<end>48351</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T One Example Is Enough to Pass Fairness Benchmarks: Rethinking Fairness Evaluation for Aligned LLMs
%A Deng, Naihao
%A Arif, Samee
%A Chang, Shuaichen
%A Chen, Yulong
%A Mihalcea, Rada
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F deng-etal-2026-one
%X Fairness benchmarks such as BBQ have become the de facto standard for fairness evaluation across major model families. We argue that these benchmarks are too easy to saturate to support that role: training Qwen 2.5 7B Base with Group Relative Policy Optimization (GRPO) on a single BBQ example lifts mean BBQ accuracy from 79.9 to 92.0 across all 11 social-bias categories, closing of the gap to the large-scale RLHF counterpart, and the effect generalizes across model families. A cross-conditioning analysis shows the improvement is carried by the reasoning traces generated by the model, not by the output head; one example suffices to elicit a category-agnostic “missing evidence” reasoning pattern. Strikingly, placing that single example in context as a one-shot demonstration—with no weight update—likewise reaches comparable BBQ accuracy and produces reasoning a fairness reward model rates as equally fair, confirming that the benchmark is saturable without gradient-based alignment. We argue that BBQ-style multiple-choice abstention benchmarks measure a single structural cue, and a model that solves them does not thereby become fair. We call for evaluation suites that cover a broader spectrum of fairness alignment.
%U https://aclanthology.org/2026.emnlp-main.2125/
%P 48329-48351
Markdown (Informal)
[One Example Is Enough to Pass Fairness Benchmarks: Rethinking Fairness Evaluation for Aligned LLMs](https://aclanthology.org/2026.emnlp-main.2125/) (Deng et al., EMNLP 2026)
ACL