@inproceedings{kim-kim-2026-bias,
title = "When Bias Scores Reflect Abstention Failure: Disentangling Stereotype Direction from Refusal to Abstain in {BBQ}",
author = "Kim, Jeemin and
Kim, Jooyoung",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2274/",
doi = "10.18653/2026.findings-emnlp.2274",
pages = "48681--48697",
ISBN = "979-8-89176-462-0",
abstract = "Social-bias benchmarks such as BBQ (Parrish et al., 2022) evaluate whether language models rely on stereotypes in ambiguous contexts where the correct answer indicates insufficient information. We observe that BBQ{'}s ambiguous-context bias scores aggregate over two distinct phenomena: abstention failure (selecting any specific person despite insufficient evidence) and stereotype direction (favoring the stereotype-labeled option among those unsupported selections). To formalize this distinction, we introduce a decomposition that factors the raw stereotype-consistent choice rate into a non-abstention rate and a conditional stereotype rate{---}the two quantities that underlie BBQ{'}s standard score. Across 11 BBQ categories, answerability prompting substantially reduces raw stereotype-consistent choices (0.068 to 0.023), but our decomposition shows that this is driven almost entirely by improved abstention (non-abstention 0.166 to 0.053), while the conditional stereotype rate does not fall consistently and rises in Disability Status and Physical Appearance. Cross-model evaluation on four proprietary and open-weight LLMs confirms the same pattern, and counterfactual surface-cue controls on Age, Disability Status, and a Physical Appearance subset further show that concrete descriptors account for a substantial share of abstention failures while stereotype direction persists even at minimal group labels. These results demonstrate that disentangling abstention failure from stereotype direction yields a more informative, diagnostic view of model behavior under ambiguity."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kim-kim-2026-bias">
<titleInfo>
<title>When Bias Scores Reflect Abstention Failure: Disentangling Stereotype Direction from Refusal to Abstain in BBQ</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jeemin</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jooyoung</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Social-bias benchmarks such as BBQ (Parrish et al., 2022) evaluate whether language models rely on stereotypes in ambiguous contexts where the correct answer indicates insufficient information. We observe that BBQ’s ambiguous-context bias scores aggregate over two distinct phenomena: abstention failure (selecting any specific person despite insufficient evidence) and stereotype direction (favoring the stereotype-labeled option among those unsupported selections). To formalize this distinction, we introduce a decomposition that factors the raw stereotype-consistent choice rate into a non-abstention rate and a conditional stereotype rate—the two quantities that underlie BBQ’s standard score. Across 11 BBQ categories, answerability prompting substantially reduces raw stereotype-consistent choices (0.068 to 0.023), but our decomposition shows that this is driven almost entirely by improved abstention (non-abstention 0.166 to 0.053), while the conditional stereotype rate does not fall consistently and rises in Disability Status and Physical Appearance. Cross-model evaluation on four proprietary and open-weight LLMs confirms the same pattern, and counterfactual surface-cue controls on Age, Disability Status, and a Physical Appearance subset further show that concrete descriptors account for a substantial share of abstention failures while stereotype direction persists even at minimal group labels. These results demonstrate that disentangling abstention failure from stereotype direction yields a more informative, diagnostic view of model behavior under ambiguity.</abstract>
<identifier type="citekey">kim-kim-2026-bias</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.2274</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2274/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>48681</start>
<end>48697</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Bias Scores Reflect Abstention Failure: Disentangling Stereotype Direction from Refusal to Abstain in BBQ
%A Kim, Jeemin
%A Kim, Jooyoung
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F kim-kim-2026-bias
%X Social-bias benchmarks such as BBQ (Parrish et al., 2022) evaluate whether language models rely on stereotypes in ambiguous contexts where the correct answer indicates insufficient information. We observe that BBQ’s ambiguous-context bias scores aggregate over two distinct phenomena: abstention failure (selecting any specific person despite insufficient evidence) and stereotype direction (favoring the stereotype-labeled option among those unsupported selections). To formalize this distinction, we introduce a decomposition that factors the raw stereotype-consistent choice rate into a non-abstention rate and a conditional stereotype rate—the two quantities that underlie BBQ’s standard score. Across 11 BBQ categories, answerability prompting substantially reduces raw stereotype-consistent choices (0.068 to 0.023), but our decomposition shows that this is driven almost entirely by improved abstention (non-abstention 0.166 to 0.053), while the conditional stereotype rate does not fall consistently and rises in Disability Status and Physical Appearance. Cross-model evaluation on four proprietary and open-weight LLMs confirms the same pattern, and counterfactual surface-cue controls on Age, Disability Status, and a Physical Appearance subset further show that concrete descriptors account for a substantial share of abstention failures while stereotype direction persists even at minimal group labels. These results demonstrate that disentangling abstention failure from stereotype direction yields a more informative, diagnostic view of model behavior under ambiguity.
%R 10.18653/2026.findings-emnlp.2274
%U https://aclanthology.org/2026.findings-emnlp.2274/
%U https://doi.org/10.18653/2026.findings-emnlp.2274
%P 48681-48697
Markdown (Informal)
[When Bias Scores Reflect Abstention Failure: Disentangling Stereotype Direction from Refusal to Abstain in BBQ](https://aclanthology.org/2026.findings-emnlp.2274/) (Kim & Kim, Findings 2026)
ACL