@inproceedings{bozdogan-etal-2026-many,
title = "How Many Instances Are Enough? Minimum Subsample Size Prediction for Stereotype Evaluation",
author = "Bozdogan, Bora and
Vosoughi, Soroush and
Ma, Weicheng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2043/",
pages = "43467--43484",
ISBN = "979-8-89176-462-0",
abstract = "Stereotype evaluation for large language models relies on heavily annotated benchmarks, yet such evaluations are constrained by limited instance availability and are often sensitive to which examples are included. We propose a framework for predicting the minimum number of evaluation instances needed to preserve benchmark-level stereotype scores under specified error and confidence criteria. Our method provides a statistically grounded way to determine how much a benchmark can be reduced without materially changing the resulting evaluation outcome. We support the framework with theoretical analysis and extensive experiments on three widely used stereotype benchmarks, CrowS-Pairs, StereoSet, and BBQ. Across datasets, models, and tolerance settings, the proposed method provides reliable subsample-size guidance without exception. As an additional extension, we show that our approach can also guide compact synthetic stereotype evaluation. Overall, our results suggest that benchmark-level stereotype evaluation can often be conducted with far fewer instances than the original datasets require, offering a practical and principled approach to more efficient bias assessment."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="bozdogan-etal-2026-many">
<titleInfo>
<title>How Many Instances Are Enough? Minimum Subsample Size Prediction for Stereotype Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Bora</namePart>
<namePart type="family">Bozdogan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Soroush</namePart>
<namePart type="family">Vosoughi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weicheng</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Stereotype evaluation for large language models relies on heavily annotated benchmarks, yet such evaluations are constrained by limited instance availability and are often sensitive to which examples are included. We propose a framework for predicting the minimum number of evaluation instances needed to preserve benchmark-level stereotype scores under specified error and confidence criteria. Our method provides a statistically grounded way to determine how much a benchmark can be reduced without materially changing the resulting evaluation outcome. We support the framework with theoretical analysis and extensive experiments on three widely used stereotype benchmarks, CrowS-Pairs, StereoSet, and BBQ. Across datasets, models, and tolerance settings, the proposed method provides reliable subsample-size guidance without exception. As an additional extension, we show that our approach can also guide compact synthetic stereotype evaluation. Overall, our results suggest that benchmark-level stereotype evaluation can often be conducted with far fewer instances than the original datasets require, offering a practical and principled approach to more efficient bias assessment.</abstract>
<identifier type="citekey">bozdogan-etal-2026-many</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2043/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>43467</start>
<end>43484</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T How Many Instances Are Enough? Minimum Subsample Size Prediction for Stereotype Evaluation
%A Bozdogan, Bora
%A Vosoughi, Soroush
%A Ma, Weicheng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F bozdogan-etal-2026-many
%X Stereotype evaluation for large language models relies on heavily annotated benchmarks, yet such evaluations are constrained by limited instance availability and are often sensitive to which examples are included. We propose a framework for predicting the minimum number of evaluation instances needed to preserve benchmark-level stereotype scores under specified error and confidence criteria. Our method provides a statistically grounded way to determine how much a benchmark can be reduced without materially changing the resulting evaluation outcome. We support the framework with theoretical analysis and extensive experiments on three widely used stereotype benchmarks, CrowS-Pairs, StereoSet, and BBQ. Across datasets, models, and tolerance settings, the proposed method provides reliable subsample-size guidance without exception. As an additional extension, we show that our approach can also guide compact synthetic stereotype evaluation. Overall, our results suggest that benchmark-level stereotype evaluation can often be conducted with far fewer instances than the original datasets require, offering a practical and principled approach to more efficient bias assessment.
%U https://aclanthology.org/2026.findings-emnlp.2043/
%P 43467-43484
Markdown (Informal)
[How Many Instances Are Enough? Minimum Subsample Size Prediction for Stereotype Evaluation](https://aclanthology.org/2026.findings-emnlp.2043/) (Bozdogan et al., Findings 2026)
ACL