@inproceedings{kang-etal-2026-scene,
title = "{SCENE}: A Scenario Completion Benchmark for Measuring Social Bias in Large Language Models",
author = "Kang, Migyeong and
Jung, Aecheon and
Jang, Eunchae and
Kim, Minseok and
Sundar, S. Shyam and
Hong, Sungeun and
Choi, YunSeok and
Han, Jinyoung",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1092/",
pages = "23126--23148",
ISBN = "979-8-89176-462-0",
abstract = "Most existing benchmarks evaluate social bias in large language models (LLMs) using multiple-choice questions (MCQ) grounded in stereotypical contexts. While MCQ formats enable straightforward quantification, how LLMs manifest social bias in task-specific contexts and under different output constraints remains underexplored. We introduce SCENE, a benchmark for measuring the social bias shown by LLMs in scenario-completion tasks. Given short scenarios involving social groups, models must complete key elements of the situation through two tasks: (i) Character Specification, which asks models to assign social attributes to characters, and (ii) Plot Continuation, which asks models to select the most plausible narrative continuation. Combining templates adapted from prior benchmarks with newly collected templates through a university-wide competition, we construct a dataset of over 75,000 Korean instances. Experiments show that bias measurements are highly sensitive to task context and output constraints, even for identical underlying scenarios, suggesting that widely used MCQ-based evaluations may not fully capture certain manifestations of stereotypes. SCENE is publicly available at https://github.com/DSAIL-SKKU/SCENE-EMNLP-2026."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kang-etal-2026-scene">
<titleInfo>
<title>SCENE: A Scenario Completion Benchmark for Measuring Social Bias in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Migyeong</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aecheon</namePart>
<namePart type="family">Jung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eunchae</namePart>
<namePart type="family">Jang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minseok</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">S</namePart>
<namePart type="given">Shyam</namePart>
<namePart type="family">Sundar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sungeun</namePart>
<namePart type="family">Hong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">YunSeok</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinyoung</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Most existing benchmarks evaluate social bias in large language models (LLMs) using multiple-choice questions (MCQ) grounded in stereotypical contexts. While MCQ formats enable straightforward quantification, how LLMs manifest social bias in task-specific contexts and under different output constraints remains underexplored. We introduce SCENE, a benchmark for measuring the social bias shown by LLMs in scenario-completion tasks. Given short scenarios involving social groups, models must complete key elements of the situation through two tasks: (i) Character Specification, which asks models to assign social attributes to characters, and (ii) Plot Continuation, which asks models to select the most plausible narrative continuation. Combining templates adapted from prior benchmarks with newly collected templates through a university-wide competition, we construct a dataset of over 75,000 Korean instances. Experiments show that bias measurements are highly sensitive to task context and output constraints, even for identical underlying scenarios, suggesting that widely used MCQ-based evaluations may not fully capture certain manifestations of stereotypes. SCENE is publicly available at https://github.com/DSAIL-SKKU/SCENE-EMNLP-2026.</abstract>
<identifier type="citekey">kang-etal-2026-scene</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1092/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>23126</start>
<end>23148</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SCENE: A Scenario Completion Benchmark for Measuring Social Bias in Large Language Models
%A Kang, Migyeong
%A Jung, Aecheon
%A Jang, Eunchae
%A Kim, Minseok
%A Sundar, S. Shyam
%A Hong, Sungeun
%A Choi, YunSeok
%A Han, Jinyoung
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F kang-etal-2026-scene
%X Most existing benchmarks evaluate social bias in large language models (LLMs) using multiple-choice questions (MCQ) grounded in stereotypical contexts. While MCQ formats enable straightforward quantification, how LLMs manifest social bias in task-specific contexts and under different output constraints remains underexplored. We introduce SCENE, a benchmark for measuring the social bias shown by LLMs in scenario-completion tasks. Given short scenarios involving social groups, models must complete key elements of the situation through two tasks: (i) Character Specification, which asks models to assign social attributes to characters, and (ii) Plot Continuation, which asks models to select the most plausible narrative continuation. Combining templates adapted from prior benchmarks with newly collected templates through a university-wide competition, we construct a dataset of over 75,000 Korean instances. Experiments show that bias measurements are highly sensitive to task context and output constraints, even for identical underlying scenarios, suggesting that widely used MCQ-based evaluations may not fully capture certain manifestations of stereotypes. SCENE is publicly available at https://github.com/DSAIL-SKKU/SCENE-EMNLP-2026.
%U https://aclanthology.org/2026.findings-emnlp.1092/
%P 23126-23148
Markdown (Informal)
[SCENE: A Scenario Completion Benchmark for Measuring Social Bias in Large Language Models](https://aclanthology.org/2026.findings-emnlp.1092/) (Kang et al., Findings 2026)
ACL
- Migyeong Kang, Aecheon Jung, Eunchae Jang, Minseok Kim, S. Shyam Sundar, Sungeun Hong, YunSeok Choi, and Jinyoung Han. 2026. SCENE: A Scenario Completion Benchmark for Measuring Social Bias in Large Language Models. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 23126–23148, Budapest, Hungary. Association for Computational Linguistics.