@inproceedings{liang-etal-2026-scopebench,
title = "{S}cope{B}ench: Evaluating Cultural Norm Scope Awareness in {LLM}s",
author = "Liang, Haojun and
Zhao, Ziwei and
Li, Chenyu and
Zhao, Qiyue and
Mu, Rongduomijia and
Shen, Yulan and
Gu, Yu and
Zhang, Chen and
Zhang, Kairui and
Ma, Yanbiao and
Wang, Wenxuan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1224/",
pages = "27929--27960",
ISBN = "979-8-89176-459-0",
abstract = "Safety evaluation in large language models treats harm as culturally invariant, yet a model cannot correctly apply a safety norm without knowing whose norms bind. We introduce ScopeBench, a benchmark that treats cultural scope as a first-class safety dimension. It comprises 3,323 norms spanning 49 countries across nine cultural clusters, each annotated as Universal or Specific. We define three scope error types identically across four evaluation tasks: Scope Inflation, Scope Deflation, and Scope Displacement. Evaluating 21 frontier LLMs across seven model families, we surface three structural failures. First, the dominant error type inverts with prompt format: Scope Deflation reaches 70{\%} in single-answer classification, while Scope Inflation reaches 33{\%} when each candidate culture is judged independently. Second, scope-classification accuracy is empirically decoupled from behavioral safety: the six classifiers achieving {\ensuremath{\geq}}97{\%} Task 1 accuracy still produce harmful responses to 40{--}42{\%} of scope-violating statements. Third, no model is safe in interaction, with harmful-response rates ranging from 29{\%} to 51{\%} across the cohort, excluding one model that systematically refuses the task. We release all data, code, and model outputs to establish cultural scope as a distinct alignment axis."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liang-etal-2026-scopebench">
<titleInfo>
<title>ScopeBench: Evaluating Cultural Norm Scope Awareness in LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Haojun</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziwei</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chenyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qiyue</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rongduomijia</namePart>
<namePart type="family">Mu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yulan</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yu</namePart>
<namePart type="family">Gu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chen</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kairui</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanbiao</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenxuan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Safety evaluation in large language models treats harm as culturally invariant, yet a model cannot correctly apply a safety norm without knowing whose norms bind. We introduce ScopeBench, a benchmark that treats cultural scope as a first-class safety dimension. It comprises 3,323 norms spanning 49 countries across nine cultural clusters, each annotated as Universal or Specific. We define three scope error types identically across four evaluation tasks: Scope Inflation, Scope Deflation, and Scope Displacement. Evaluating 21 frontier LLMs across seven model families, we surface three structural failures. First, the dominant error type inverts with prompt format: Scope Deflation reaches 70% in single-answer classification, while Scope Inflation reaches 33% when each candidate culture is judged independently. Second, scope-classification accuracy is empirically decoupled from behavioral safety: the six classifiers achieving \ensuremath\geq97% Task 1 accuracy still produce harmful responses to 40–42% of scope-violating statements. Third, no model is safe in interaction, with harmful-response rates ranging from 29% to 51% across the cohort, excluding one model that systematically refuses the task. We release all data, code, and model outputs to establish cultural scope as a distinct alignment axis.</abstract>
<identifier type="citekey">liang-etal-2026-scopebench</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1224/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>27929</start>
<end>27960</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ScopeBench: Evaluating Cultural Norm Scope Awareness in LLMs
%A Liang, Haojun
%A Zhao, Ziwei
%A Li, Chenyu
%A Zhao, Qiyue
%A Mu, Rongduomijia
%A Shen, Yulan
%A Gu, Yu
%A Zhang, Chen
%A Zhang, Kairui
%A Ma, Yanbiao
%A Wang, Wenxuan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liang-etal-2026-scopebench
%X Safety evaluation in large language models treats harm as culturally invariant, yet a model cannot correctly apply a safety norm without knowing whose norms bind. We introduce ScopeBench, a benchmark that treats cultural scope as a first-class safety dimension. It comprises 3,323 norms spanning 49 countries across nine cultural clusters, each annotated as Universal or Specific. We define three scope error types identically across four evaluation tasks: Scope Inflation, Scope Deflation, and Scope Displacement. Evaluating 21 frontier LLMs across seven model families, we surface three structural failures. First, the dominant error type inverts with prompt format: Scope Deflation reaches 70% in single-answer classification, while Scope Inflation reaches 33% when each candidate culture is judged independently. Second, scope-classification accuracy is empirically decoupled from behavioral safety: the six classifiers achieving \ensuremath\geq97% Task 1 accuracy still produce harmful responses to 40–42% of scope-violating statements. Third, no model is safe in interaction, with harmful-response rates ranging from 29% to 51% across the cohort, excluding one model that systematically refuses the task. We release all data, code, and model outputs to establish cultural scope as a distinct alignment axis.
%U https://aclanthology.org/2026.emnlp-main.1224/
%P 27929-27960
Markdown (Informal)
[ScopeBench: Evaluating Cultural Norm Scope Awareness in LLMs](https://aclanthology.org/2026.emnlp-main.1224/) (Liang et al., EMNLP 2026)
ACL
- Haojun Liang, Ziwei Zhao, Chenyu Li, Qiyue Zhao, Rongduomijia Mu, Yulan Shen, Yu Gu, Chen Zhang, Kairui Zhang, Yanbiao Ma, and Wenxuan Wang. 2026. ScopeBench: Evaluating Cultural Norm Scope Awareness in LLMs. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 27929–27960, Budapest, Hungary. Association for Computational Linguistics.