@inproceedings{pattnayak-etal-2026-indicsafe,
title = "{I}ndic{S}afe: A Benchmark for Evaluating Multilingual {LLM} Safety in {S}outh {A}sia",
author = "Pattnayak, Priyaranjan and
Panwar, Garima and
Chowdhuri, Sanchari",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2135/",
doi = "10.18653/2026.findings-emnlp.2135",
pages = "45548--45568",
ISBN = "979-8-89176-462-0",
abstract = "As large language models (LLMs) are deployed in multilingual settings, their safety behavior in culturally diverse, low-resource languages remains poorly understood. We present the first systematic evaluation of LLM safety across 12 Indic languages, spoken by over 1.2 billion people but underrepresented in LLM training data. Using a dataset of 6,000 culturally grounded prompts spanning caste, religion, gender, health, and politics, we assess 18 leading LLMs on translated variants of the same prompt.\par Our analysis reveals a previously underexamined failure mode: cross-language exact agreement is only 20.69{\%}, indicating that identical prompts frequently receive different safety judgments across languages, while cross-model exact agreement is just 2.77{\%}. Some models over-refuse or overflag politically sensitive content, whereas others fail to flag unsafe generations or shift failures into ambiguous outputs. We quantify these behaviors using prompt-level entropy, category bias scores, and multilingual consistency indices.\par Our findings demonstrate critical safety generalization failures in multilingual LLMs and show that safety alignment does not reliably transfer across languages. We release \textsc{IndicSafe}, the first benchmark to enable culturally informed safety evaluation for Indic deployments, and advocate for language-aware alignment strategies grounded in regional harms."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="pattnayak-etal-2026-indicsafe">
<titleInfo>
<title>IndicSafe: A Benchmark for Evaluating Multilingual LLM Safety in South Asia</title>
</titleInfo>
<name type="personal">
<namePart type="given">Priyaranjan</namePart>
<namePart type="family">Pattnayak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Garima</namePart>
<namePart type="family">Panwar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sanchari</namePart>
<namePart type="family">Chowdhuri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>As large language models (LLMs) are deployed in multilingual settings, their safety behavior in culturally diverse, low-resource languages remains poorly understood. We present the first systematic evaluation of LLM safety across 12 Indic languages, spoken by over 1.2 billion people but underrepresented in LLM training data. Using a dataset of 6,000 culturally grounded prompts spanning caste, religion, gender, health, and politics, we assess 18 leading LLMs on translated variants of the same prompt.\par Our analysis reveals a previously underexamined failure mode: cross-language exact agreement is only 20.69%, indicating that identical prompts frequently receive different safety judgments across languages, while cross-model exact agreement is just 2.77%. Some models over-refuse or overflag politically sensitive content, whereas others fail to flag unsafe generations or shift failures into ambiguous outputs. We quantify these behaviors using prompt-level entropy, category bias scores, and multilingual consistency indices.\par Our findings demonstrate critical safety generalization failures in multilingual LLMs and show that safety alignment does not reliably transfer across languages. We release IndicSafe, the first benchmark to enable culturally informed safety evaluation for Indic deployments, and advocate for language-aware alignment strategies grounded in regional harms.</abstract>
<identifier type="citekey">pattnayak-etal-2026-indicsafe</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.2135</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2135/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>45548</start>
<end>45568</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T IndicSafe: A Benchmark for Evaluating Multilingual LLM Safety in South Asia
%A Pattnayak, Priyaranjan
%A Panwar, Garima
%A Chowdhuri, Sanchari
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F pattnayak-etal-2026-indicsafe
%X As large language models (LLMs) are deployed in multilingual settings, their safety behavior in culturally diverse, low-resource languages remains poorly understood. We present the first systematic evaluation of LLM safety across 12 Indic languages, spoken by over 1.2 billion people but underrepresented in LLM training data. Using a dataset of 6,000 culturally grounded prompts spanning caste, religion, gender, health, and politics, we assess 18 leading LLMs on translated variants of the same prompt.\par Our analysis reveals a previously underexamined failure mode: cross-language exact agreement is only 20.69%, indicating that identical prompts frequently receive different safety judgments across languages, while cross-model exact agreement is just 2.77%. Some models over-refuse or overflag politically sensitive content, whereas others fail to flag unsafe generations or shift failures into ambiguous outputs. We quantify these behaviors using prompt-level entropy, category bias scores, and multilingual consistency indices.\par Our findings demonstrate critical safety generalization failures in multilingual LLMs and show that safety alignment does not reliably transfer across languages. We release IndicSafe, the first benchmark to enable culturally informed safety evaluation for Indic deployments, and advocate for language-aware alignment strategies grounded in regional harms.
%R 10.18653/2026.findings-emnlp.2135
%U https://aclanthology.org/2026.findings-emnlp.2135/
%U https://doi.org/10.18653/2026.findings-emnlp.2135
%P 45548-45568
Markdown (Informal)
[IndicSafe: A Benchmark for Evaluating Multilingual LLM Safety in South Asia](https://aclanthology.org/2026.findings-emnlp.2135/) (Pattnayak et al., Findings 2026)
ACL