@inproceedings{oroojlooy-patel-2026-one,
title = "No One Model Catches Every Harm: Benchmarking Content Moderation Across Safety Scenarios",
author = "Oroojlooy, Afshin and
Patel, Hitesh Laxmichand",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1748/",
doi = "10.18653/2026.emnlp-main.1748",
pages = "39879--39903",
ISBN = "979-8-89176-459-0",
abstract = "Large Language Models (LLMs) are increasingly deployed in real-world applications, yet remain vulnerable to harms ranging from adversarial jailbreaks to implicit hate. Specialized moderators and general-purpose LLMs are both used as safety layers, but there is little systematic guidance on which model to choose for which harm. We present the most comprehensive evaluation of LLM safety to date, testing 53 models on 11 datasets organized into four challenge categories, under both prompt-only (Q) and prompt-with-response (QA) settings. Large frontier models that lead on one category fall behind smaller specialized models on others, and real-world conversational safety remains unsolved across model families. These findings challenge the assumption that scale alone ensures safety and offer practitioners a structured basis for model selection."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="oroojlooy-patel-2026-one">
<titleInfo>
<title>No One Model Catches Every Harm: Benchmarking Content Moderation Across Safety Scenarios</title>
</titleInfo>
<name type="personal">
<namePart type="given">Afshin</namePart>
<namePart type="family">Oroojlooy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hitesh</namePart>
<namePart type="given">Laxmichand</namePart>
<namePart type="family">Patel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) are increasingly deployed in real-world applications, yet remain vulnerable to harms ranging from adversarial jailbreaks to implicit hate. Specialized moderators and general-purpose LLMs are both used as safety layers, but there is little systematic guidance on which model to choose for which harm. We present the most comprehensive evaluation of LLM safety to date, testing 53 models on 11 datasets organized into four challenge categories, under both prompt-only (Q) and prompt-with-response (QA) settings. Large frontier models that lead on one category fall behind smaller specialized models on others, and real-world conversational safety remains unsolved across model families. These findings challenge the assumption that scale alone ensures safety and offer practitioners a structured basis for model selection.</abstract>
<identifier type="citekey">oroojlooy-patel-2026-one</identifier>
<identifier type="doi">10.18653/2026.emnlp-main.1748</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1748/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>39879</start>
<end>39903</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T No One Model Catches Every Harm: Benchmarking Content Moderation Across Safety Scenarios
%A Oroojlooy, Afshin
%A Patel, Hitesh Laxmichand
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F oroojlooy-patel-2026-one
%X Large Language Models (LLMs) are increasingly deployed in real-world applications, yet remain vulnerable to harms ranging from adversarial jailbreaks to implicit hate. Specialized moderators and general-purpose LLMs are both used as safety layers, but there is little systematic guidance on which model to choose for which harm. We present the most comprehensive evaluation of LLM safety to date, testing 53 models on 11 datasets organized into four challenge categories, under both prompt-only (Q) and prompt-with-response (QA) settings. Large frontier models that lead on one category fall behind smaller specialized models on others, and real-world conversational safety remains unsolved across model families. These findings challenge the assumption that scale alone ensures safety and offer practitioners a structured basis for model selection.
%R 10.18653/2026.emnlp-main.1748
%U https://aclanthology.org/2026.emnlp-main.1748/
%U https://doi.org/10.18653/2026.emnlp-main.1748
%P 39879-39903
Markdown (Informal)
[No One Model Catches Every Harm: Benchmarking Content Moderation Across Safety Scenarios](https://aclanthology.org/2026.emnlp-main.1748/) (Oroojlooy & Patel, EMNLP 2026)
ACL