@inproceedings{hirota-etal-2026-guardrail,
title = "Guardrail Models Collapse under Adversarial Paraphrases",
author = "Hirota, Yusuke and
Ghosh, Shaona and
Wang, Yu-Chiang Frank and
Hachiuma, Ryo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.907/",
pages = "20895--20917",
ISBN = "979-8-89176-459-0",
abstract = "We study the vulnerability of guardrail models to adversarial paraphrases that exploit superficial textual variation: meaning-preserving changes that alter expressions spuriously correlated with safety labels. While prior work probes this vulnerability through simple heuristics, such as keyword-label co-occurrence statistics, we argue that guardrails may rely on more complex textual expressions involving phrases or syntactic structures that cannot be captured by heuristics. To enable systematic evaluation of such hidden vulnerabilities, we propose ASPro, an agentic adversarial evaluation framework that automatically synthesizes meaning-preserving paraphrases containing expressions that contradict guardrails' learned spurious expression-label correlations. Applying ASPro to five safety benchmarks and 13 guardrail models, we find that the average F1 drops from 77.83 to 40.25, substantially more than under existing heuristic baselines. Moreover, on ASPro prompts, guardrail models often fail to outperform their base LLMs before safety fine-tuning. These results suggest that current safety fine-tuning may overfit to spurious expressions rather than truly improve guardrails' understanding of prompt safety."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hirota-etal-2026-guardrail">
<titleInfo>
<title>Guardrail Models Collapse under Adversarial Paraphrases</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yusuke</namePart>
<namePart type="family">Hirota</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shaona</namePart>
<namePart type="family">Ghosh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yu-Chiang</namePart>
<namePart type="given">Frank</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryo</namePart>
<namePart type="family">Hachiuma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>We study the vulnerability of guardrail models to adversarial paraphrases that exploit superficial textual variation: meaning-preserving changes that alter expressions spuriously correlated with safety labels. While prior work probes this vulnerability through simple heuristics, such as keyword-label co-occurrence statistics, we argue that guardrails may rely on more complex textual expressions involving phrases or syntactic structures that cannot be captured by heuristics. To enable systematic evaluation of such hidden vulnerabilities, we propose ASPro, an agentic adversarial evaluation framework that automatically synthesizes meaning-preserving paraphrases containing expressions that contradict guardrails’ learned spurious expression-label correlations. Applying ASPro to five safety benchmarks and 13 guardrail models, we find that the average F1 drops from 77.83 to 40.25, substantially more than under existing heuristic baselines. Moreover, on ASPro prompts, guardrail models often fail to outperform their base LLMs before safety fine-tuning. These results suggest that current safety fine-tuning may overfit to spurious expressions rather than truly improve guardrails’ understanding of prompt safety.</abstract>
<identifier type="citekey">hirota-etal-2026-guardrail</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.907/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>20895</start>
<end>20917</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Guardrail Models Collapse under Adversarial Paraphrases
%A Hirota, Yusuke
%A Ghosh, Shaona
%A Wang, Yu-Chiang Frank
%A Hachiuma, Ryo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F hirota-etal-2026-guardrail
%X We study the vulnerability of guardrail models to adversarial paraphrases that exploit superficial textual variation: meaning-preserving changes that alter expressions spuriously correlated with safety labels. While prior work probes this vulnerability through simple heuristics, such as keyword-label co-occurrence statistics, we argue that guardrails may rely on more complex textual expressions involving phrases or syntactic structures that cannot be captured by heuristics. To enable systematic evaluation of such hidden vulnerabilities, we propose ASPro, an agentic adversarial evaluation framework that automatically synthesizes meaning-preserving paraphrases containing expressions that contradict guardrails’ learned spurious expression-label correlations. Applying ASPro to five safety benchmarks and 13 guardrail models, we find that the average F1 drops from 77.83 to 40.25, substantially more than under existing heuristic baselines. Moreover, on ASPro prompts, guardrail models often fail to outperform their base LLMs before safety fine-tuning. These results suggest that current safety fine-tuning may overfit to spurious expressions rather than truly improve guardrails’ understanding of prompt safety.
%U https://aclanthology.org/2026.emnlp-main.907/
%P 20895-20917
Markdown (Informal)
[Guardrail Models Collapse under Adversarial Paraphrases](https://aclanthology.org/2026.emnlp-main.907/) (Hirota et al., EMNLP 2026)
ACL
- Yusuke Hirota, Shaona Ghosh, Yu-Chiang Frank Wang, and Ryo Hachiuma. 2026. Guardrail Models Collapse under Adversarial Paraphrases. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 20895–20917, Budapest, Hungary. Association for Computational Linguistics.