@inproceedings{ye-etal-2026-harmful,
title = "When Harmful Intent Lies Between the Lines: Pragmatics-Based Automated Red Teaming for {LLM} Safety Evaluation",
author = "Ye, Shuying and
Ternero, Carolina and
Xia, Fei",
editor = "Calabrese, Agostina and
Davidson, Thomas and
de Kock, Christine and
Khurana, Urja and
Marchiori Manerba, Marta and
Piot, Paloma and
Talat, Zeerak",
booktitle = "Proceedings of the 10th Workshop on Online Abuse and Harms ({WOAH})",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.woah-1.14/",
pages = "157--172",
ISBN = "979-8-89176-487-3",
abstract = "As large language models (LLMs) are increasingly deployed in real-world applications, ensuring their safety is a critical concern. Most safety evaluations focus on direct harmful requests, overlooking cases where the same harmful intent is disguised through indirect, polite, hypothetical, or contextually framed language. To address this limitation, we propose a pragmatics-based automated red-teaming framework that tests whether LLM safety behavior changes when malicious intent is repackaged through different pragmatic strategies. First, we develop a linguistically grounded taxonomy of seven pragmatic strategies{---}presupposition, implicature, indirect speech acts, denial disclaimers, politeness, hypothetical framing, and contextual framing{---}and use it to generate adversarial prompts across seven harm categories. Second, we evaluate these prompts on three Target Models (GPT-4o-mini, Qwen2.5-7B, Llama-3.3-70B), with Claude Haiku as judge and human validation. Our experiments show that pragmatically transformed prompts consistently yield higher Attack Success Rate (ASR) than original seed prompts, with contextual framing emerging as particularly effective. These results suggest LLM safety evaluation should account for pragmatic meaning and inferred intent, not just explicit harmful content."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ye-etal-2026-harmful">
<titleInfo>
<title>When Harmful Intent Lies Between the Lines: Pragmatics-Based Automated Red Teaming for LLM Safety Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shuying</namePart>
<namePart type="family">Ye</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Carolina</namePart>
<namePart type="family">Ternero</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fei</namePart>
<namePart type="family">Xia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 10th Workshop on Online Abuse and Harms (WOAH)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Agostina</namePart>
<namePart type="family">Calabrese</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thomas</namePart>
<namePart type="family">Davidson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christine</namePart>
<namePart type="family">de Kock</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Urja</namePart>
<namePart type="family">Khurana</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marta</namePart>
<namePart type="family">Marchiori Manerba</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paloma</namePart>
<namePart type="family">Piot</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zeerak</namePart>
<namePart type="family">Talat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-487-3</identifier>
</relatedItem>
<abstract>As large language models (LLMs) are increasingly deployed in real-world applications, ensuring their safety is a critical concern. Most safety evaluations focus on direct harmful requests, overlooking cases where the same harmful intent is disguised through indirect, polite, hypothetical, or contextually framed language. To address this limitation, we propose a pragmatics-based automated red-teaming framework that tests whether LLM safety behavior changes when malicious intent is repackaged through different pragmatic strategies. First, we develop a linguistically grounded taxonomy of seven pragmatic strategies—presupposition, implicature, indirect speech acts, denial disclaimers, politeness, hypothetical framing, and contextual framing—and use it to generate adversarial prompts across seven harm categories. Second, we evaluate these prompts on three Target Models (GPT-4o-mini, Qwen2.5-7B, Llama-3.3-70B), with Claude Haiku as judge and human validation. Our experiments show that pragmatically transformed prompts consistently yield higher Attack Success Rate (ASR) than original seed prompts, with contextual framing emerging as particularly effective. These results suggest LLM safety evaluation should account for pragmatic meaning and inferred intent, not just explicit harmful content.</abstract>
<identifier type="citekey">ye-etal-2026-harmful</identifier>
<location>
<url>https://aclanthology.org/2026.woah-1.14/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>157</start>
<end>172</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Harmful Intent Lies Between the Lines: Pragmatics-Based Automated Red Teaming for LLM Safety Evaluation
%A Ye, Shuying
%A Ternero, Carolina
%A Xia, Fei
%Y Calabrese, Agostina
%Y Davidson, Thomas
%Y de Kock, Christine
%Y Khurana, Urja
%Y Marchiori Manerba, Marta
%Y Piot, Paloma
%Y Talat, Zeerak
%S Proceedings of the 10th Workshop on Online Abuse and Harms (WOAH)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-487-3
%F ye-etal-2026-harmful
%X As large language models (LLMs) are increasingly deployed in real-world applications, ensuring their safety is a critical concern. Most safety evaluations focus on direct harmful requests, overlooking cases where the same harmful intent is disguised through indirect, polite, hypothetical, or contextually framed language. To address this limitation, we propose a pragmatics-based automated red-teaming framework that tests whether LLM safety behavior changes when malicious intent is repackaged through different pragmatic strategies. First, we develop a linguistically grounded taxonomy of seven pragmatic strategies—presupposition, implicature, indirect speech acts, denial disclaimers, politeness, hypothetical framing, and contextual framing—and use it to generate adversarial prompts across seven harm categories. Second, we evaluate these prompts on three Target Models (GPT-4o-mini, Qwen2.5-7B, Llama-3.3-70B), with Claude Haiku as judge and human validation. Our experiments show that pragmatically transformed prompts consistently yield higher Attack Success Rate (ASR) than original seed prompts, with contextual framing emerging as particularly effective. These results suggest LLM safety evaluation should account for pragmatic meaning and inferred intent, not just explicit harmful content.
%U https://aclanthology.org/2026.woah-1.14/
%P 157-172
Markdown (Informal)
[When Harmful Intent Lies Between the Lines: Pragmatics-Based Automated Red Teaming for LLM Safety Evaluation](https://aclanthology.org/2026.woah-1.14/) (Ye et al., WOAH 2026)
ACL