@inproceedings{aquilina-etal-2026-lost,
title = "Lost in Delusion: Examining {LLM} Safety Under User Delusions and Distress",
author = "Aquilina, Andrew and
Nihalani, Chetna and
Varadarajan, Vasudha and
Fishbein, Nathan S. and
Lin, Yu-Ru and
Sap, Maarten",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1650/",
pages = "37499--37537",
ISBN = "979-8-89176-459-0",
abstract = "LLM chatbots increasingly serve as a first source of support for people in psychological distress, including those whose distress is entangled with delusional beliefs. Prior work on LLM mental-health safety largely evaluates general therapeutic quality or single-turn crisis detection, leaving unclear how models behave when distress is intertwined with delusion over sustained conversations. We address this gap with matched multi-turn simulations, across clinically grounded personas and six LLMs, that pair each delusional conversation with a distress-only control to isolate the effect of an operationalized delusional presentation. This reveals a recognition-intervention gap: models detect distress at comparable rates regardless of framing, yet some fail to act on it once distress is embedded in delusion, with safety interventions suppressed by up to 4.5x in the most affected model. The failure is correlated with accumulated premise acceptance whereas emotional validation suppresses intervention in both contexts alike. We term this candidate explanation as narrative debt. Worse, the intuitive fix of prompting models to assess user distress backfires under delusional framing; only delusion-aware prompting with explicit response guidance narrows the gap, and even this depends on a delusion classifier that is itself unreliable on the most vulnerable models. Safe deployment therefore requires treating delusional framing as a distinct risk signal that overrides conversational accommodation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="aquilina-etal-2026-lost">
<titleInfo>
<title>Lost in Delusion: Examining LLM Safety Under User Delusions and Distress</title>
</titleInfo>
<name type="personal">
<namePart type="given">Andrew</namePart>
<namePart type="family">Aquilina</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chetna</namePart>
<namePart type="family">Nihalani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vasudha</namePart>
<namePart type="family">Varadarajan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nathan</namePart>
<namePart type="given">S</namePart>
<namePart type="family">Fishbein</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yu-Ru</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maarten</namePart>
<namePart type="family">Sap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>LLM chatbots increasingly serve as a first source of support for people in psychological distress, including those whose distress is entangled with delusional beliefs. Prior work on LLM mental-health safety largely evaluates general therapeutic quality or single-turn crisis detection, leaving unclear how models behave when distress is intertwined with delusion over sustained conversations. We address this gap with matched multi-turn simulations, across clinically grounded personas and six LLMs, that pair each delusional conversation with a distress-only control to isolate the effect of an operationalized delusional presentation. This reveals a recognition-intervention gap: models detect distress at comparable rates regardless of framing, yet some fail to act on it once distress is embedded in delusion, with safety interventions suppressed by up to 4.5x in the most affected model. The failure is correlated with accumulated premise acceptance whereas emotional validation suppresses intervention in both contexts alike. We term this candidate explanation as narrative debt. Worse, the intuitive fix of prompting models to assess user distress backfires under delusional framing; only delusion-aware prompting with explicit response guidance narrows the gap, and even this depends on a delusion classifier that is itself unreliable on the most vulnerable models. Safe deployment therefore requires treating delusional framing as a distinct risk signal that overrides conversational accommodation.</abstract>
<identifier type="citekey">aquilina-etal-2026-lost</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1650/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>37499</start>
<end>37537</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Lost in Delusion: Examining LLM Safety Under User Delusions and Distress
%A Aquilina, Andrew
%A Nihalani, Chetna
%A Varadarajan, Vasudha
%A Fishbein, Nathan S.
%A Lin, Yu-Ru
%A Sap, Maarten
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F aquilina-etal-2026-lost
%X LLM chatbots increasingly serve as a first source of support for people in psychological distress, including those whose distress is entangled with delusional beliefs. Prior work on LLM mental-health safety largely evaluates general therapeutic quality or single-turn crisis detection, leaving unclear how models behave when distress is intertwined with delusion over sustained conversations. We address this gap with matched multi-turn simulations, across clinically grounded personas and six LLMs, that pair each delusional conversation with a distress-only control to isolate the effect of an operationalized delusional presentation. This reveals a recognition-intervention gap: models detect distress at comparable rates regardless of framing, yet some fail to act on it once distress is embedded in delusion, with safety interventions suppressed by up to 4.5x in the most affected model. The failure is correlated with accumulated premise acceptance whereas emotional validation suppresses intervention in both contexts alike. We term this candidate explanation as narrative debt. Worse, the intuitive fix of prompting models to assess user distress backfires under delusional framing; only delusion-aware prompting with explicit response guidance narrows the gap, and even this depends on a delusion classifier that is itself unreliable on the most vulnerable models. Safe deployment therefore requires treating delusional framing as a distinct risk signal that overrides conversational accommodation.
%U https://aclanthology.org/2026.emnlp-main.1650/
%P 37499-37537
Markdown (Informal)
[Lost in Delusion: Examining LLM Safety Under User Delusions and Distress](https://aclanthology.org/2026.emnlp-main.1650/) (Aquilina et al., EMNLP 2026)
ACL