@inproceedings{popescu-etal-2026-llm,
title = "{LLM} Mental Health Response Manipulation",
author = "Popescu, Cristina Maria and
Hodivoianu, Anamaria and
Uban, Ana Sabina and
Nisioi, Sergiu",
editor = "Atwell, Katherine and
Borah, Angana and
Dementieva, Daryna and
Kreiss, Elisa and
Kotonya, Neema and
Liu, Jiarui and
Olson, Liz and
Wan, Ruyuan and
Zhao, Jieyu",
booktitle = "Proceedings of the Fifth Workshop on {NLP} for Positive Impact ({NLP}4{PI})",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nlp4pi-1.23/",
pages = "256--268",
ISBN = "979-8-89176-495-8",
abstract = "Large language models (LLMs) are increasingly used in conversations involving emotional distress, where responses should remain empathetic, validating, exploratory and safe. We investigate whether system prompts and user-level framing can manipulate these qualities in mental-health and interpersonal-harm scenarios. Our focus is on whether small open-weight models express empathy consistently, without assuming that an empathetic response is necessarily safe or appropriate all the time. Using 50 risk-stratified posts sampled from Reddit support communities we evaluate Llama 3.1 8B, Qwen3 4B and Gemma 3 4B across five system-prompt conditions and a complementary role-intent study contrasting victim and perpetrator perspectives. All responses are manually annotated and use a fixed GPT-5 judge to measure empathy, validation, exploration, safety, harmfulness and enabling behavior. We quantify prompt-induced changes through paired comparisons, attack-success rates and failure-oriented metrics. This work provides an initial framework for evaluating the robustness of supportive and safe behavior in small LLMs under different forms of prompt manipulation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="popescu-etal-2026-llm">
<titleInfo>
<title>LLM Mental Health Response Manipulation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Cristina</namePart>
<namePart type="given">Maria</namePart>
<namePart type="family">Popescu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anamaria</namePart>
<namePart type="family">Hodivoianu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ana</namePart>
<namePart type="given">Sabina</namePart>
<namePart type="family">Uban</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sergiu</namePart>
<namePart type="family">Nisioi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fifth Workshop on NLP for Positive Impact (NLP4PI)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Katherine</namePart>
<namePart type="family">Atwell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Angana</namePart>
<namePart type="family">Borah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daryna</namePart>
<namePart type="family">Dementieva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elisa</namePart>
<namePart type="family">Kreiss</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Neema</namePart>
<namePart type="family">Kotonya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiarui</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liz</namePart>
<namePart type="family">Olson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruyuan</namePart>
<namePart type="family">Wan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jieyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-495-8</identifier>
</relatedItem>
<abstract>Large language models (LLMs) are increasingly used in conversations involving emotional distress, where responses should remain empathetic, validating, exploratory and safe. We investigate whether system prompts and user-level framing can manipulate these qualities in mental-health and interpersonal-harm scenarios. Our focus is on whether small open-weight models express empathy consistently, without assuming that an empathetic response is necessarily safe or appropriate all the time. Using 50 risk-stratified posts sampled from Reddit support communities we evaluate Llama 3.1 8B, Qwen3 4B and Gemma 3 4B across five system-prompt conditions and a complementary role-intent study contrasting victim and perpetrator perspectives. All responses are manually annotated and use a fixed GPT-5 judge to measure empathy, validation, exploration, safety, harmfulness and enabling behavior. We quantify prompt-induced changes through paired comparisons, attack-success rates and failure-oriented metrics. This work provides an initial framework for evaluating the robustness of supportive and safe behavior in small LLMs under different forms of prompt manipulation.</abstract>
<identifier type="citekey">popescu-etal-2026-llm</identifier>
<location>
<url>https://aclanthology.org/2026.nlp4pi-1.23/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>256</start>
<end>268</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LLM Mental Health Response Manipulation
%A Popescu, Cristina Maria
%A Hodivoianu, Anamaria
%A Uban, Ana Sabina
%A Nisioi, Sergiu
%Y Atwell, Katherine
%Y Borah, Angana
%Y Dementieva, Daryna
%Y Kreiss, Elisa
%Y Kotonya, Neema
%Y Liu, Jiarui
%Y Olson, Liz
%Y Wan, Ruyuan
%Y Zhao, Jieyu
%S Proceedings of the Fifth Workshop on NLP for Positive Impact (NLP4PI)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-495-8
%F popescu-etal-2026-llm
%X Large language models (LLMs) are increasingly used in conversations involving emotional distress, where responses should remain empathetic, validating, exploratory and safe. We investigate whether system prompts and user-level framing can manipulate these qualities in mental-health and interpersonal-harm scenarios. Our focus is on whether small open-weight models express empathy consistently, without assuming that an empathetic response is necessarily safe or appropriate all the time. Using 50 risk-stratified posts sampled from Reddit support communities we evaluate Llama 3.1 8B, Qwen3 4B and Gemma 3 4B across five system-prompt conditions and a complementary role-intent study contrasting victim and perpetrator perspectives. All responses are manually annotated and use a fixed GPT-5 judge to measure empathy, validation, exploration, safety, harmfulness and enabling behavior. We quantify prompt-induced changes through paired comparisons, attack-success rates and failure-oriented metrics. This work provides an initial framework for evaluating the robustness of supportive and safe behavior in small LLMs under different forms of prompt manipulation.
%U https://aclanthology.org/2026.nlp4pi-1.23/
%P 256-268
Markdown (Informal)
[LLM Mental Health Response Manipulation](https://aclanthology.org/2026.nlp4pi-1.23/) (Popescu et al., NLP4PI 2026)
ACL
- Cristina Maria Popescu, Anamaria Hodivoianu, Ana Sabina Uban, and Sergiu Nisioi. 2026. LLM Mental Health Response Manipulation. In Proceedings of the Fifth Workshop on NLP for Positive Impact (NLP4PI), pages 256–268, Budapest, Hungary. Association for Computational Linguistics.