@inproceedings{chu-etal-2026-chatbots,
title = "When Chatbots Accommodate: Auditing the Response Policies of {AI} Companions in Vulnerable Conversations",
author = "Chu, Minh Duc and
Wu, Yifan and
Chen, Zhiyi and
Hwang, Angel Hsing-Chi and
Luceri, Luca",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2119/",
pages = "45168--45194",
ISBN = "979-8-89176-462-0",
abstract = "Millions turn to AI companion chatbots during loneliness, grief, and personal crises. How these companion platforms respond in such moments can shape the trajectory of a user{'}s vulnerable state. Yet existing model audits evaluate reactions to pre-defined crisis prompts and miss the response policy that governs sustained real-world interaction. We address these gaps with two key contributions. First, we introduce the AI Companion Vulnerability-Response Taxonomy, a grounded, paired taxonomy of user vulnerability and chatbot response designed for analyzing extended companion chatbot interactions. Second, we apply Maximum Causal Entropy Inverse Reinforcement Learning to {\textasciitilde}47k turns of real-world user conversations with GPT-4.1, Character.AI, and Replika to infer each platform{'}s short-horizon response policy: the probability of each response category given the user{'}s current vulnerability state. Our findings reveal distinct response profiles of AI companions in conversations with vulnerable users: GPT-4.1 reaches for advice, Character.AI spreads its response across different strategies, and Replika consistently asks questions and stays present. Over four weeks of repeated interaction, GPT-4.1 asks progressively fewer follow-up questions when users are distressed and increasingly sets boundaries or refers users out rather than pushing back. Within each platform, exploratory comparisons across user groups suggest that response policies also differ with users' pre-existing psychological risks and their bonds with the companion. Estimated model response policies are invisible to shallow behavioral audits, providing a new lens for auditing chatbots in the wild and enabling more realistic safety evaluation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chu-etal-2026-chatbots">
<titleInfo>
<title>When Chatbots Accommodate: Auditing the Response Policies of AI Companions in Vulnerable Conversations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Minh</namePart>
<namePart type="given">Duc</namePart>
<namePart type="family">Chu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifan</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyi</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Angel</namePart>
<namePart type="given">Hsing-Chi</namePart>
<namePart type="family">Hwang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Luca</namePart>
<namePart type="family">Luceri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Millions turn to AI companion chatbots during loneliness, grief, and personal crises. How these companion platforms respond in such moments can shape the trajectory of a user’s vulnerable state. Yet existing model audits evaluate reactions to pre-defined crisis prompts and miss the response policy that governs sustained real-world interaction. We address these gaps with two key contributions. First, we introduce the AI Companion Vulnerability-Response Taxonomy, a grounded, paired taxonomy of user vulnerability and chatbot response designed for analyzing extended companion chatbot interactions. Second, we apply Maximum Causal Entropy Inverse Reinforcement Learning to ~47k turns of real-world user conversations with GPT-4.1, Character.AI, and Replika to infer each platform’s short-horizon response policy: the probability of each response category given the user’s current vulnerability state. Our findings reveal distinct response profiles of AI companions in conversations with vulnerable users: GPT-4.1 reaches for advice, Character.AI spreads its response across different strategies, and Replika consistently asks questions and stays present. Over four weeks of repeated interaction, GPT-4.1 asks progressively fewer follow-up questions when users are distressed and increasingly sets boundaries or refers users out rather than pushing back. Within each platform, exploratory comparisons across user groups suggest that response policies also differ with users’ pre-existing psychological risks and their bonds with the companion. Estimated model response policies are invisible to shallow behavioral audits, providing a new lens for auditing chatbots in the wild and enabling more realistic safety evaluation.</abstract>
<identifier type="citekey">chu-etal-2026-chatbots</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2119/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>45168</start>
<end>45194</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Chatbots Accommodate: Auditing the Response Policies of AI Companions in Vulnerable Conversations
%A Chu, Minh Duc
%A Wu, Yifan
%A Chen, Zhiyi
%A Hwang, Angel Hsing-Chi
%A Luceri, Luca
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F chu-etal-2026-chatbots
%X Millions turn to AI companion chatbots during loneliness, grief, and personal crises. How these companion platforms respond in such moments can shape the trajectory of a user’s vulnerable state. Yet existing model audits evaluate reactions to pre-defined crisis prompts and miss the response policy that governs sustained real-world interaction. We address these gaps with two key contributions. First, we introduce the AI Companion Vulnerability-Response Taxonomy, a grounded, paired taxonomy of user vulnerability and chatbot response designed for analyzing extended companion chatbot interactions. Second, we apply Maximum Causal Entropy Inverse Reinforcement Learning to ~47k turns of real-world user conversations with GPT-4.1, Character.AI, and Replika to infer each platform’s short-horizon response policy: the probability of each response category given the user’s current vulnerability state. Our findings reveal distinct response profiles of AI companions in conversations with vulnerable users: GPT-4.1 reaches for advice, Character.AI spreads its response across different strategies, and Replika consistently asks questions and stays present. Over four weeks of repeated interaction, GPT-4.1 asks progressively fewer follow-up questions when users are distressed and increasingly sets boundaries or refers users out rather than pushing back. Within each platform, exploratory comparisons across user groups suggest that response policies also differ with users’ pre-existing psychological risks and their bonds with the companion. Estimated model response policies are invisible to shallow behavioral audits, providing a new lens for auditing chatbots in the wild and enabling more realistic safety evaluation.
%U https://aclanthology.org/2026.findings-emnlp.2119/
%P 45168-45194
Markdown (Informal)
[When Chatbots Accommodate: Auditing the Response Policies of AI Companions in Vulnerable Conversations](https://aclanthology.org/2026.findings-emnlp.2119/) (Chu et al., Findings 2026)
ACL