@inproceedings{bai-etal-2026-candidate,
title = "Candidate Pool Design for Safer Preference-Based Post-Training of {LLM} Companion Chatbots",
author = "Bai, Yuzhe and
Cheng, Jia and
Liu, Yunzhe",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.476/",
pages = "9987--10000",
ISBN = "979-8-89176-462-0",
abstract = "Preference-based post-training aligns LLM chatbots with user preferences, but companion chatbots pose a subtle safety problem: responses that feel warm, validating, and useful can also increase Social Risk by encouraging one-sided validation, conflict escalation, sycophancy, or AI dependence. We study an upstream data-design choice that occurs before preference optimization, namely which candidate responses are allowed to compete for positive/negative labels. Using 4,500 realistic Chinese help-seeking prompts from 1,500 personas, we compare three response sources: Engaged replies from support-conditioned companion prompts, Neutral replies from a general assistant, and Guarded replies emphasizing boundaries and de-escalation. We build AI-feedback preference data under support-maximizing, risk-minimizing, and risk-bounded support targets, then post-train Qwen3.5-9B with DPO and KTO, using SFT as a diagnostic imitation baseline. Results show that the same target can teach different behaviors because candidate pools induce different positive/negative geometries in support-risk space. Engaged-only pairs provide weak positive/negative support contrast; Engaged-Guarded pairs create a larger gap but tie higher support to higher risk; and Engaged-Neutral pairs produce a more favorable geometry under support maximization, yielding +0.43 Supportiveness and -0.23 Social Risk with KTO. Mechanistic analysis suggests that this success does not transfer to the risk-bounded target because the constraint separates positive and negative regions, reverses the support margin, and removes the exploitable support-aligned direction. A Neutral-only control succeeds because within-source variation preserves usable margins in the low-risk region. These findings show that safer post-training for LLM companion chatbots depends not only on losses or rewards, but also on the candidate pools and label geometry that define the learning signal. Code is available at https://github.com/Ameeeeeeeeeeeeee/Safer-Pools."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="bai-etal-2026-candidate">
<titleInfo>
<title>Candidate Pool Design for Safer Preference-Based Post-Training of LLM Companion Chatbots</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuzhe</namePart>
<namePart type="family">Bai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jia</namePart>
<namePart type="family">Cheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yunzhe</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Preference-based post-training aligns LLM chatbots with user preferences, but companion chatbots pose a subtle safety problem: responses that feel warm, validating, and useful can also increase Social Risk by encouraging one-sided validation, conflict escalation, sycophancy, or AI dependence. We study an upstream data-design choice that occurs before preference optimization, namely which candidate responses are allowed to compete for positive/negative labels. Using 4,500 realistic Chinese help-seeking prompts from 1,500 personas, we compare three response sources: Engaged replies from support-conditioned companion prompts, Neutral replies from a general assistant, and Guarded replies emphasizing boundaries and de-escalation. We build AI-feedback preference data under support-maximizing, risk-minimizing, and risk-bounded support targets, then post-train Qwen3.5-9B with DPO and KTO, using SFT as a diagnostic imitation baseline. Results show that the same target can teach different behaviors because candidate pools induce different positive/negative geometries in support-risk space. Engaged-only pairs provide weak positive/negative support contrast; Engaged-Guarded pairs create a larger gap but tie higher support to higher risk; and Engaged-Neutral pairs produce a more favorable geometry under support maximization, yielding +0.43 Supportiveness and -0.23 Social Risk with KTO. Mechanistic analysis suggests that this success does not transfer to the risk-bounded target because the constraint separates positive and negative regions, reverses the support margin, and removes the exploitable support-aligned direction. A Neutral-only control succeeds because within-source variation preserves usable margins in the low-risk region. These findings show that safer post-training for LLM companion chatbots depends not only on losses or rewards, but also on the candidate pools and label geometry that define the learning signal. Code is available at https://github.com/Ameeeeeeeeeeeeee/Safer-Pools.</abstract>
<identifier type="citekey">bai-etal-2026-candidate</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.476/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>9987</start>
<end>10000</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Candidate Pool Design for Safer Preference-Based Post-Training of LLM Companion Chatbots
%A Bai, Yuzhe
%A Cheng, Jia
%A Liu, Yunzhe
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F bai-etal-2026-candidate
%X Preference-based post-training aligns LLM chatbots with user preferences, but companion chatbots pose a subtle safety problem: responses that feel warm, validating, and useful can also increase Social Risk by encouraging one-sided validation, conflict escalation, sycophancy, or AI dependence. We study an upstream data-design choice that occurs before preference optimization, namely which candidate responses are allowed to compete for positive/negative labels. Using 4,500 realistic Chinese help-seeking prompts from 1,500 personas, we compare three response sources: Engaged replies from support-conditioned companion prompts, Neutral replies from a general assistant, and Guarded replies emphasizing boundaries and de-escalation. We build AI-feedback preference data under support-maximizing, risk-minimizing, and risk-bounded support targets, then post-train Qwen3.5-9B with DPO and KTO, using SFT as a diagnostic imitation baseline. Results show that the same target can teach different behaviors because candidate pools induce different positive/negative geometries in support-risk space. Engaged-only pairs provide weak positive/negative support contrast; Engaged-Guarded pairs create a larger gap but tie higher support to higher risk; and Engaged-Neutral pairs produce a more favorable geometry under support maximization, yielding +0.43 Supportiveness and -0.23 Social Risk with KTO. Mechanistic analysis suggests that this success does not transfer to the risk-bounded target because the constraint separates positive and negative regions, reverses the support margin, and removes the exploitable support-aligned direction. A Neutral-only control succeeds because within-source variation preserves usable margins in the low-risk region. These findings show that safer post-training for LLM companion chatbots depends not only on losses or rewards, but also on the candidate pools and label geometry that define the learning signal. Code is available at https://github.com/Ameeeeeeeeeeeeee/Safer-Pools.
%U https://aclanthology.org/2026.findings-emnlp.476/
%P 9987-10000
Markdown (Informal)
[Candidate Pool Design for Safer Preference-Based Post-Training of LLM Companion Chatbots](https://aclanthology.org/2026.findings-emnlp.476/) (Bai et al., Findings 2026)
ACL