@inproceedings{zhang-li-2026-gsspo,
title = "{GSSPO}: Group Sentence-Specified Sequence Policy Optimization for {LLM} Personality Alignment",
author = "Zhang, Shurui and
Li, Zikai",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.773/",
doi = "10.18653/2026.findings-emnlp.773",
pages = "16331--16362",
ISBN = "979-8-89176-462-0",
abstract = "Large Language Models (LLMs) have demonstrated strong reasoning capabilities, yet achieving nuanced personality alignment for simulating human conversation remains challenging. This difficulty stems from two bottlenecks: the scarcity of scalable, privacy-respecting training data and the limited ability of conventional reinforcement learning (RL) rewards to capture subtle personality-relevant signals. In this work, we introduce a unified framework to address these issues. First, we propose the Personality Alignment Choice Forge (PACF), an automatic pipeline that transforms existing conversational datasets into structured, multiple-choice benchmarks, constraining the model{'}s action space and enabling scalable evaluation grounded in observed human choices. Second, we present Group Sentence-Specified Sequence Policy Optimization (GSSPO), an RL algorithm designed to address the coarse credit assignment problem in methods that apply a uniform reward across all tokens. By integrating stable sequence-level updates with differentiated, rubric-guided token-level feedback, GSSPO assigns reward signals to more localized spans of the response. This targeted approach mitigates optimization instabilities caused by undifferentiated reward signals, leading to more effective training in our experiments. On the Qwen3 model family, GSSPO achieves the best personality-alignment accuracy on every benchmark and scale in the SFT+RL regime, and leads the other RL methods on five of six in the SFT-Free regime. These results suggest a scalable route toward training LLMs that better match measured personality-alignment preferences. Our core code is available at \url{https://github.com/HKUST-FintechLab/GSSPO}"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-li-2026-gsspo">
<titleInfo>
<title>GSSPO: Group Sentence-Specified Sequence Policy Optimization for LLM Personality Alignment</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shurui</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zikai</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) have demonstrated strong reasoning capabilities, yet achieving nuanced personality alignment for simulating human conversation remains challenging. This difficulty stems from two bottlenecks: the scarcity of scalable, privacy-respecting training data and the limited ability of conventional reinforcement learning (RL) rewards to capture subtle personality-relevant signals. In this work, we introduce a unified framework to address these issues. First, we propose the Personality Alignment Choice Forge (PACF), an automatic pipeline that transforms existing conversational datasets into structured, multiple-choice benchmarks, constraining the model’s action space and enabling scalable evaluation grounded in observed human choices. Second, we present Group Sentence-Specified Sequence Policy Optimization (GSSPO), an RL algorithm designed to address the coarse credit assignment problem in methods that apply a uniform reward across all tokens. By integrating stable sequence-level updates with differentiated, rubric-guided token-level feedback, GSSPO assigns reward signals to more localized spans of the response. This targeted approach mitigates optimization instabilities caused by undifferentiated reward signals, leading to more effective training in our experiments. On the Qwen3 model family, GSSPO achieves the best personality-alignment accuracy on every benchmark and scale in the SFT+RL regime, and leads the other RL methods on five of six in the SFT-Free regime. These results suggest a scalable route toward training LLMs that better match measured personality-alignment preferences. Our core code is available at https://github.com/HKUST-FintechLab/GSSPO</abstract>
<identifier type="citekey">zhang-li-2026-gsspo</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.773</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.773/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>16331</start>
<end>16362</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T GSSPO: Group Sentence-Specified Sequence Policy Optimization for LLM Personality Alignment
%A Zhang, Shurui
%A Li, Zikai
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhang-li-2026-gsspo
%X Large Language Models (LLMs) have demonstrated strong reasoning capabilities, yet achieving nuanced personality alignment for simulating human conversation remains challenging. This difficulty stems from two bottlenecks: the scarcity of scalable, privacy-respecting training data and the limited ability of conventional reinforcement learning (RL) rewards to capture subtle personality-relevant signals. In this work, we introduce a unified framework to address these issues. First, we propose the Personality Alignment Choice Forge (PACF), an automatic pipeline that transforms existing conversational datasets into structured, multiple-choice benchmarks, constraining the model’s action space and enabling scalable evaluation grounded in observed human choices. Second, we present Group Sentence-Specified Sequence Policy Optimization (GSSPO), an RL algorithm designed to address the coarse credit assignment problem in methods that apply a uniform reward across all tokens. By integrating stable sequence-level updates with differentiated, rubric-guided token-level feedback, GSSPO assigns reward signals to more localized spans of the response. This targeted approach mitigates optimization instabilities caused by undifferentiated reward signals, leading to more effective training in our experiments. On the Qwen3 model family, GSSPO achieves the best personality-alignment accuracy on every benchmark and scale in the SFT+RL regime, and leads the other RL methods on five of six in the SFT-Free regime. These results suggest a scalable route toward training LLMs that better match measured personality-alignment preferences. Our core code is available at https://github.com/HKUST-FintechLab/GSSPO
%R 10.18653/2026.findings-emnlp.773
%U https://aclanthology.org/2026.findings-emnlp.773/
%U https://doi.org/10.18653/2026.findings-emnlp.773
%P 16331-16362
Markdown (Informal)
[GSSPO: Group Sentence-Specified Sequence Policy Optimization for LLM Personality Alignment](https://aclanthology.org/2026.findings-emnlp.773/) (Zhang & Li, Findings 2026)
ACL