@inproceedings{tao-etal-2026-towards,
title = "Towards Faithful and Controllable Personalization via Critique-Post-Edit Reinforcement Learning",
author = "Tao, Meiling and
Zhu, Chenghao and
Ding, Dongyi and
Wang, Tiannan and
Jiang, Yuchen Eleanor and
Zhou, Wangchunshu",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1968/",
pages = "41965--41985",
ISBN = "979-8-89176-462-0",
abstract = "Reinforcement learning has driven the largest post-training gains on tasks with verifiable rewards, such as code and math. Extending this success to subjective tasks remains difficult: good responses must satisfy multiple soft criteria simultaneously, and no scalar reward can indicate which aspect of a response should improve. Personalization is a clear instance of this. A personalized response must balance helpfulness, user-specific relevance, and naturalness, yet standard verifiers cannot localize which specific dimension needs improvement. In this regime, supervised fine-tuning quickly saturates, and PPO with Bradley-Terry reward models is prone to reward hacking, especially via length bias, producing verbose or superficially personalized responses. We propose Critique-Post-Edit, an RL framework that combines (1) a personalized Generative Reward Model (GRM) producing multi-dimensional scores and natural-language critiques, and (2) a critique-guided editing step that turns feedback into nearby alternative responses for policy learning. The reward signal stays scalar; our contribution is to use personalized critiques to construct more informative local alternatives (responses close to the on-policy rollout in edit distance) during training. Under length-controlled evaluation, Critique-Post-Edit improves over PPO by 11.0 and 11.6 win-rate points on Qwen2.5-7B and 14B models, respectively. Notably, the 14B model matches or outperforms GPT-4.1 on the evaluated benchmarks. A static mechanism study further shows that critique-guided edits stay more local than extra rollouts and shift scores more favorably than generic rewrites. All of our experiments are on personalization; systematic evaluation on other subjective tasks is left to future work."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="tao-etal-2026-towards">
<titleInfo>
<title>Towards Faithful and Controllable Personalization via Critique-Post-Edit Reinforcement Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Meiling</namePart>
<namePart type="family">Tao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chenghao</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dongyi</namePart>
<namePart type="family">Ding</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tiannan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuchen</namePart>
<namePart type="given">Eleanor</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wangchunshu</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Reinforcement learning has driven the largest post-training gains on tasks with verifiable rewards, such as code and math. Extending this success to subjective tasks remains difficult: good responses must satisfy multiple soft criteria simultaneously, and no scalar reward can indicate which aspect of a response should improve. Personalization is a clear instance of this. A personalized response must balance helpfulness, user-specific relevance, and naturalness, yet standard verifiers cannot localize which specific dimension needs improvement. In this regime, supervised fine-tuning quickly saturates, and PPO with Bradley-Terry reward models is prone to reward hacking, especially via length bias, producing verbose or superficially personalized responses. We propose Critique-Post-Edit, an RL framework that combines (1) a personalized Generative Reward Model (GRM) producing multi-dimensional scores and natural-language critiques, and (2) a critique-guided editing step that turns feedback into nearby alternative responses for policy learning. The reward signal stays scalar; our contribution is to use personalized critiques to construct more informative local alternatives (responses close to the on-policy rollout in edit distance) during training. Under length-controlled evaluation, Critique-Post-Edit improves over PPO by 11.0 and 11.6 win-rate points on Qwen2.5-7B and 14B models, respectively. Notably, the 14B model matches or outperforms GPT-4.1 on the evaluated benchmarks. A static mechanism study further shows that critique-guided edits stay more local than extra rollouts and shift scores more favorably than generic rewrites. All of our experiments are on personalization; systematic evaluation on other subjective tasks is left to future work.</abstract>
<identifier type="citekey">tao-etal-2026-towards</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1968/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>41965</start>
<end>41985</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Towards Faithful and Controllable Personalization via Critique-Post-Edit Reinforcement Learning
%A Tao, Meiling
%A Zhu, Chenghao
%A Ding, Dongyi
%A Wang, Tiannan
%A Jiang, Yuchen Eleanor
%A Zhou, Wangchunshu
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F tao-etal-2026-towards
%X Reinforcement learning has driven the largest post-training gains on tasks with verifiable rewards, such as code and math. Extending this success to subjective tasks remains difficult: good responses must satisfy multiple soft criteria simultaneously, and no scalar reward can indicate which aspect of a response should improve. Personalization is a clear instance of this. A personalized response must balance helpfulness, user-specific relevance, and naturalness, yet standard verifiers cannot localize which specific dimension needs improvement. In this regime, supervised fine-tuning quickly saturates, and PPO with Bradley-Terry reward models is prone to reward hacking, especially via length bias, producing verbose or superficially personalized responses. We propose Critique-Post-Edit, an RL framework that combines (1) a personalized Generative Reward Model (GRM) producing multi-dimensional scores and natural-language critiques, and (2) a critique-guided editing step that turns feedback into nearby alternative responses for policy learning. The reward signal stays scalar; our contribution is to use personalized critiques to construct more informative local alternatives (responses close to the on-policy rollout in edit distance) during training. Under length-controlled evaluation, Critique-Post-Edit improves over PPO by 11.0 and 11.6 win-rate points on Qwen2.5-7B and 14B models, respectively. Notably, the 14B model matches or outperforms GPT-4.1 on the evaluated benchmarks. A static mechanism study further shows that critique-guided edits stay more local than extra rollouts and shift scores more favorably than generic rewrites. All of our experiments are on personalization; systematic evaluation on other subjective tasks is left to future work.
%U https://aclanthology.org/2026.findings-emnlp.1968/
%P 41965-41985
Markdown (Informal)
[Towards Faithful and Controllable Personalization via Critique-Post-Edit Reinforcement Learning](https://aclanthology.org/2026.findings-emnlp.1968/) (Tao et al., Findings 2026)
ACL