@inproceedings{zhu-etal-2026-gagpo,
title = "{GAGPO}: Generalized Advantage Grouped Policy Optimization",
author = "Zhu, Siyuan and
Yu, Chao and
Yang, Rongxin and
Liu, Zongkai and
Hu, Jinjun and
Chen, Qiwen and
Zhang, Yibo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.864/",
pages = "18261--18274",
ISBN = "979-8-89176-462-0",
abstract = "Reinforcement learning (RL) has emerged as a powerful paradigm for post-training large language model (LLM) agents. However, credit assignment in multi-turn environments remains a challenge. Agents typically receive sparse, trajectory-level rewards only at the end of an episode, making it difficult to identify which specific intermediate actions led to success or failure. Consequently, effectively propagating delayed outcomes back to individual steps without relying on costly auxiliary value models remains an open problem. In this paper, we propose Generalized Advantage Grouped Policy Optimization (GAGPO), a critic-free RL method that enables precise, step-aligned temporal credit assignment. GAGPO constructs a non-parametric grouped value proxy from sampled rollouts to compute TD/GAE-style temporal advantages, recursively propagating outcome supervision backward through time. Coupled with group-wise advantage normalization and an action-level importance ratio, GAGPO extracts stable and localized optimization signals directly from multi-turn trajectories. Experiments on ALFWorld and WebShop demonstrate that GAGPO outperforms strong RL baselines. Further analyses reveal faster early-stage learning, improved interaction efficiency, and smoother optimization dynamics, offering a simple yet highly effective framework for multi-turn agentic RL."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhu-etal-2026-gagpo">
<titleInfo>
<title>GAGPO: Generalized Advantage Grouped Policy Optimization</title>
</titleInfo>
<name type="personal">
<namePart type="given">Siyuan</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rongxin</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zongkai</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinjun</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qiwen</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yibo</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Reinforcement learning (RL) has emerged as a powerful paradigm for post-training large language model (LLM) agents. However, credit assignment in multi-turn environments remains a challenge. Agents typically receive sparse, trajectory-level rewards only at the end of an episode, making it difficult to identify which specific intermediate actions led to success or failure. Consequently, effectively propagating delayed outcomes back to individual steps without relying on costly auxiliary value models remains an open problem. In this paper, we propose Generalized Advantage Grouped Policy Optimization (GAGPO), a critic-free RL method that enables precise, step-aligned temporal credit assignment. GAGPO constructs a non-parametric grouped value proxy from sampled rollouts to compute TD/GAE-style temporal advantages, recursively propagating outcome supervision backward through time. Coupled with group-wise advantage normalization and an action-level importance ratio, GAGPO extracts stable and localized optimization signals directly from multi-turn trajectories. Experiments on ALFWorld and WebShop demonstrate that GAGPO outperforms strong RL baselines. Further analyses reveal faster early-stage learning, improved interaction efficiency, and smoother optimization dynamics, offering a simple yet highly effective framework for multi-turn agentic RL.</abstract>
<identifier type="citekey">zhu-etal-2026-gagpo</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.864/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>18261</start>
<end>18274</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T GAGPO: Generalized Advantage Grouped Policy Optimization
%A Zhu, Siyuan
%A Yu, Chao
%A Yang, Rongxin
%A Liu, Zongkai
%A Hu, Jinjun
%A Chen, Qiwen
%A Zhang, Yibo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhu-etal-2026-gagpo
%X Reinforcement learning (RL) has emerged as a powerful paradigm for post-training large language model (LLM) agents. However, credit assignment in multi-turn environments remains a challenge. Agents typically receive sparse, trajectory-level rewards only at the end of an episode, making it difficult to identify which specific intermediate actions led to success or failure. Consequently, effectively propagating delayed outcomes back to individual steps without relying on costly auxiliary value models remains an open problem. In this paper, we propose Generalized Advantage Grouped Policy Optimization (GAGPO), a critic-free RL method that enables precise, step-aligned temporal credit assignment. GAGPO constructs a non-parametric grouped value proxy from sampled rollouts to compute TD/GAE-style temporal advantages, recursively propagating outcome supervision backward through time. Coupled with group-wise advantage normalization and an action-level importance ratio, GAGPO extracts stable and localized optimization signals directly from multi-turn trajectories. Experiments on ALFWorld and WebShop demonstrate that GAGPO outperforms strong RL baselines. Further analyses reveal faster early-stage learning, improved interaction efficiency, and smoother optimization dynamics, offering a simple yet highly effective framework for multi-turn agentic RL.
%U https://aclanthology.org/2026.findings-emnlp.864/
%P 18261-18274
Markdown (Informal)
[GAGPO: Generalized Advantage Grouped Policy Optimization](https://aclanthology.org/2026.findings-emnlp.864/) (Zhu et al., Findings 2026)
ACL
- Siyuan Zhu, Chao Yu, Rongxin Yang, Zongkai Liu, Jinjun Hu, Qiwen Chen, and Yibo Zhang. 2026. GAGPO: Generalized Advantage Grouped Policy Optimization. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 18261–18274, Budapest, Hungary. Association for Computational Linguistics.