@article{simoni-etal-2026-gtpo,
title = "{GTPO}: Stabilizing Group Relative Policy Optimization via Gradient and Entropy Control",
author = "Simoni, Marco and
Fontana, Aleksandar and
Rossolini, Giulio and
Saracino, Andrea and
Mori, Paolo",
journal = "Transactions of the Association for Computational Linguistics",
volume = "14",
year = "2026",
address = "Cambridge, MA",
publisher = "MIT Press",
url = "https://aclanthology.org/2026.tacl-1.102/",
doi = "10.1162/tacl.a.800",
pages = "2258--2280",
abstract = "Group Relative Policy Optimization (GRPO) is a promising policy-based approach for Large Language Model alignment, yet its performance is often limited by training instability and suboptimal convergence. In this paper, we identify and analyze two main GRPO issues: (i) the token-level penalization, where valuable tokens shared across different responses receive contradictory feedback signals, leading to conflicting gradient updates that can reduce their likelihood; and (ii) the policy collapse, where negatively rewarded completions may penalize confident responses and shift model decisions toward unlikely tokens, destabilizing training process. To address these issues we introduce GTPO (Group-relative Trajectory-based Policy Optimization), which prevents conflicting gradients on valuable tokens by skipping negative updates while amplifying positive ones and filters out completions whose entropy exceeds a provable threshold, to prevent policy collapse. By omitting KL-divergence regularization, GTPO eliminates the reference model while achieving superior stability and performance over GRPO and DAPO. Extensive evaluations across GSM8K, MATH, R1, AIME 2024{--}2025, AMC 2023, and MMLU validate these gains. The code is available here.1."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="simoni-etal-2026-gtpo">
<titleInfo>
<title>GTPO: Stabilizing Group Relative Policy Optimization via Gradient and Entropy Control</title>
</titleInfo>
<name type="personal">
<namePart type="given">Marco</namePart>
<namePart type="family">Simoni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aleksandar</namePart>
<namePart type="family">Fontana</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Giulio</namePart>
<namePart type="family">Rossolini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrea</namePart>
<namePart type="family">Saracino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paolo</namePart>
<namePart type="family">Mori</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<genre authority="bibutilsgt">journal article</genre>
<relatedItem type="host">
<titleInfo>
<title>Transactions of the Association for Computational Linguistics</title>
</titleInfo>
<originInfo>
<issuance>continuing</issuance>
<publisher>MIT Press</publisher>
<place>
<placeTerm type="text">Cambridge, MA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">periodical</genre>
<genre authority="bibutilsgt">academic journal</genre>
</relatedItem>
<abstract>Group Relative Policy Optimization (GRPO) is a promising policy-based approach for Large Language Model alignment, yet its performance is often limited by training instability and suboptimal convergence. In this paper, we identify and analyze two main GRPO issues: (i) the token-level penalization, where valuable tokens shared across different responses receive contradictory feedback signals, leading to conflicting gradient updates that can reduce their likelihood; and (ii) the policy collapse, where negatively rewarded completions may penalize confident responses and shift model decisions toward unlikely tokens, destabilizing training process. To address these issues we introduce GTPO (Group-relative Trajectory-based Policy Optimization), which prevents conflicting gradients on valuable tokens by skipping negative updates while amplifying positive ones and filters out completions whose entropy exceeds a provable threshold, to prevent policy collapse. By omitting KL-divergence regularization, GTPO eliminates the reference model while achieving superior stability and performance over GRPO and DAPO. Extensive evaluations across GSM8K, MATH, R1, AIME 2024–2025, AMC 2023, and MMLU validate these gains. The code is available here.1.</abstract>
<identifier type="citekey">simoni-etal-2026-gtpo</identifier>
<identifier type="doi">10.1162/tacl.a.800</identifier>
<location>
<url>https://aclanthology.org/2026.tacl-1.102/</url>
</location>
<part>
<date>2026</date>
<detail type="volume"><number>14</number></detail>
<extent unit="page">
<start>2258</start>
<end>2280</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Journal Article
%T GTPO: Stabilizing Group Relative Policy Optimization via Gradient and Entropy Control
%A Simoni, Marco
%A Fontana, Aleksandar
%A Rossolini, Giulio
%A Saracino, Andrea
%A Mori, Paolo
%J Transactions of the Association for Computational Linguistics
%D 2026
%V 14
%I MIT Press
%C Cambridge, MA
%F simoni-etal-2026-gtpo
%X Group Relative Policy Optimization (GRPO) is a promising policy-based approach for Large Language Model alignment, yet its performance is often limited by training instability and suboptimal convergence. In this paper, we identify and analyze two main GRPO issues: (i) the token-level penalization, where valuable tokens shared across different responses receive contradictory feedback signals, leading to conflicting gradient updates that can reduce their likelihood; and (ii) the policy collapse, where negatively rewarded completions may penalize confident responses and shift model decisions toward unlikely tokens, destabilizing training process. To address these issues we introduce GTPO (Group-relative Trajectory-based Policy Optimization), which prevents conflicting gradients on valuable tokens by skipping negative updates while amplifying positive ones and filters out completions whose entropy exceeds a provable threshold, to prevent policy collapse. By omitting KL-divergence regularization, GTPO eliminates the reference model while achieving superior stability and performance over GRPO and DAPO. Extensive evaluations across GSM8K, MATH, R1, AIME 2024–2025, AMC 2023, and MMLU validate these gains. The code is available here.1.
%R 10.1162/tacl.a.800
%U https://aclanthology.org/2026.tacl-1.102/
%U https://doi.org/10.1162/tacl.a.800
%P 2258-2280
Markdown (Informal)
[GTPO: Stabilizing Group Relative Policy Optimization via Gradient and Entropy Control](https://aclanthology.org/2026.tacl-1.102/) (Simoni et al., TACL 2026)
ACL