@inproceedings{liu-etal-2026-trace,
title = "{TRACE}-{GRPO}: Teacher-Refined Advantage Credit Estimation for Long-Horizon Agentic {RL}",
author = "Liu, Yuchen and
Yao, Tianchu and
Chen, Deshu and
Zhang, Yinghao and
Li, Xigui and
Ling, Zehui and
Guo, Xin and
Cheng, Yuan and
Qu, Chao and
Ma, Lintao and
Qi, Yuan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1522/",
pages = "34746--34765",
ISBN = "979-8-89176-459-0",
abstract = "Group-Relative Policy Optimization (GRPO) is widely used to post-train LLM agents, but its trajectory-level advantage is ill-suited to long-horizon credit assignment. Reward-homogeneous rollout groups yield zero advantage; otherwise, one scalar uniformly scales every token, obscuring which decisions determine the outcome. We propose **TRACE-GRPO** (**T**eacher-**R**efined **A**dvantage **C**redit **E**stimation for GRPO), which converts teacher-LLM feedback into hierarchical credit. Teacher scores provide a sample-level fallback for zero-variance groups, reweight trajectory advantage at the turn level, and redistribute turn credit at the token level using the policy{'}s log-probability shift under a teacher critique. This yields dense training-time credit without token-level annotations or learned critics. We also introduce **CAPA** (Critique-Aware Paged Attention), which reuses trajectory KV caches and reduces the isolated critique-conditioned forward from roughly $2\times$ a vanilla GRPO step to approximately 5{\%} of a training step. On ALFWorld and ScienceWorld with Qwen2.5-7B-Instruct, TRACE-GRPO achieves 96.3{\%} and 73.4{\%} success, respectively, outperforming strong agent RL baselines. Ablations show cumulative gains from the three credit levels, and comparison with supervised fine-tuning on teacher-generated successes confirms that critique-guided policy optimization provides benefits beyond imitation. Code is available at [https://github.com/Lau-Jonathan/TRACE-GRPO](https://github.com/Lau-Jonathan/TRACE-GRPO)."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-trace">
<titleInfo>
<title>TRACE-GRPO: Teacher-Refined Advantage Credit Estimation for Long-Horizon Agentic RL</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuchen</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianchu</namePart>
<namePart type="family">Yao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Deshu</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yinghao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xigui</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zehui</namePart>
<namePart type="family">Ling</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Guo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuan</namePart>
<namePart type="family">Cheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao</namePart>
<namePart type="family">Qu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lintao</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuan</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Group-Relative Policy Optimization (GRPO) is widely used to post-train LLM agents, but its trajectory-level advantage is ill-suited to long-horizon credit assignment. Reward-homogeneous rollout groups yield zero advantage; otherwise, one scalar uniformly scales every token, obscuring which decisions determine the outcome. We propose **TRACE-GRPO** (**T**eacher-**R**efined **A**dvantage **C**redit **E**stimation for GRPO), which converts teacher-LLM feedback into hierarchical credit. Teacher scores provide a sample-level fallback for zero-variance groups, reweight trajectory advantage at the turn level, and redistribute turn credit at the token level using the policy’s log-probability shift under a teacher critique. This yields dense training-time credit without token-level annotations or learned critics. We also introduce **CAPA** (Critique-Aware Paged Attention), which reuses trajectory KV caches and reduces the isolated critique-conditioned forward from roughly 2\times a vanilla GRPO step to approximately 5% of a training step. On ALFWorld and ScienceWorld with Qwen2.5-7B-Instruct, TRACE-GRPO achieves 96.3% and 73.4% success, respectively, outperforming strong agent RL baselines. Ablations show cumulative gains from the three credit levels, and comparison with supervised fine-tuning on teacher-generated successes confirms that critique-guided policy optimization provides benefits beyond imitation. Code is available at [https://github.com/Lau-Jonathan/TRACE-GRPO](https://github.com/Lau-Jonathan/TRACE-GRPO).</abstract>
<identifier type="citekey">liu-etal-2026-trace</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1522/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>34746</start>
<end>34765</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T TRACE-GRPO: Teacher-Refined Advantage Credit Estimation for Long-Horizon Agentic RL
%A Liu, Yuchen
%A Yao, Tianchu
%A Chen, Deshu
%A Zhang, Yinghao
%A Li, Xigui
%A Ling, Zehui
%A Guo, Xin
%A Cheng, Yuan
%A Qu, Chao
%A Ma, Lintao
%A Qi, Yuan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liu-etal-2026-trace
%X Group-Relative Policy Optimization (GRPO) is widely used to post-train LLM agents, but its trajectory-level advantage is ill-suited to long-horizon credit assignment. Reward-homogeneous rollout groups yield zero advantage; otherwise, one scalar uniformly scales every token, obscuring which decisions determine the outcome. We propose **TRACE-GRPO** (**T**eacher-**R**efined **A**dvantage **C**redit **E**stimation for GRPO), which converts teacher-LLM feedback into hierarchical credit. Teacher scores provide a sample-level fallback for zero-variance groups, reweight trajectory advantage at the turn level, and redistribute turn credit at the token level using the policy’s log-probability shift under a teacher critique. This yields dense training-time credit without token-level annotations or learned critics. We also introduce **CAPA** (Critique-Aware Paged Attention), which reuses trajectory KV caches and reduces the isolated critique-conditioned forward from roughly 2\times a vanilla GRPO step to approximately 5% of a training step. On ALFWorld and ScienceWorld with Qwen2.5-7B-Instruct, TRACE-GRPO achieves 96.3% and 73.4% success, respectively, outperforming strong agent RL baselines. Ablations show cumulative gains from the three credit levels, and comparison with supervised fine-tuning on teacher-generated successes confirms that critique-guided policy optimization provides benefits beyond imitation. Code is available at [https://github.com/Lau-Jonathan/TRACE-GRPO](https://github.com/Lau-Jonathan/TRACE-GRPO).
%U https://aclanthology.org/2026.emnlp-main.1522/
%P 34746-34765
Markdown (Informal)
[TRACE-GRPO: Teacher-Refined Advantage Credit Estimation for Long-Horizon Agentic RL](https://aclanthology.org/2026.emnlp-main.1522/) (Liu et al., EMNLP 2026)
ACL
- Yuchen Liu, Tianchu Yao, Deshu Chen, Yinghao Zhang, Xigui Li, Zehui Ling, Xin Guo, Yuan Cheng, Chao Qu, Lintao Ma, and Yuan Qi. 2026. TRACE-GRPO: Teacher-Refined Advantage Credit Estimation for Long-Horizon Agentic RL. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 34746–34765, Budapest, Hungary. Association for Computational Linguistics.