@inproceedings{dong-etal-2026-deppo,
title = "{DEPPO}: Dual Experience Pool Policy Optimization for Long-Horizon Reinforcement Learning",
author = "Dong, Haosen and
Guo, Jiaxin and
Yan, Muyang and
Yuan, Ye and
Jia, Mengxi and
Sun, Hao",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1333/",
pages = "30402--30418",
ISBN = "979-8-89176-459-0",
abstract = "Reinforcement learning for long-horizon agents and multi-step reasoning models typically relies on outcome-level rewards, which provide only trajectory-level supervision and make it difficult to identify the contribution of individual intermediate actions. As a result, redundant actions in successful trajectories may be over-reinforced, while useful actions in failed trajectories may be undesirably penalized. To address this issue, we propose **Dual Experience Pool Policy Optimization (DEPPO)**, a critic-free and experience-driven framework for adaptive step-level credit assignment. DEPPO maintains two structured experience pools for successful and failed trajectories, and estimates the relative quality of each action by comparing its state-conditional likelihood under the two pools. The resulting success{--}failure preference signal is used to adaptively rescale the advantage, amplifying actions aligned with successful patterns while suppressing those associated with failure modes. Furthermore, DEPPO introduces an asymmetric and dynamically evolving update mechanism: the success pool emphasizes both correctness and novelty through freshness-aware insertion, while the failure pool focuses on consistently risky behaviors, with decay and pruning to keep the memory compact and up-to-date. Without additional value models or process-level supervision, DEPPO provides a lightweight, interpretable, and effective solution for fine-grained credit assignment in long-horizon reinforcement learning. Code available at: https://anonymous.4open.science/r/DEPPO-1051/"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dong-etal-2026-deppo">
<titleInfo>
<title>DEPPO: Dual Experience Pool Policy Optimization for Long-Horizon Reinforcement Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Haosen</namePart>
<namePart type="family">Dong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiaxin</namePart>
<namePart type="family">Guo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Muyang</namePart>
<namePart type="family">Yan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ye</namePart>
<namePart type="family">Yuan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mengxi</namePart>
<namePart type="family">Jia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Reinforcement learning for long-horizon agents and multi-step reasoning models typically relies on outcome-level rewards, which provide only trajectory-level supervision and make it difficult to identify the contribution of individual intermediate actions. As a result, redundant actions in successful trajectories may be over-reinforced, while useful actions in failed trajectories may be undesirably penalized. To address this issue, we propose **Dual Experience Pool Policy Optimization (DEPPO)**, a critic-free and experience-driven framework for adaptive step-level credit assignment. DEPPO maintains two structured experience pools for successful and failed trajectories, and estimates the relative quality of each action by comparing its state-conditional likelihood under the two pools. The resulting success–failure preference signal is used to adaptively rescale the advantage, amplifying actions aligned with successful patterns while suppressing those associated with failure modes. Furthermore, DEPPO introduces an asymmetric and dynamically evolving update mechanism: the success pool emphasizes both correctness and novelty through freshness-aware insertion, while the failure pool focuses on consistently risky behaviors, with decay and pruning to keep the memory compact and up-to-date. Without additional value models or process-level supervision, DEPPO provides a lightweight, interpretable, and effective solution for fine-grained credit assignment in long-horizon reinforcement learning. Code available at: https://anonymous.4open.science/r/DEPPO-1051/</abstract>
<identifier type="citekey">dong-etal-2026-deppo</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1333/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>30402</start>
<end>30418</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DEPPO: Dual Experience Pool Policy Optimization for Long-Horizon Reinforcement Learning
%A Dong, Haosen
%A Guo, Jiaxin
%A Yan, Muyang
%A Yuan, Ye
%A Jia, Mengxi
%A Sun, Hao
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F dong-etal-2026-deppo
%X Reinforcement learning for long-horizon agents and multi-step reasoning models typically relies on outcome-level rewards, which provide only trajectory-level supervision and make it difficult to identify the contribution of individual intermediate actions. As a result, redundant actions in successful trajectories may be over-reinforced, while useful actions in failed trajectories may be undesirably penalized. To address this issue, we propose **Dual Experience Pool Policy Optimization (DEPPO)**, a critic-free and experience-driven framework for adaptive step-level credit assignment. DEPPO maintains two structured experience pools for successful and failed trajectories, and estimates the relative quality of each action by comparing its state-conditional likelihood under the two pools. The resulting success–failure preference signal is used to adaptively rescale the advantage, amplifying actions aligned with successful patterns while suppressing those associated with failure modes. Furthermore, DEPPO introduces an asymmetric and dynamically evolving update mechanism: the success pool emphasizes both correctness and novelty through freshness-aware insertion, while the failure pool focuses on consistently risky behaviors, with decay and pruning to keep the memory compact and up-to-date. Without additional value models or process-level supervision, DEPPO provides a lightweight, interpretable, and effective solution for fine-grained credit assignment in long-horizon reinforcement learning. Code available at: https://anonymous.4open.science/r/DEPPO-1051/
%U https://aclanthology.org/2026.emnlp-main.1333/
%P 30402-30418
Markdown (Informal)
[DEPPO: Dual Experience Pool Policy Optimization for Long-Horizon Reinforcement Learning](https://aclanthology.org/2026.emnlp-main.1333/) (Dong et al., EMNLP 2026)
ACL