@inproceedings{liu-etal-2026-past,
title = "The Past Is Not Past: Memory-Enhanced Dynamic Reward Shaping",
author = "Liu, Yang and
Wang, Enxi and
Gao, Yufei and
Zhang, Weixin and
Wang, Bo and
Zeng, Zhiyuan and
Zhang, Yikai and
Zheng, Yining and
Qiu, Xipeng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1975/",
pages = "44948--44971",
ISBN = "979-8-89176-459-0",
abstract = "Despite the remarkable progress of reinforcement learning for large language models, effective exploration remains a persistent challenge, as policies may collapse into recurrent failure modes and repeatedly generate similar erroneous behaviors across rollouts. While classical entropy regularization promotes randomness under the current policy, it provides limited mechanisms for recognizing and suppressing such recurrent failure patterns over time. To address this limitation, we propose MEDS, a Memory-Enhanced Dynamic reward Shaping framework that augments the reward signal with informative cues from historical behaviors. MEDS turns intermediate model representations into a lightweight behavioral memory that captures reasoning patterns from past rollouts, enabling dynamic reward shaping that steers the policy away from recurrent failure modes and toward alternative solution paths. Across five datasets and three base models, MEDS consistently improves average performance over existing baselines, achieving gains of up to 4.13 Pass@1 points and 4.37 Pass@128 points. LLM-based annotations and diversity metrics further confirm that MEDS promotes more diverse sampling behaviors. Our code is available at https://github.com/Linxi000/MEDS."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-past">
<titleInfo>
<title>The Past Is Not Past: Memory-Enhanced Dynamic Reward Shaping</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Enxi</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yufei</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weixin</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bo</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyuan</namePart>
<namePart type="family">Zeng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yikai</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yining</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xipeng</namePart>
<namePart type="family">Qiu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Despite the remarkable progress of reinforcement learning for large language models, effective exploration remains a persistent challenge, as policies may collapse into recurrent failure modes and repeatedly generate similar erroneous behaviors across rollouts. While classical entropy regularization promotes randomness under the current policy, it provides limited mechanisms for recognizing and suppressing such recurrent failure patterns over time. To address this limitation, we propose MEDS, a Memory-Enhanced Dynamic reward Shaping framework that augments the reward signal with informative cues from historical behaviors. MEDS turns intermediate model representations into a lightweight behavioral memory that captures reasoning patterns from past rollouts, enabling dynamic reward shaping that steers the policy away from recurrent failure modes and toward alternative solution paths. Across five datasets and three base models, MEDS consistently improves average performance over existing baselines, achieving gains of up to 4.13 Pass@1 points and 4.37 Pass@128 points. LLM-based annotations and diversity metrics further confirm that MEDS promotes more diverse sampling behaviors. Our code is available at https://github.com/Linxi000/MEDS.</abstract>
<identifier type="citekey">liu-etal-2026-past</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1975/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44948</start>
<end>44971</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Past Is Not Past: Memory-Enhanced Dynamic Reward Shaping
%A Liu, Yang
%A Wang, Enxi
%A Gao, Yufei
%A Zhang, Weixin
%A Wang, Bo
%A Zeng, Zhiyuan
%A Zhang, Yikai
%A Zheng, Yining
%A Qiu, Xipeng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liu-etal-2026-past
%X Despite the remarkable progress of reinforcement learning for large language models, effective exploration remains a persistent challenge, as policies may collapse into recurrent failure modes and repeatedly generate similar erroneous behaviors across rollouts. While classical entropy regularization promotes randomness under the current policy, it provides limited mechanisms for recognizing and suppressing such recurrent failure patterns over time. To address this limitation, we propose MEDS, a Memory-Enhanced Dynamic reward Shaping framework that augments the reward signal with informative cues from historical behaviors. MEDS turns intermediate model representations into a lightweight behavioral memory that captures reasoning patterns from past rollouts, enabling dynamic reward shaping that steers the policy away from recurrent failure modes and toward alternative solution paths. Across five datasets and three base models, MEDS consistently improves average performance over existing baselines, achieving gains of up to 4.13 Pass@1 points and 4.37 Pass@128 points. LLM-based annotations and diversity metrics further confirm that MEDS promotes more diverse sampling behaviors. Our code is available at https://github.com/Linxi000/MEDS.
%U https://aclanthology.org/2026.emnlp-main.1975/
%P 44948-44971
Markdown (Informal)
[The Past Is Not Past: Memory-Enhanced Dynamic Reward Shaping](https://aclanthology.org/2026.emnlp-main.1975/) (Liu et al., EMNLP 2026)
ACL
- Yang Liu, Enxi Wang, Yufei Gao, Weixin Zhang, Bo Wang, Zhiyuan Zeng, Yikai Zhang, Yining Zheng, and Xipeng Qiu. 2026. The Past Is Not Past: Memory-Enhanced Dynamic Reward Shaping. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 44948–44971, Budapest, Hungary. Association for Computational Linguistics.