@inproceedings{jin-etal-2026-exprl,
title = "{E}xp{RL}: Unified Non-Parametric and Parametric Experience Learning for {LLM} Agent Evolution",
author = "Jin, Zhuoran and
Wang, Lingshuai and
Hao, Yupu and
Men, Tianyi and
He, Zhitao and
Zeng, Daojian and
Liu, Kang and
Zhao, Jun",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1666/",
pages = "35524--35546",
ISBN = "979-8-89176-462-0",
abstract = "Large language model (LLM)-based agents solve complex long-horizon tasks through iterative interaction with environments, where effectively learning from experience under sparse or delayed feedback remains a key challenge. Existing approaches typically rely on either non-parametric memory-based experience reusing or parametric reinforcement learning, each with inherent limitations when used in isolation. To this end, we propose **ExpRL**, a unified experience learning framework that bridges non-parametric memory and parametric policy optimization for LLM agents. ExpRL induces abstracted success and failure experiences from on-policy interaction trajectories, stores them in an external memory, and integrates experience acquisition and utilization directly into the reinforcement learning process via an agentic experience acquisition mechanism. This design enables memory and policy to co-evolve and mutually reinforce each other, effectively unifying online policy learning with offline experience reuse. Experiments on ALFWorld and WebShop demonstrate that ExpRL achieves an average performance improvement of **13.0** over GRPO, with improved learning efficiency and stability. We further show that the induced experience memory generalizes across different models and can be directly transferred to stronger LLMs without additional training."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jin-etal-2026-exprl">
<titleInfo>
<title>ExpRL: Unified Non-Parametric and Parametric Experience Learning for LLM Agent Evolution</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhuoran</namePart>
<namePart type="family">Jin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lingshuai</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yupu</namePart>
<namePart type="family">Hao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianyi</namePart>
<namePart type="family">Men</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhitao</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daojian</namePart>
<namePart type="family">Zeng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language model (LLM)-based agents solve complex long-horizon tasks through iterative interaction with environments, where effectively learning from experience under sparse or delayed feedback remains a key challenge. Existing approaches typically rely on either non-parametric memory-based experience reusing or parametric reinforcement learning, each with inherent limitations when used in isolation. To this end, we propose **ExpRL**, a unified experience learning framework that bridges non-parametric memory and parametric policy optimization for LLM agents. ExpRL induces abstracted success and failure experiences from on-policy interaction trajectories, stores them in an external memory, and integrates experience acquisition and utilization directly into the reinforcement learning process via an agentic experience acquisition mechanism. This design enables memory and policy to co-evolve and mutually reinforce each other, effectively unifying online policy learning with offline experience reuse. Experiments on ALFWorld and WebShop demonstrate that ExpRL achieves an average performance improvement of **13.0** over GRPO, with improved learning efficiency and stability. We further show that the induced experience memory generalizes across different models and can be directly transferred to stronger LLMs without additional training.</abstract>
<identifier type="citekey">jin-etal-2026-exprl</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1666/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>35524</start>
<end>35546</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ExpRL: Unified Non-Parametric and Parametric Experience Learning for LLM Agent Evolution
%A Jin, Zhuoran
%A Wang, Lingshuai
%A Hao, Yupu
%A Men, Tianyi
%A He, Zhitao
%A Zeng, Daojian
%A Liu, Kang
%A Zhao, Jun
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F jin-etal-2026-exprl
%X Large language model (LLM)-based agents solve complex long-horizon tasks through iterative interaction with environments, where effectively learning from experience under sparse or delayed feedback remains a key challenge. Existing approaches typically rely on either non-parametric memory-based experience reusing or parametric reinforcement learning, each with inherent limitations when used in isolation. To this end, we propose **ExpRL**, a unified experience learning framework that bridges non-parametric memory and parametric policy optimization for LLM agents. ExpRL induces abstracted success and failure experiences from on-policy interaction trajectories, stores them in an external memory, and integrates experience acquisition and utilization directly into the reinforcement learning process via an agentic experience acquisition mechanism. This design enables memory and policy to co-evolve and mutually reinforce each other, effectively unifying online policy learning with offline experience reuse. Experiments on ALFWorld and WebShop demonstrate that ExpRL achieves an average performance improvement of **13.0** over GRPO, with improved learning efficiency and stability. We further show that the induced experience memory generalizes across different models and can be directly transferred to stronger LLMs without additional training.
%U https://aclanthology.org/2026.findings-emnlp.1666/
%P 35524-35546
Markdown (Informal)
[ExpRL: Unified Non-Parametric and Parametric Experience Learning for LLM Agent Evolution](https://aclanthology.org/2026.findings-emnlp.1666/) (Jin et al., Findings 2026)
ACL
- Zhuoran Jin, Lingshuai Wang, Yupu Hao, Tianyi Men, Zhitao He, Daojian Zeng, Kang Liu, and Jun Zhao. 2026. ExpRL: Unified Non-Parametric and Parametric Experience Learning for LLM Agent Evolution. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 35524–35546, Budapest, Hungary. Association for Computational Linguistics.