@inproceedings{hong-etal-2026-r,
title = "{R}-{IDEAL}: Reinforcing Insight-Driven Experience for Agent Learning",
author = "Hong, Wentao and
Zhou, Xin and
Xu, Yukai and
Yu, Jianing and
Yang, Sheng and
Gao, Sheng and
Xu, Weiran",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1658/",
pages = "37694--37708",
ISBN = "979-8-89176-459-0",
abstract = "Learning from experience is essential for agents in real-world applications. However, most existing methods rely on static imitation of successful expert trajectories, overlooking the informative signals exposed from failed experiences and lacking a interactive feedback loop with the environment. In this paper, we propose R-IDEAL, a unified framework for insight-driven experience generation and optimization. R-IDEAL first perform contrastive analysis between successful and failed trajectories to automatically synthesize structured insights, consisting of positive plans and negative lessons, which guide agents not only on what to do, but also on what to avoid. These synthesized insights serve as supervision to train a lightweight insight model that generates task-specific guidance. While this supervision stabilizes insight generation, it does not guarantee the effectiveness of the insights due to distribution shifts and execution discrepancies across agents. Therefore, we further optimize the insight model via reinforcement learning using execution feedback from heterogeneous agents in real environments, ensuring that the generated insights effectively improve agent behavior. Experiments on AlfWorld and SciWorld show that R-IDEAL substantially outperforms existing methods, demonstrating strong robustness in out-of-distribution scenarios and serving as a lightweight, plug-and-play module for both closed-source and large-scale models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hong-etal-2026-r">
<titleInfo>
<title>R-IDEAL: Reinforcing Insight-Driven Experience for Agent Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wentao</namePart>
<namePart type="family">Hong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yukai</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianing</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sheng</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sheng</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weiran</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Learning from experience is essential for agents in real-world applications. However, most existing methods rely on static imitation of successful expert trajectories, overlooking the informative signals exposed from failed experiences and lacking a interactive feedback loop with the environment. In this paper, we propose R-IDEAL, a unified framework for insight-driven experience generation and optimization. R-IDEAL first perform contrastive analysis between successful and failed trajectories to automatically synthesize structured insights, consisting of positive plans and negative lessons, which guide agents not only on what to do, but also on what to avoid. These synthesized insights serve as supervision to train a lightweight insight model that generates task-specific guidance. While this supervision stabilizes insight generation, it does not guarantee the effectiveness of the insights due to distribution shifts and execution discrepancies across agents. Therefore, we further optimize the insight model via reinforcement learning using execution feedback from heterogeneous agents in real environments, ensuring that the generated insights effectively improve agent behavior. Experiments on AlfWorld and SciWorld show that R-IDEAL substantially outperforms existing methods, demonstrating strong robustness in out-of-distribution scenarios and serving as a lightweight, plug-and-play module for both closed-source and large-scale models.</abstract>
<identifier type="citekey">hong-etal-2026-r</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1658/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>37694</start>
<end>37708</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T R-IDEAL: Reinforcing Insight-Driven Experience for Agent Learning
%A Hong, Wentao
%A Zhou, Xin
%A Xu, Yukai
%A Yu, Jianing
%A Yang, Sheng
%A Gao, Sheng
%A Xu, Weiran
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F hong-etal-2026-r
%X Learning from experience is essential for agents in real-world applications. However, most existing methods rely on static imitation of successful expert trajectories, overlooking the informative signals exposed from failed experiences and lacking a interactive feedback loop with the environment. In this paper, we propose R-IDEAL, a unified framework for insight-driven experience generation and optimization. R-IDEAL first perform contrastive analysis between successful and failed trajectories to automatically synthesize structured insights, consisting of positive plans and negative lessons, which guide agents not only on what to do, but also on what to avoid. These synthesized insights serve as supervision to train a lightweight insight model that generates task-specific guidance. While this supervision stabilizes insight generation, it does not guarantee the effectiveness of the insights due to distribution shifts and execution discrepancies across agents. Therefore, we further optimize the insight model via reinforcement learning using execution feedback from heterogeneous agents in real environments, ensuring that the generated insights effectively improve agent behavior. Experiments on AlfWorld and SciWorld show that R-IDEAL substantially outperforms existing methods, demonstrating strong robustness in out-of-distribution scenarios and serving as a lightweight, plug-and-play module for both closed-source and large-scale models.
%U https://aclanthology.org/2026.emnlp-main.1658/
%P 37694-37708
Markdown (Informal)
[R-IDEAL: Reinforcing Insight-Driven Experience for Agent Learning](https://aclanthology.org/2026.emnlp-main.1658/) (Hong et al., EMNLP 2026)
ACL
- Wentao Hong, Xin Zhou, Yukai Xu, Jianing Yu, Sheng Yang, Sheng Gao, and Weiran Xu. 2026. R-IDEAL: Reinforcing Insight-Driven Experience for Agent Learning. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 37694–37708, Budapest, Hungary. Association for Computational Linguistics.