@inproceedings{qian-etal-2026-userrl,
title = "{U}ser{RL}: A Gym-Based Testbed for User-Centric Reinforcement Learning",
author = "Qian, Cheng and
Liu, Zuxin and
Prabhakar, Akshara and
Qiu, Jielin and
Liu, Zhiwei and
Chen, Haolin and
Kokane, Shirley and
Ji, Heng and
Yao, Weiran and
Savarese, Silvio and
Xiong, Caiming and
Wang, Huan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.780/",
pages = "18133--18163",
ISBN = "979-8-89176-459-0",
abstract = "Reinforcement learning (RL) has shown promise for training agentic models that engage in dynamic, multi-turn interactions. However, the value of such agents ultimately depends on their ability to assist users with diverse goals, preferences, and evolving interaction patterns. We introduce **UserRL**, a reproducible and extensible testbed for training and evaluating user-centric agents through standardized gym environments paired with simulated users. UserRL supports flexible reward specification by decoupling turn-level reward shaping from trajectory-level scoring, enabling controlled studies of credit assignment in multi-turn RL. Across Qwen3 models, our experiments show that SFT cold start provides an effective initialization for user-centric RL; trajectory-level reward-to-go scoring improves both effectiveness and interaction efficiency; and stronger simulated users improve training, while open-source simulators remain a cost-effective alternative with positive cross-simulator and preliminary real-user transfer. These results highlight reward design and user simulation as key to user-centric agent training, positioning UserRL as a practical foundation for interactive agent RL."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="qian-etal-2026-userrl">
<titleInfo>
<title>UserRL: A Gym-Based Testbed for User-Centric Reinforcement Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Cheng</namePart>
<namePart type="family">Qian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zuxin</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akshara</namePart>
<namePart type="family">Prabhakar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jielin</namePart>
<namePart type="family">Qiu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiwei</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haolin</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shirley</namePart>
<namePart type="family">Kokane</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Heng</namePart>
<namePart type="family">Ji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weiran</namePart>
<namePart type="family">Yao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Silvio</namePart>
<namePart type="family">Savarese</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Caiming</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Huan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Reinforcement learning (RL) has shown promise for training agentic models that engage in dynamic, multi-turn interactions. However, the value of such agents ultimately depends on their ability to assist users with diverse goals, preferences, and evolving interaction patterns. We introduce **UserRL**, a reproducible and extensible testbed for training and evaluating user-centric agents through standardized gym environments paired with simulated users. UserRL supports flexible reward specification by decoupling turn-level reward shaping from trajectory-level scoring, enabling controlled studies of credit assignment in multi-turn RL. Across Qwen3 models, our experiments show that SFT cold start provides an effective initialization for user-centric RL; trajectory-level reward-to-go scoring improves both effectiveness and interaction efficiency; and stronger simulated users improve training, while open-source simulators remain a cost-effective alternative with positive cross-simulator and preliminary real-user transfer. These results highlight reward design and user simulation as key to user-centric agent training, positioning UserRL as a practical foundation for interactive agent RL.</abstract>
<identifier type="citekey">qian-etal-2026-userrl</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.780/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>18133</start>
<end>18163</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T UserRL: A Gym-Based Testbed for User-Centric Reinforcement Learning
%A Qian, Cheng
%A Liu, Zuxin
%A Prabhakar, Akshara
%A Qiu, Jielin
%A Liu, Zhiwei
%A Chen, Haolin
%A Kokane, Shirley
%A Ji, Heng
%A Yao, Weiran
%A Savarese, Silvio
%A Xiong, Caiming
%A Wang, Huan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F qian-etal-2026-userrl
%X Reinforcement learning (RL) has shown promise for training agentic models that engage in dynamic, multi-turn interactions. However, the value of such agents ultimately depends on their ability to assist users with diverse goals, preferences, and evolving interaction patterns. We introduce **UserRL**, a reproducible and extensible testbed for training and evaluating user-centric agents through standardized gym environments paired with simulated users. UserRL supports flexible reward specification by decoupling turn-level reward shaping from trajectory-level scoring, enabling controlled studies of credit assignment in multi-turn RL. Across Qwen3 models, our experiments show that SFT cold start provides an effective initialization for user-centric RL; trajectory-level reward-to-go scoring improves both effectiveness and interaction efficiency; and stronger simulated users improve training, while open-source simulators remain a cost-effective alternative with positive cross-simulator and preliminary real-user transfer. These results highlight reward design and user simulation as key to user-centric agent training, positioning UserRL as a practical foundation for interactive agent RL.
%U https://aclanthology.org/2026.emnlp-main.780/
%P 18133-18163
Markdown (Informal)
[UserRL: A Gym-Based Testbed for User-Centric Reinforcement Learning](https://aclanthology.org/2026.emnlp-main.780/) (Qian et al., EMNLP 2026)
ACL
- Cheng Qian, Zuxin Liu, Akshara Prabhakar, Jielin Qiu, Zhiwei Liu, Haolin Chen, Shirley Kokane, Heng Ji, Weiran Yao, Silvio Savarese, Caiming Xiong, and Huan Wang. 2026. UserRL: A Gym-Based Testbed for User-Centric Reinforcement Learning. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 18133–18163, Budapest, Hungary. Association for Computational Linguistics.