@inproceedings{yu-etal-2026-one,
title = "One Frozen Simulator Is Not Enough: Simulator Collapse in Multi-Agent {RL}",
author = "Yu, Simon and
Tomlin, Nicholas and
Abdulhai, Marwa and
Chong, Derek and
Lu, Ximing and
Hou, Abe Bohan and
Soylu, Dilara and
Levine, Sergey and
Manning, Christopher D and
Shi, Weiyan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.790/",
pages = "18336--18373",
ISBN = "979-8-89176-459-0",
abstract = "Multi-agent reinforcement learning for human-AI interaction typically relies on a single large language model to simulate user behavior. We show that this approach systematically fails to generalize, and trace the failure to **simulator collapse**: because the simulator LLM is mode-collapsed, a LLM policy trained against it overfits to narrow strategies that exploit the simulator{'}s mode, and such a policy transfers poorly to unseen simulators and real users. We formalize this collapse theoretically and propose two solutions. The inference-time solution, Verbalized Sampling, broadens the simulator{'}s behavior by sampling from a verbalized response distribution, reducing mode collapse. The training-time solution, Co-Training, jointly optimizes the policy against a population of trainable simulators, preventing it from overfitting to any single simulator{'}s mode. We validate both solutions on three multi-turn benchmarks: Persuasion for Good, $\tau^2$-bench, and CooperBench. Verbalized Sampling improves held-out success by up to 9{\%} over single-simulator RL, and Co-Training pushes gains further to 14{\%}; the human study shows similar gain on real users in both. Both solutions preserve the policy diversity that collapses under single-simulator RL. Broadly, our results {\%}challenge the common practice of a frozen simulator and suggest that the diversity of the \textit{training environment} is critical to the generalization of multi-agent RL to real users."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yu-etal-2026-one">
<titleInfo>
<title>One Frozen Simulator Is Not Enough: Simulator Collapse in Multi-Agent RL</title>
</titleInfo>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicholas</namePart>
<namePart type="family">Tomlin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marwa</namePart>
<namePart type="family">Abdulhai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Derek</namePart>
<namePart type="family">Chong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ximing</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abe</namePart>
<namePart type="given">Bohan</namePart>
<namePart type="family">Hou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dilara</namePart>
<namePart type="family">Soylu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sergey</namePart>
<namePart type="family">Levine</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christopher</namePart>
<namePart type="given">D</namePart>
<namePart type="family">Manning</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weiyan</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Multi-agent reinforcement learning for human-AI interaction typically relies on a single large language model to simulate user behavior. We show that this approach systematically fails to generalize, and trace the failure to **simulator collapse**: because the simulator LLM is mode-collapsed, a LLM policy trained against it overfits to narrow strategies that exploit the simulator’s mode, and such a policy transfers poorly to unseen simulators and real users. We formalize this collapse theoretically and propose two solutions. The inference-time solution, Verbalized Sampling, broadens the simulator’s behavior by sampling from a verbalized response distribution, reducing mode collapse. The training-time solution, Co-Training, jointly optimizes the policy against a population of trainable simulators, preventing it from overfitting to any single simulator’s mode. We validate both solutions on three multi-turn benchmarks: Persuasion for Good, τ²-bench, and CooperBench. Verbalized Sampling improves held-out success by up to 9% over single-simulator RL, and Co-Training pushes gains further to 14%; the human study shows similar gain on real users in both. Both solutions preserve the policy diversity that collapses under single-simulator RL. Broadly, our results %challenge the common practice of a frozen simulator and suggest that the diversity of the training environment is critical to the generalization of multi-agent RL to real users.</abstract>
<identifier type="citekey">yu-etal-2026-one</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.790/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>18336</start>
<end>18373</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T One Frozen Simulator Is Not Enough: Simulator Collapse in Multi-Agent RL
%A Yu, Simon
%A Tomlin, Nicholas
%A Abdulhai, Marwa
%A Chong, Derek
%A Lu, Ximing
%A Hou, Abe Bohan
%A Soylu, Dilara
%A Levine, Sergey
%A Manning, Christopher D.
%A Shi, Weiyan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yu-etal-2026-one
%X Multi-agent reinforcement learning for human-AI interaction typically relies on a single large language model to simulate user behavior. We show that this approach systematically fails to generalize, and trace the failure to **simulator collapse**: because the simulator LLM is mode-collapsed, a LLM policy trained against it overfits to narrow strategies that exploit the simulator’s mode, and such a policy transfers poorly to unseen simulators and real users. We formalize this collapse theoretically and propose two solutions. The inference-time solution, Verbalized Sampling, broadens the simulator’s behavior by sampling from a verbalized response distribution, reducing mode collapse. The training-time solution, Co-Training, jointly optimizes the policy against a population of trainable simulators, preventing it from overfitting to any single simulator’s mode. We validate both solutions on three multi-turn benchmarks: Persuasion for Good, τ²-bench, and CooperBench. Verbalized Sampling improves held-out success by up to 9% over single-simulator RL, and Co-Training pushes gains further to 14%; the human study shows similar gain on real users in both. Both solutions preserve the policy diversity that collapses under single-simulator RL. Broadly, our results %challenge the common practice of a frozen simulator and suggest that the diversity of the training environment is critical to the generalization of multi-agent RL to real users.
%U https://aclanthology.org/2026.emnlp-main.790/
%P 18336-18373
Markdown (Informal)
[One Frozen Simulator Is Not Enough: Simulator Collapse in Multi-Agent RL](https://aclanthology.org/2026.emnlp-main.790/) (Yu et al., EMNLP 2026)
ACL
- Simon Yu, Nicholas Tomlin, Marwa Abdulhai, Derek Chong, Ximing Lu, Abe Bohan Hou, Dilara Soylu, Sergey Levine, Christopher D Manning, and Weiyan Shi. 2026. One Frozen Simulator Is Not Enough: Simulator Collapse in Multi-Agent RL. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 18336–18373, Budapest, Hungary. Association for Computational Linguistics.