@inproceedings{cao-etal-2026-mobiledreamer,
title = "{M}obile{D}reamer: Generative Sketch World Model for {GUI} Agent",
author = "Cao, Yilin and
Zhong, Yufeng and
Zeng, Zhixiong and
Zheng, Liming and
Huang, Jing and
Qiu, Haibo and
Shi, Peng and
Mao, Wenji",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1337/",
pages = "28345--28359",
ISBN = "979-8-89176-462-0",
abstract = "Mobile GUI agents have shown strong potential in real-world automation and practical applications. However, most existing agents remain reactive, making decisions mainly from the current screen, which limits their performance on long-horizon tasks. To address this limitation, a natural approach is to use the forecasting ability of world models to support better decision-making. This is challenging because existing world models often lack spatial awareness when predicting post-action states and are too expensive for practical deployment. In this paper, we propose MobileDreamer, an efficient world-model-based lookahead framework to equip the GUI agents based on the future imagination provided by the world model. It consists of textual sketch world model and rollout imagination for GUI agent. Textual sketch world model forecasts post-action states through a learning process to transform digital images into key task-related sketches, and designs a novel order-invariant learning strategy to preserve the spatial information of GUI elements. The rollout imagination strategy for the GUI agent optimizes the action-selection process by leveraging the prediction capability of world model. Empirical results on Android World demonstrate that MobileDreamer consistently improves the task success rate across multiple base models, achieving an average performance gain of 5.3{\%}. World model evaluations further verify that our textual sketch modeling accurately forecasts key GUI elements."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="cao-etal-2026-mobiledreamer">
<titleInfo>
<title>MobileDreamer: Generative Sketch World Model for GUI Agent</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yilin</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yufeng</namePart>
<namePart type="family">Zhong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhixiong</namePart>
<namePart type="family">Zeng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liming</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jing</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haibo</namePart>
<namePart type="family">Qiu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peng</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenji</namePart>
<namePart type="family">Mao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Mobile GUI agents have shown strong potential in real-world automation and practical applications. However, most existing agents remain reactive, making decisions mainly from the current screen, which limits their performance on long-horizon tasks. To address this limitation, a natural approach is to use the forecasting ability of world models to support better decision-making. This is challenging because existing world models often lack spatial awareness when predicting post-action states and are too expensive for practical deployment. In this paper, we propose MobileDreamer, an efficient world-model-based lookahead framework to equip the GUI agents based on the future imagination provided by the world model. It consists of textual sketch world model and rollout imagination for GUI agent. Textual sketch world model forecasts post-action states through a learning process to transform digital images into key task-related sketches, and designs a novel order-invariant learning strategy to preserve the spatial information of GUI elements. The rollout imagination strategy for the GUI agent optimizes the action-selection process by leveraging the prediction capability of world model. Empirical results on Android World demonstrate that MobileDreamer consistently improves the task success rate across multiple base models, achieving an average performance gain of 5.3%. World model evaluations further verify that our textual sketch modeling accurately forecasts key GUI elements.</abstract>
<identifier type="citekey">cao-etal-2026-mobiledreamer</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1337/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>28345</start>
<end>28359</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MobileDreamer: Generative Sketch World Model for GUI Agent
%A Cao, Yilin
%A Zhong, Yufeng
%A Zeng, Zhixiong
%A Zheng, Liming
%A Huang, Jing
%A Qiu, Haibo
%A Shi, Peng
%A Mao, Wenji
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F cao-etal-2026-mobiledreamer
%X Mobile GUI agents have shown strong potential in real-world automation and practical applications. However, most existing agents remain reactive, making decisions mainly from the current screen, which limits their performance on long-horizon tasks. To address this limitation, a natural approach is to use the forecasting ability of world models to support better decision-making. This is challenging because existing world models often lack spatial awareness when predicting post-action states and are too expensive for practical deployment. In this paper, we propose MobileDreamer, an efficient world-model-based lookahead framework to equip the GUI agents based on the future imagination provided by the world model. It consists of textual sketch world model and rollout imagination for GUI agent. Textual sketch world model forecasts post-action states through a learning process to transform digital images into key task-related sketches, and designs a novel order-invariant learning strategy to preserve the spatial information of GUI elements. The rollout imagination strategy for the GUI agent optimizes the action-selection process by leveraging the prediction capability of world model. Empirical results on Android World demonstrate that MobileDreamer consistently improves the task success rate across multiple base models, achieving an average performance gain of 5.3%. World model evaluations further verify that our textual sketch modeling accurately forecasts key GUI elements.
%U https://aclanthology.org/2026.findings-emnlp.1337/
%P 28345-28359
Markdown (Informal)
[MobileDreamer: Generative Sketch World Model for GUI Agent](https://aclanthology.org/2026.findings-emnlp.1337/) (Cao et al., Findings 2026)
ACL
- Yilin Cao, Yufeng Zhong, Zhixiong Zeng, Liming Zheng, Jing Huang, Haibo Qiu, Peng Shi, and Wenji Mao. 2026. MobileDreamer: Generative Sketch World Model for GUI Agent. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 28345–28359, Budapest, Hungary. Association for Computational Linguistics.