@inproceedings{li-etal-2026-skill-3d,
title = "{S}kill-3{D}: Evolving Scene-Aware Skills for Agentic 3{D} Spatial Reasoning",
author = "Li, Haoyuan and
Hu, Zhengdong and
Wang, Jun and
Fan, Hehe and
Yang, Yi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.203/",
pages = "4273--4294",
ISBN = "979-8-89176-462-0",
abstract = "This paper explores agentic 3D spatial understanding, \textit{i.e.,} MLLM agents performing indoor 3D reasoning through tool use. Existing methods often misuse tools and exhibit biased tool preferences, leaving the agentic paradigm with only marginal gains over non-agentic strategies. We reveal that 3D spatial reasoning tasks are heterogeneous across scenes, while these agents apply a uniform tool-use strategy to all scenes rather than selecting tools according to the specific scene and task. To address this, we propose \textbf{Skill-3D}, a framework that learns self-evolving scene-aware skills. Specifically, Skill-3D identifies the task scene and records the agent{'}s tool-use trajectory into a \textit{Scene Memory}, where successful trajectories from similar scenes are aggregated and distilled into a reusable scene-aware skill, with failed ones attached to the skill as lessons. During training, once a similar scene recurs, the corresponding skill is injected to guide the agent, producing new trajectories whose successes and failures further refine the skill, forming a loop in which the memory and the skill library co-evolve. Experiments show that Skill-3D substantially improves tool utilization in 3D spatial reasoning (from 39{\%}to 78{\%} on VSI-Bench), driving the agent toward correct and sufficient tool use. For instance, it improves Gemini-3-Flash by 67{\%} on MMSI-Bench. Furthermore, we conduct agentic reinforcement learning over skill-guided trajectories, which boosts Qwen3-VL-8B by 60{\%} on VSI-Bench."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-skill-3d">
<titleInfo>
<title>Skill-3D: Evolving Scene-Aware Skills for Agentic 3D Spatial Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Haoyuan</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhengdong</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hehe</namePart>
<namePart type="family">Fan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>This paper explores agentic 3D spatial understanding, i.e., MLLM agents performing indoor 3D reasoning through tool use. Existing methods often misuse tools and exhibit biased tool preferences, leaving the agentic paradigm with only marginal gains over non-agentic strategies. We reveal that 3D spatial reasoning tasks are heterogeneous across scenes, while these agents apply a uniform tool-use strategy to all scenes rather than selecting tools according to the specific scene and task. To address this, we propose Skill-3D, a framework that learns self-evolving scene-aware skills. Specifically, Skill-3D identifies the task scene and records the agent’s tool-use trajectory into a Scene Memory, where successful trajectories from similar scenes are aggregated and distilled into a reusable scene-aware skill, with failed ones attached to the skill as lessons. During training, once a similar scene recurs, the corresponding skill is injected to guide the agent, producing new trajectories whose successes and failures further refine the skill, forming a loop in which the memory and the skill library co-evolve. Experiments show that Skill-3D substantially improves tool utilization in 3D spatial reasoning (from 39%to 78% on VSI-Bench), driving the agent toward correct and sufficient tool use. For instance, it improves Gemini-3-Flash by 67% on MMSI-Bench. Furthermore, we conduct agentic reinforcement learning over skill-guided trajectories, which boosts Qwen3-VL-8B by 60% on VSI-Bench.</abstract>
<identifier type="citekey">li-etal-2026-skill-3d</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.203/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>4273</start>
<end>4294</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Skill-3D: Evolving Scene-Aware Skills for Agentic 3D Spatial Reasoning
%A Li, Haoyuan
%A Hu, Zhengdong
%A Wang, Jun
%A Fan, Hehe
%A Yang, Yi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F li-etal-2026-skill-3d
%X This paper explores agentic 3D spatial understanding, i.e., MLLM agents performing indoor 3D reasoning through tool use. Existing methods often misuse tools and exhibit biased tool preferences, leaving the agentic paradigm with only marginal gains over non-agentic strategies. We reveal that 3D spatial reasoning tasks are heterogeneous across scenes, while these agents apply a uniform tool-use strategy to all scenes rather than selecting tools according to the specific scene and task. To address this, we propose Skill-3D, a framework that learns self-evolving scene-aware skills. Specifically, Skill-3D identifies the task scene and records the agent’s tool-use trajectory into a Scene Memory, where successful trajectories from similar scenes are aggregated and distilled into a reusable scene-aware skill, with failed ones attached to the skill as lessons. During training, once a similar scene recurs, the corresponding skill is injected to guide the agent, producing new trajectories whose successes and failures further refine the skill, forming a loop in which the memory and the skill library co-evolve. Experiments show that Skill-3D substantially improves tool utilization in 3D spatial reasoning (from 39%to 78% on VSI-Bench), driving the agent toward correct and sufficient tool use. For instance, it improves Gemini-3-Flash by 67% on MMSI-Bench. Furthermore, we conduct agentic reinforcement learning over skill-guided trajectories, which boosts Qwen3-VL-8B by 60% on VSI-Bench.
%U https://aclanthology.org/2026.findings-emnlp.203/
%P 4273-4294
Markdown (Informal)
[Skill-3D: Evolving Scene-Aware Skills for Agentic 3D Spatial Reasoning](https://aclanthology.org/2026.findings-emnlp.203/) (Li et al., Findings 2026)
ACL