@inproceedings{wang-etal-2026-think,
title = "Think, Act, Build: An Agentic Framework with Vision Language Models for Zero-Shot 3{D} Visual Grounding",
author = "Wang, Haibo and
Lin, Zihao and
Xu, Zhiyang and
Huang, Lifu",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.649/",
pages = "14935--14953",
ISBN = "979-8-89176-459-0",
abstract = "3D Visual Grounding (3D-VG) localizes objects in 3D scenes via natural language descriptions. Current zero-shot Vision-Language Models (VLMs) typically rely on preprocessed 3D point clouds and pretrained 3D detectors, reducing grounding to static proposal matching. To address this, we propose ``Think, Act, Build (TAB)'', a dynamic agentic framework that reformulates 3D-VG as a 2D-to-3D reconstruction paradigm operating directly on raw RGB-D streams. TAB decouples the task by leveraging 2D VLMs for complex spatial semantics and multi-view geometry for 3D structure instantiation. Guided by a specialized skill, our VLM agent dynamically invokes visual tools to track targets across 2D frames. To overcome multi-view coverage deficits inherent in strict VLM semantic tracking, we introduce Semantic-Anchored Geometric Expansion. This mechanism anchors the target in a reference video clip and uses multi-view geometry to propagate its spatial location across unobserved frames, mapping 2D visual cues to 3D coordinates. Additionally, we manually refine existing benchmarks to eliminate reference ambiguity and category errors. Extensive experiments on ScanRefer and Nr3D demonstrate that TAB, relying entirely on open-source models, significantly outperforms previous zero-shot methods and even supervised baselines."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-think">
<titleInfo>
<title>Think, Act, Build: An Agentic Framework with Vision Language Models for Zero-Shot 3D Visual Grounding</title>
</titleInfo>
<name type="personal">
<namePart type="given">Haibo</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zihao</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyang</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lifu</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>3D Visual Grounding (3D-VG) localizes objects in 3D scenes via natural language descriptions. Current zero-shot Vision-Language Models (VLMs) typically rely on preprocessed 3D point clouds and pretrained 3D detectors, reducing grounding to static proposal matching. To address this, we propose “Think, Act, Build (TAB)”, a dynamic agentic framework that reformulates 3D-VG as a 2D-to-3D reconstruction paradigm operating directly on raw RGB-D streams. TAB decouples the task by leveraging 2D VLMs for complex spatial semantics and multi-view geometry for 3D structure instantiation. Guided by a specialized skill, our VLM agent dynamically invokes visual tools to track targets across 2D frames. To overcome multi-view coverage deficits inherent in strict VLM semantic tracking, we introduce Semantic-Anchored Geometric Expansion. This mechanism anchors the target in a reference video clip and uses multi-view geometry to propagate its spatial location across unobserved frames, mapping 2D visual cues to 3D coordinates. Additionally, we manually refine existing benchmarks to eliminate reference ambiguity and category errors. Extensive experiments on ScanRefer and Nr3D demonstrate that TAB, relying entirely on open-source models, significantly outperforms previous zero-shot methods and even supervised baselines.</abstract>
<identifier type="citekey">wang-etal-2026-think</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.649/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>14935</start>
<end>14953</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Think, Act, Build: An Agentic Framework with Vision Language Models for Zero-Shot 3D Visual Grounding
%A Wang, Haibo
%A Lin, Zihao
%A Xu, Zhiyang
%A Huang, Lifu
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wang-etal-2026-think
%X 3D Visual Grounding (3D-VG) localizes objects in 3D scenes via natural language descriptions. Current zero-shot Vision-Language Models (VLMs) typically rely on preprocessed 3D point clouds and pretrained 3D detectors, reducing grounding to static proposal matching. To address this, we propose “Think, Act, Build (TAB)”, a dynamic agentic framework that reformulates 3D-VG as a 2D-to-3D reconstruction paradigm operating directly on raw RGB-D streams. TAB decouples the task by leveraging 2D VLMs for complex spatial semantics and multi-view geometry for 3D structure instantiation. Guided by a specialized skill, our VLM agent dynamically invokes visual tools to track targets across 2D frames. To overcome multi-view coverage deficits inherent in strict VLM semantic tracking, we introduce Semantic-Anchored Geometric Expansion. This mechanism anchors the target in a reference video clip and uses multi-view geometry to propagate its spatial location across unobserved frames, mapping 2D visual cues to 3D coordinates. Additionally, we manually refine existing benchmarks to eliminate reference ambiguity and category errors. Extensive experiments on ScanRefer and Nr3D demonstrate that TAB, relying entirely on open-source models, significantly outperforms previous zero-shot methods and even supervised baselines.
%U https://aclanthology.org/2026.emnlp-main.649/
%P 14935-14953
Markdown (Informal)
[Think, Act, Build: An Agentic Framework with Vision Language Models for Zero-Shot 3D Visual Grounding](https://aclanthology.org/2026.emnlp-main.649/) (Wang et al., EMNLP 2026)
ACL