@inproceedings{wang-etal-2026-illusion-visual,
title = "The Illusion of Visual Tool-Use: A Causal Audit of Thinking with Images",
author = "Wang, Zhiheng and
Peng, Bo and
Wei, Lai and
Lu, Chaochao",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.728/",
doi = "10.18653/2026.findings-emnlp.728",
pages = "15298--15318",
ISBN = "979-8-89176-462-0",
abstract = "The ``thinking-with-images'' paradigm equips multimodal LLMs with active visual operations such as crop-and-zoom. However, models using these operations often achieve only marginal or negative gains over direct inference at substantially higher token cost. They may also repeatedly crop irrelevant regions and fail on questions that direct inference answers correctly. We ask \textbf{whether the returned visual evidence causally affects the answer}. To answer this question, we formulate visual tool-use as a causal graph that separates \textit{observation-mediated paths} from \textit{action-induced shortcuts}. We then audit it through interventions at the three levels: policy (comparing tool-use with direct inference), trajectory (corrupting all observations during rollout), and step (counterfactually replacing one individual observation under a fixed prefix). Our step-level estimand, \textit{Visual Evidence Gain}, isolates the contribution of each returned observation. Across six representative models and five fine-grained perception benchmarks, we uncover \textit{policy miscalibration} with two failure modes. In \textit{Calling Without Looking}, returned observations have no causal effect on the answer. In \textit{Looking Without Planning}, observations are informative but the call schedule is incoherent. A trajectory-level diagnostic decomposes the policy-level accuracy gain into per-group contributions and shows that the gain is concentrated in a \textit{Calibrated} minority. We term this discrepancy the \textbf{\textit{illusion of visual tool-use}}: despite aggregate accuracy gains, visual tool-use is not causally effective across a broad range of rollouts. The code is available at \url{https://github.com/OpenCausaLab/CauAudit}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-illusion-visual">
<titleInfo>
<title>The Illusion of Visual Tool-Use: A Causal Audit of Thinking with Images</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhiheng</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bo</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lai</namePart>
<namePart type="family">Wei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chaochao</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>The “thinking-with-images” paradigm equips multimodal LLMs with active visual operations such as crop-and-zoom. However, models using these operations often achieve only marginal or negative gains over direct inference at substantially higher token cost. They may also repeatedly crop irrelevant regions and fail on questions that direct inference answers correctly. We ask whether the returned visual evidence causally affects the answer. To answer this question, we formulate visual tool-use as a causal graph that separates observation-mediated paths from action-induced shortcuts. We then audit it through interventions at the three levels: policy (comparing tool-use with direct inference), trajectory (corrupting all observations during rollout), and step (counterfactually replacing one individual observation under a fixed prefix). Our step-level estimand, Visual Evidence Gain, isolates the contribution of each returned observation. Across six representative models and five fine-grained perception benchmarks, we uncover policy miscalibration with two failure modes. In Calling Without Looking, returned observations have no causal effect on the answer. In Looking Without Planning, observations are informative but the call schedule is incoherent. A trajectory-level diagnostic decomposes the policy-level accuracy gain into per-group contributions and shows that the gain is concentrated in a Calibrated minority. We term this discrepancy the illusion of visual tool-use: despite aggregate accuracy gains, visual tool-use is not causally effective across a broad range of rollouts. The code is available at https://github.com/OpenCausaLab/CauAudit.</abstract>
<identifier type="citekey">wang-etal-2026-illusion-visual</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.728</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.728/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>15298</start>
<end>15318</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Illusion of Visual Tool-Use: A Causal Audit of Thinking with Images
%A Wang, Zhiheng
%A Peng, Bo
%A Wei, Lai
%A Lu, Chaochao
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wang-etal-2026-illusion-visual
%X The “thinking-with-images” paradigm equips multimodal LLMs with active visual operations such as crop-and-zoom. However, models using these operations often achieve only marginal or negative gains over direct inference at substantially higher token cost. They may also repeatedly crop irrelevant regions and fail on questions that direct inference answers correctly. We ask whether the returned visual evidence causally affects the answer. To answer this question, we formulate visual tool-use as a causal graph that separates observation-mediated paths from action-induced shortcuts. We then audit it through interventions at the three levels: policy (comparing tool-use with direct inference), trajectory (corrupting all observations during rollout), and step (counterfactually replacing one individual observation under a fixed prefix). Our step-level estimand, Visual Evidence Gain, isolates the contribution of each returned observation. Across six representative models and five fine-grained perception benchmarks, we uncover policy miscalibration with two failure modes. In Calling Without Looking, returned observations have no causal effect on the answer. In Looking Without Planning, observations are informative but the call schedule is incoherent. A trajectory-level diagnostic decomposes the policy-level accuracy gain into per-group contributions and shows that the gain is concentrated in a Calibrated minority. We term this discrepancy the illusion of visual tool-use: despite aggregate accuracy gains, visual tool-use is not causally effective across a broad range of rollouts. The code is available at https://github.com/OpenCausaLab/CauAudit.
%R 10.18653/2026.findings-emnlp.728
%U https://aclanthology.org/2026.findings-emnlp.728/
%U https://doi.org/10.18653/2026.findings-emnlp.728
%P 15298-15318
Markdown (Informal)
[The Illusion of Visual Tool-Use: A Causal Audit of Thinking with Images](https://aclanthology.org/2026.findings-emnlp.728/) (Wang et al., Findings 2026)
ACL