@inproceedings{wan-etal-2026-end,
title = "End-to-End Differentiable Optimization of Tool-Augmented Visual Reasoning via Probabilistic Graphs",
author = "Wan, Wentao and
Wu, Kaiyu and
Ma, Qingyang and
Kang, Nan and
Chen, Yunjie and
Lin, Liang and
Wang, Keze",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2444/",
pages = "52467--52482",
ISBN = "979-8-89176-462-0",
abstract = "Tool-augmented visual reasoning systems, exemplified by Visual Programming (VP), decompose complex visual questions into executable programs that invoke specialized visual models as tools, offering interpretable and compositional reasoning. However, these systems face a fundamental optimization challenge: the non-differentiable pipeline and the absence of intermediate supervision prevent end-to-end gradient-based optimization of invoked tools using only task-level labels. Existing methods either optimize the Large Language Model (LLM) planner while neglecting tool quality, or rely on teacher models and additional data collection, none achieving true end-to-end differentiable optimization. We propose **eDoT**, the first **e**nd-to-end **D**ifferentiable **o**ptimization method for **T**ool-augmented visual reasoning. Our key insight is that any program{'}s variable dependency structure is isomorphic to a directed probabilistic graph. Applying Variable Elimination - an exact probabilistic inference technique - transforms the non-differentiable VP execution into a fully differentiable process, enabling gradient-based optimization with only task-level supervision, without additional data or teacher models. Experiments on GQA, NLVRv2 and Open Images demonstrate improvements of 3.7{\%}, 7.6{\%} and 20{\%}, validating eDoT{'}s effectiveness across multiple visual models and cross-task generalization."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wan-etal-2026-end">
<titleInfo>
<title>End-to-End Differentiable Optimization of Tool-Augmented Visual Reasoning via Probabilistic Graphs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wentao</namePart>
<namePart type="family">Wan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kaiyu</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qingyang</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nan</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yunjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liang</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keze</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Tool-augmented visual reasoning systems, exemplified by Visual Programming (VP), decompose complex visual questions into executable programs that invoke specialized visual models as tools, offering interpretable and compositional reasoning. However, these systems face a fundamental optimization challenge: the non-differentiable pipeline and the absence of intermediate supervision prevent end-to-end gradient-based optimization of invoked tools using only task-level labels. Existing methods either optimize the Large Language Model (LLM) planner while neglecting tool quality, or rely on teacher models and additional data collection, none achieving true end-to-end differentiable optimization. We propose **eDoT**, the first **e**nd-to-end **D**ifferentiable **o**ptimization method for **T**ool-augmented visual reasoning. Our key insight is that any program’s variable dependency structure is isomorphic to a directed probabilistic graph. Applying Variable Elimination - an exact probabilistic inference technique - transforms the non-differentiable VP execution into a fully differentiable process, enabling gradient-based optimization with only task-level supervision, without additional data or teacher models. Experiments on GQA, NLVRv2 and Open Images demonstrate improvements of 3.7%, 7.6% and 20%, validating eDoT’s effectiveness across multiple visual models and cross-task generalization.</abstract>
<identifier type="citekey">wan-etal-2026-end</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2444/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>52467</start>
<end>52482</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T End-to-End Differentiable Optimization of Tool-Augmented Visual Reasoning via Probabilistic Graphs
%A Wan, Wentao
%A Wu, Kaiyu
%A Ma, Qingyang
%A Kang, Nan
%A Chen, Yunjie
%A Lin, Liang
%A Wang, Keze
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wan-etal-2026-end
%X Tool-augmented visual reasoning systems, exemplified by Visual Programming (VP), decompose complex visual questions into executable programs that invoke specialized visual models as tools, offering interpretable and compositional reasoning. However, these systems face a fundamental optimization challenge: the non-differentiable pipeline and the absence of intermediate supervision prevent end-to-end gradient-based optimization of invoked tools using only task-level labels. Existing methods either optimize the Large Language Model (LLM) planner while neglecting tool quality, or rely on teacher models and additional data collection, none achieving true end-to-end differentiable optimization. We propose **eDoT**, the first **e**nd-to-end **D**ifferentiable **o**ptimization method for **T**ool-augmented visual reasoning. Our key insight is that any program’s variable dependency structure is isomorphic to a directed probabilistic graph. Applying Variable Elimination - an exact probabilistic inference technique - transforms the non-differentiable VP execution into a fully differentiable process, enabling gradient-based optimization with only task-level supervision, without additional data or teacher models. Experiments on GQA, NLVRv2 and Open Images demonstrate improvements of 3.7%, 7.6% and 20%, validating eDoT’s effectiveness across multiple visual models and cross-task generalization.
%U https://aclanthology.org/2026.findings-emnlp.2444/
%P 52467-52482
Markdown (Informal)
[End-to-End Differentiable Optimization of Tool-Augmented Visual Reasoning via Probabilistic Graphs](https://aclanthology.org/2026.findings-emnlp.2444/) (Wan et al., Findings 2026)
ACL