@inproceedings{yoshida-etal-2026-signals,
title = "Signals Are Present, Predictions Fail: Probing Diagram Representations in Large Vision-Language Models",
author = "Yoshida, Haruto and
Kudo, Keito and
Aoki, Yoichi and
Tanaka, Ryota and
Saito, Itsumi and
Sakaguchi, Keisuke and
Inui, Kentaro",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2689/",
pages = "61160--61197",
ISBN = "979-8-89176-459-0",
abstract = "Large vision-language models (LVLMs) demonstrate strong performance on diagram understanding benchmarks, yet they still struggle to understand relationships between elements, particularly those represented by directed edges between nodes. To investigate the causes of this limitation, we probe the internal representations of LVLMs using a synthetic diagram dataset based on directed graphs. Our linear probing experiments reveal that signals about nodes and edges are linearly decodable from the hidden states of LVLMs, even for aspects that LVLMs struggle with in VQA, such as edge direction and connectivity between nodes. These results reveal a gap between internal signals and model outputs. We further perform causal interventions on the model{'}s hidden states, showing that, for some aspects, the signals identified through probing causally contribute to the model{'}s final predictions. These findings also hold on flowcharts, not only on synthetic diagrams."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yoshida-etal-2026-signals">
<titleInfo>
<title>Signals Are Present, Predictions Fail: Probing Diagram Representations in Large Vision-Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Haruto</namePart>
<namePart type="family">Yoshida</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keito</namePart>
<namePart type="family">Kudo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yoichi</namePart>
<namePart type="family">Aoki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryota</namePart>
<namePart type="family">Tanaka</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Itsumi</namePart>
<namePart type="family">Saito</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keisuke</namePart>
<namePart type="family">Sakaguchi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kentaro</namePart>
<namePart type="family">Inui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large vision-language models (LVLMs) demonstrate strong performance on diagram understanding benchmarks, yet they still struggle to understand relationships between elements, particularly those represented by directed edges between nodes. To investigate the causes of this limitation, we probe the internal representations of LVLMs using a synthetic diagram dataset based on directed graphs. Our linear probing experiments reveal that signals about nodes and edges are linearly decodable from the hidden states of LVLMs, even for aspects that LVLMs struggle with in VQA, such as edge direction and connectivity between nodes. These results reveal a gap between internal signals and model outputs. We further perform causal interventions on the model’s hidden states, showing that, for some aspects, the signals identified through probing causally contribute to the model’s final predictions. These findings also hold on flowcharts, not only on synthetic diagrams.</abstract>
<identifier type="citekey">yoshida-etal-2026-signals</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2689/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>61160</start>
<end>61197</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Signals Are Present, Predictions Fail: Probing Diagram Representations in Large Vision-Language Models
%A Yoshida, Haruto
%A Kudo, Keito
%A Aoki, Yoichi
%A Tanaka, Ryota
%A Saito, Itsumi
%A Sakaguchi, Keisuke
%A Inui, Kentaro
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yoshida-etal-2026-signals
%X Large vision-language models (LVLMs) demonstrate strong performance on diagram understanding benchmarks, yet they still struggle to understand relationships between elements, particularly those represented by directed edges between nodes. To investigate the causes of this limitation, we probe the internal representations of LVLMs using a synthetic diagram dataset based on directed graphs. Our linear probing experiments reveal that signals about nodes and edges are linearly decodable from the hidden states of LVLMs, even for aspects that LVLMs struggle with in VQA, such as edge direction and connectivity between nodes. These results reveal a gap between internal signals and model outputs. We further perform causal interventions on the model’s hidden states, showing that, for some aspects, the signals identified through probing causally contribute to the model’s final predictions. These findings also hold on flowcharts, not only on synthetic diagrams.
%U https://aclanthology.org/2026.emnlp-main.2689/
%P 61160-61197
Markdown (Informal)
[Signals Are Present, Predictions Fail: Probing Diagram Representations in Large Vision-Language Models](https://aclanthology.org/2026.emnlp-main.2689/) (Yoshida et al., EMNLP 2026)
ACL
- Haruto Yoshida, Keito Kudo, Yoichi Aoki, Ryota Tanaka, Itsumi Saito, Keisuke Sakaguchi, and Kentaro Inui. 2026. Signals Are Present, Predictions Fail: Probing Diagram Representations in Large Vision-Language Models. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 61160–61197, Budapest, Hungary. Association for Computational Linguistics.