@inproceedings{zhou-etal-2026-vision,
title = "Where Do Vision-Language Models Fail on Abstract Reasoning? An Information Processing Analysis of {ARC}",
author = "Zhou, Xuan and
Wei, Xuefeng and
Qu, Zhi and
Sakai, Yusuke and
Kamigaito, Hidetaka and
Watanabe, Taro",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2595/",
pages = "58927--58947",
ISBN = "979-8-89176-459-0",
abstract = "Current vision-language models (VLMs) perform well on many multimodal benchmarks, but still struggle with ARC-style abstract reasoning, where models must infer novel grid transformations from few examples. We analyze these failures through an information-processing decomposition into encoding, transformation, and construction, comparing three controlled settings: direct image-based solving, solving from model-produced text grids, and solving from gold text grids. Across existing ARC-1 and ARC-2 together with our controllable diagnostic benchmark V-ARAOC, we observe benchmark-dependent diagnostic signatures: ARC-1 is highly sensitive to encoding quality, ARC-2 remains difficult even after transcription noise is removed, and V-ARAOC is highly sensitive to constrained output construction. These results reveal benchmark-dependent information-processing bottlenecks in VLM-based ARC solving, with different benchmarks exhibiting distinct dominant failure modes."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhou-etal-2026-vision">
<titleInfo>
<title>Where Do Vision-Language Models Fail on Abstract Reasoning? An Information Processing Analysis of ARC</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xuan</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuefeng</namePart>
<namePart type="family">Wei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhi</namePart>
<namePart type="family">Qu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yusuke</namePart>
<namePart type="family">Sakai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hidetaka</namePart>
<namePart type="family">Kamigaito</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Taro</namePart>
<namePart type="family">Watanabe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Current vision-language models (VLMs) perform well on many multimodal benchmarks, but still struggle with ARC-style abstract reasoning, where models must infer novel grid transformations from few examples. We analyze these failures through an information-processing decomposition into encoding, transformation, and construction, comparing three controlled settings: direct image-based solving, solving from model-produced text grids, and solving from gold text grids. Across existing ARC-1 and ARC-2 together with our controllable diagnostic benchmark V-ARAOC, we observe benchmark-dependent diagnostic signatures: ARC-1 is highly sensitive to encoding quality, ARC-2 remains difficult even after transcription noise is removed, and V-ARAOC is highly sensitive to constrained output construction. These results reveal benchmark-dependent information-processing bottlenecks in VLM-based ARC solving, with different benchmarks exhibiting distinct dominant failure modes.</abstract>
<identifier type="citekey">zhou-etal-2026-vision</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2595/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>58927</start>
<end>58947</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Where Do Vision-Language Models Fail on Abstract Reasoning? An Information Processing Analysis of ARC
%A Zhou, Xuan
%A Wei, Xuefeng
%A Qu, Zhi
%A Sakai, Yusuke
%A Kamigaito, Hidetaka
%A Watanabe, Taro
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhou-etal-2026-vision
%X Current vision-language models (VLMs) perform well on many multimodal benchmarks, but still struggle with ARC-style abstract reasoning, where models must infer novel grid transformations from few examples. We analyze these failures through an information-processing decomposition into encoding, transformation, and construction, comparing three controlled settings: direct image-based solving, solving from model-produced text grids, and solving from gold text grids. Across existing ARC-1 and ARC-2 together with our controllable diagnostic benchmark V-ARAOC, we observe benchmark-dependent diagnostic signatures: ARC-1 is highly sensitive to encoding quality, ARC-2 remains difficult even after transcription noise is removed, and V-ARAOC is highly sensitive to constrained output construction. These results reveal benchmark-dependent information-processing bottlenecks in VLM-based ARC solving, with different benchmarks exhibiting distinct dominant failure modes.
%U https://aclanthology.org/2026.emnlp-main.2595/
%P 58927-58947
Markdown (Informal)
[Where Do Vision-Language Models Fail on Abstract Reasoning? An Information Processing Analysis of ARC](https://aclanthology.org/2026.emnlp-main.2595/) (Zhou et al., EMNLP 2026)
ACL