@inproceedings{fime-etal-2026-benchmarks,
title = "From Benchmarks to Real-World Use: Evaluating the Robustness of Vision-Language Models for Chart Understanding",
author = "Fime, Awal Ahmed and
Islam, Mohammed Saidul and
Laskar, Md Tahmid Rahman and
Hossain, Md Zarif and
Mahbub, Ridwan and
Nayeem, Mir Tafseer and
Rahman, Mizanur and
Imteaj, Ahmed and
Prince, Enamul Hoque",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.165/",
pages = "2794--2814",
ISBN = "979-8-89176-460-6",
abstract = "Recent advances in large Vision-Language Models (VLMs) have led to impressive performance on chart understanding tasks. However, existing evaluations are largely limited to clean, canonical charts and do not adequately measure robustness under realistic conditions. To address this gap, we introduce ROBUSTCHART360, a comprehensive benchmark for evaluating chart understanding robustness under realistic real-world variations. ROBUSTCHART360 spans six robustness dimensions covering textual and cross-modal interference, rendering and display degradation, accessibility-oriented transformations, and visual style variation. Using this benchmark, we conduct a large-scale evaluation of 14 proprietary and open-source VLMs across 2,505 chart instances and multiple robustness conditions designed to simulate practical deployment scenarios. Our analysis on 35,070 model responses reveals that current VLMs are highly sensitive to seemingly minor variations such as low-resolution rendering, and mobile layouts, with performance dropping by as much as 33.9{\%} under certain conditions, such as modality gap. We further find that models often over-rely on textual priors when visual and textual signals conflict, indicating weak visual grounding despite strong performance on standard benchmarks. Our findings highlight a substantial gap between benchmark accuracy and real-world robustness, motivating the development of more reliable and visually grounded multimodal systems for chart understanding. We will release our code and dataset at {\ensuremath{<}}redacted{\ensuremath{>}}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="fime-etal-2026-benchmarks">
<titleInfo>
<title>From Benchmarks to Real-World Use: Evaluating the Robustness of Vision-Language Models for Chart Understanding</title>
</titleInfo>
<name type="personal">
<namePart type="given">Awal</namePart>
<namePart type="given">Ahmed</namePart>
<namePart type="family">Fime</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohammed</namePart>
<namePart type="given">Saidul</namePart>
<namePart type="family">Islam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Md</namePart>
<namePart type="given">Tahmid</namePart>
<namePart type="given">Rahman</namePart>
<namePart type="family">Laskar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Md</namePart>
<namePart type="given">Zarif</namePart>
<namePart type="family">Hossain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ridwan</namePart>
<namePart type="family">Mahbub</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mir</namePart>
<namePart type="given">Tafseer</namePart>
<namePart type="family">Nayeem</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mizanur</namePart>
<namePart type="family">Rahman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ahmed</namePart>
<namePart type="family">Imteaj</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Enamul</namePart>
<namePart type="given">Hoque</namePart>
<namePart type="family">Prince</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Recent advances in large Vision-Language Models (VLMs) have led to impressive performance on chart understanding tasks. However, existing evaluations are largely limited to clean, canonical charts and do not adequately measure robustness under realistic conditions. To address this gap, we introduce ROBUSTCHART360, a comprehensive benchmark for evaluating chart understanding robustness under realistic real-world variations. ROBUSTCHART360 spans six robustness dimensions covering textual and cross-modal interference, rendering and display degradation, accessibility-oriented transformations, and visual style variation. Using this benchmark, we conduct a large-scale evaluation of 14 proprietary and open-source VLMs across 2,505 chart instances and multiple robustness conditions designed to simulate practical deployment scenarios. Our analysis on 35,070 model responses reveals that current VLMs are highly sensitive to seemingly minor variations such as low-resolution rendering, and mobile layouts, with performance dropping by as much as 33.9% under certain conditions, such as modality gap. We further find that models often over-rely on textual priors when visual and textual signals conflict, indicating weak visual grounding despite strong performance on standard benchmarks. Our findings highlight a substantial gap between benchmark accuracy and real-world robustness, motivating the development of more reliable and visually grounded multimodal systems for chart understanding. We will release our code and dataset at \ensuremath<redacted\ensuremath>.</abstract>
<identifier type="citekey">fime-etal-2026-benchmarks</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.165/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>2794</start>
<end>2814</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T From Benchmarks to Real-World Use: Evaluating the Robustness of Vision-Language Models for Chart Understanding
%A Fime, Awal Ahmed
%A Islam, Mohammed Saidul
%A Laskar, Md Tahmid Rahman
%A Hossain, Md Zarif
%A Mahbub, Ridwan
%A Nayeem, Mir Tafseer
%A Rahman, Mizanur
%A Imteaj, Ahmed
%A Prince, Enamul Hoque
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F fime-etal-2026-benchmarks
%X Recent advances in large Vision-Language Models (VLMs) have led to impressive performance on chart understanding tasks. However, existing evaluations are largely limited to clean, canonical charts and do not adequately measure robustness under realistic conditions. To address this gap, we introduce ROBUSTCHART360, a comprehensive benchmark for evaluating chart understanding robustness under realistic real-world variations. ROBUSTCHART360 spans six robustness dimensions covering textual and cross-modal interference, rendering and display degradation, accessibility-oriented transformations, and visual style variation. Using this benchmark, we conduct a large-scale evaluation of 14 proprietary and open-source VLMs across 2,505 chart instances and multiple robustness conditions designed to simulate practical deployment scenarios. Our analysis on 35,070 model responses reveals that current VLMs are highly sensitive to seemingly minor variations such as low-resolution rendering, and mobile layouts, with performance dropping by as much as 33.9% under certain conditions, such as modality gap. We further find that models often over-rely on textual priors when visual and textual signals conflict, indicating weak visual grounding despite strong performance on standard benchmarks. Our findings highlight a substantial gap between benchmark accuracy and real-world robustness, motivating the development of more reliable and visually grounded multimodal systems for chart understanding. We will release our code and dataset at \ensuremath<redacted\ensuremath>.
%U https://aclanthology.org/2026.emnlp-industry.165/
%P 2794-2814
Markdown (Informal)
[From Benchmarks to Real-World Use: Evaluating the Robustness of Vision-Language Models for Chart Understanding](https://aclanthology.org/2026.emnlp-industry.165/) (Fime et al., EMNLP 2026)
ACL
- Awal Ahmed Fime, Mohammed Saidul Islam, Md Tahmid Rahman Laskar, Md Zarif Hossain, Ridwan Mahbub, Mir Tafseer Nayeem, Mizanur Rahman, Ahmed Imteaj, and Enamul Hoque Prince. 2026. From Benchmarks to Real-World Use: Evaluating the Robustness of Vision-Language Models for Chart Understanding. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track, pages 2794–2814, Budapest, Hungary. Association for Computational Linguistics.