@inproceedings{mouilleron-etal-2026-tables,
title = "When Tables Go Crazy: Evaluating Multimodal Models on {F}rench Financial Documents",
author = "Mouilleron, Virginie and
Lasnier, Th{\'e}o and
Mosolova, Anna and
Seddah, Djam{\'e}",
editor = "El-Haj, Mo and
Moreno Sandoval, Antonio and
Garcia-Serrano, Ana and
Chen, Chung-Chi and
Rayson, Paul and
Torterolo Orta, Yanco Amor and
Martinez, Paloma and
Porta, Jordi",
booktitle = "The 7th Financial Narrative Processing Workshop",
month = may,
year = "2026",
address = "Palma de Mallorca, Spain",
publisher = "European Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.fnp-1.2/",
doi = "10.63317/3bk8fk9sek7b",
pages = "12--27",
abstract = "Vision-language models (VLMs) perform well on many document understanding tasks, yet their reliability in specialized, non-English domains remains underexplored. This gap is especially critical in finance, where documents mix dense regulatory text, numerical tables, and visual charts, and where extraction errors can have real-world consequences. We introduce SCRIBE FINANCE, the first multimodal benchmark for evaluating French financial document understanding. The dataset contains 1,204 expert-validated questions spanning text extraction, table comprehension, chart interpretation, and multi-turn conversational reasoning, drawn from real investment prospectuses, KIDs, and PRIIPs. We evaluate six open-weight VLMs (8B{--}124B parameters) using an LLM-as-judge protocol. While models achieve strong performance on text and table tasks (85{--}90{\%} accuracy), they struggle with chart interpretation (34{--}62{\%}). Most notably, multi-turn dialogue reveals a sharp failure mode: early mistakes propagate across turns, driving accuracy down to roughly 50{\%} regardless of model size. These results show that current VLMs are effective for well-defined extraction tasks but remain brittle in interactive, multi-step financial analysis. SCRIBE FINANCE offers a challenging benchmark to measure and drive progress in this high-stakes setting."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mouilleron-etal-2026-tables">
<titleInfo>
<title>When Tables Go Crazy: Evaluating Multimodal Models on French Financial Documents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Virginie</namePart>
<namePart type="family">Mouilleron</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Théo</namePart>
<namePart type="family">Lasnier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Mosolova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Djamé</namePart>
<namePart type="family">Seddah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The 7th Financial Narrative Processing Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mo</namePart>
<namePart type="family">El-Haj</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="family">Moreno Sandoval</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ana</namePart>
<namePart type="family">Garcia-Serrano</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chung-Chi</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paul</namePart>
<namePart type="family">Rayson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanco</namePart>
<namePart type="given">Amor</namePart>
<namePart type="family">Torterolo Orta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paloma</namePart>
<namePart type="family">Martinez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jordi</namePart>
<namePart type="family">Porta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>European Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma de Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Vision-language models (VLMs) perform well on many document understanding tasks, yet their reliability in specialized, non-English domains remains underexplored. This gap is especially critical in finance, where documents mix dense regulatory text, numerical tables, and visual charts, and where extraction errors can have real-world consequences. We introduce SCRIBE FINANCE, the first multimodal benchmark for evaluating French financial document understanding. The dataset contains 1,204 expert-validated questions spanning text extraction, table comprehension, chart interpretation, and multi-turn conversational reasoning, drawn from real investment prospectuses, KIDs, and PRIIPs. We evaluate six open-weight VLMs (8B–124B parameters) using an LLM-as-judge protocol. While models achieve strong performance on text and table tasks (85–90% accuracy), they struggle with chart interpretation (34–62%). Most notably, multi-turn dialogue reveals a sharp failure mode: early mistakes propagate across turns, driving accuracy down to roughly 50% regardless of model size. These results show that current VLMs are effective for well-defined extraction tasks but remain brittle in interactive, multi-step financial analysis. SCRIBE FINANCE offers a challenging benchmark to measure and drive progress in this high-stakes setting.</abstract>
<identifier type="citekey">mouilleron-etal-2026-tables</identifier>
<identifier type="doi">10.63317/3bk8fk9sek7b</identifier>
<location>
<url>https://aclanthology.org/2026.fnp-1.2/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>12</start>
<end>27</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Tables Go Crazy: Evaluating Multimodal Models on French Financial Documents
%A Mouilleron, Virginie
%A Lasnier, Théo
%A Mosolova, Anna
%A Seddah, Djamé
%Y El-Haj, Mo
%Y Moreno Sandoval, Antonio
%Y Garcia-Serrano, Ana
%Y Chen, Chung-Chi
%Y Rayson, Paul
%Y Torterolo Orta, Yanco Amor
%Y Martinez, Paloma
%Y Porta, Jordi
%S The 7th Financial Narrative Processing Workshop
%D 2026
%8 May
%I European Language Resources Association (ELRA)
%C Palma de Mallorca, Spain
%F mouilleron-etal-2026-tables
%X Vision-language models (VLMs) perform well on many document understanding tasks, yet their reliability in specialized, non-English domains remains underexplored. This gap is especially critical in finance, where documents mix dense regulatory text, numerical tables, and visual charts, and where extraction errors can have real-world consequences. We introduce SCRIBE FINANCE, the first multimodal benchmark for evaluating French financial document understanding. The dataset contains 1,204 expert-validated questions spanning text extraction, table comprehension, chart interpretation, and multi-turn conversational reasoning, drawn from real investment prospectuses, KIDs, and PRIIPs. We evaluate six open-weight VLMs (8B–124B parameters) using an LLM-as-judge protocol. While models achieve strong performance on text and table tasks (85–90% accuracy), they struggle with chart interpretation (34–62%). Most notably, multi-turn dialogue reveals a sharp failure mode: early mistakes propagate across turns, driving accuracy down to roughly 50% regardless of model size. These results show that current VLMs are effective for well-defined extraction tasks but remain brittle in interactive, multi-step financial analysis. SCRIBE FINANCE offers a challenging benchmark to measure and drive progress in this high-stakes setting.
%R 10.63317/3bk8fk9sek7b
%U https://aclanthology.org/2026.fnp-1.2/
%U https://doi.org/10.63317/3bk8fk9sek7b
%P 12-27
Markdown (Informal)
[When Tables Go Crazy: Evaluating Multimodal Models on French Financial Documents](https://aclanthology.org/2026.fnp-1.2/) (Mouilleron et al., FNP 2026)
ACL