@inproceedings{mousi-etal-2026-correct,
title = "Once Correct, Still Wrong: Counterfactual Hallucination in Multilingual Vision-Language Models",
author = "Mousi, Basel and
Dalvi, Fahim and
Chowdhury, Shammur Absar and
Alam, Firoj and
Durrani, Nadir",
editor = "Liakata, Maria and
Moreira, Viviane P. and
Zhang, Jiajun and
Jurgens, David",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026",
month = jul,
year = "2026",
address = "San Diego, California, United States",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-acl.234/",
pages = "4763--4788",
ISBN = "979-8-89176-395-1",
abstract = "Vision{--}language models (VLMs) can achieve high accuracy while still accepting **culturally plausible but visually incorrect** interpretations. Existing hallucination benchmarks rarely test this failure mode, particularly outside Western contexts and English. We introduce **M$^2$CQA**, a culturally grounded multimodal benchmark built from images spanning 17 MENA countries, paired with contrastive true and counterfactual statements in English, Arabic, and its dialects. To isolate hallucination beyond raw accuracy, we propose the **CounterFactual Hallucination Rate (CFHR)**, which measures counterfactual acceptance conditioned on correctly answering the true statement. Evaluating state-of-the-art VLMs under multiple prompting strategies, we find that CFHR rises sharply in Arabic, especially in dialects, even when true-statement accuracy remains high.Moreover, reasoning-first prompting consistently increases counterfactual hallucination, while answering before justifying improves robustness. We make the dataset publicly available for the community (https://huggingface.co/datasets/QCRI/M2CQA))."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mousi-etal-2026-correct">
<titleInfo>
<title>Once Correct, Still Wrong: Counterfactual Hallucination in Multilingual Vision-Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Basel</namePart>
<namePart type="family">Mousi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fahim</namePart>
<namePart type="family">Dalvi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shammur</namePart>
<namePart type="given">Absar</namePart>
<namePart type="family">Chowdhury</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Firoj</namePart>
<namePart type="family">Alam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nadir</namePart>
<namePart type="family">Durrani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: ACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Maria</namePart>
<namePart type="family">Liakata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Viviane</namePart>
<namePart type="given">P</namePart>
<namePart type="family">Moreira</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiajun</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Jurgens</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">San Diego, California, United States</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-395-1</identifier>
</relatedItem>
<abstract>Vision–language models (VLMs) can achieve high accuracy while still accepting **culturally plausible but visually incorrect** interpretations. Existing hallucination benchmarks rarely test this failure mode, particularly outside Western contexts and English. We introduce **M²CQA**, a culturally grounded multimodal benchmark built from images spanning 17 MENA countries, paired with contrastive true and counterfactual statements in English, Arabic, and its dialects. To isolate hallucination beyond raw accuracy, we propose the **CounterFactual Hallucination Rate (CFHR)**, which measures counterfactual acceptance conditioned on correctly answering the true statement. Evaluating state-of-the-art VLMs under multiple prompting strategies, we find that CFHR rises sharply in Arabic, especially in dialects, even when true-statement accuracy remains high.Moreover, reasoning-first prompting consistently increases counterfactual hallucination, while answering before justifying improves robustness. We make the dataset publicly available for the community (https://huggingface.co/datasets/QCRI/M2CQA)).</abstract>
<identifier type="citekey">mousi-etal-2026-correct</identifier>
<location>
<url>https://aclanthology.org/2026.findings-acl.234/</url>
</location>
<part>
<date>2026-07</date>
<extent unit="page">
<start>4763</start>
<end>4788</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Once Correct, Still Wrong: Counterfactual Hallucination in Multilingual Vision-Language Models
%A Mousi, Basel
%A Dalvi, Fahim
%A Chowdhury, Shammur Absar
%A Alam, Firoj
%A Durrani, Nadir
%Y Liakata, Maria
%Y Moreira, Viviane P.
%Y Zhang, Jiajun
%Y Jurgens, David
%S Findings of the Association for Computational Linguistics: ACL 2026
%D 2026
%8 July
%I Association for Computational Linguistics
%C San Diego, California, United States
%@ 979-8-89176-395-1
%F mousi-etal-2026-correct
%X Vision–language models (VLMs) can achieve high accuracy while still accepting **culturally plausible but visually incorrect** interpretations. Existing hallucination benchmarks rarely test this failure mode, particularly outside Western contexts and English. We introduce **M²CQA**, a culturally grounded multimodal benchmark built from images spanning 17 MENA countries, paired with contrastive true and counterfactual statements in English, Arabic, and its dialects. To isolate hallucination beyond raw accuracy, we propose the **CounterFactual Hallucination Rate (CFHR)**, which measures counterfactual acceptance conditioned on correctly answering the true statement. Evaluating state-of-the-art VLMs under multiple prompting strategies, we find that CFHR rises sharply in Arabic, especially in dialects, even when true-statement accuracy remains high.Moreover, reasoning-first prompting consistently increases counterfactual hallucination, while answering before justifying improves robustness. We make the dataset publicly available for the community (https://huggingface.co/datasets/QCRI/M2CQA)).
%U https://aclanthology.org/2026.findings-acl.234/
%P 4763-4788
Markdown (Informal)
[Once Correct, Still Wrong: Counterfactual Hallucination in Multilingual Vision-Language Models](https://aclanthology.org/2026.findings-acl.234/) (Mousi et al., Findings 2026)
ACL