@inproceedings{lee-etal-2026-semantics,
title = "Same Semantics, Different Outcome: On the Modality Robustness of Multimodal {LLM}s under Knowledge Conflict",
author = "Lee, Jungyeon and
Yoon, Yejin and
Kim, Taeuk",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.693/",
pages = "14529--14548",
ISBN = "979-8-89176-462-0",
abstract = "Multimodal large language models are increasingly given contextual evidence in heterogeneous forms: as a text passage, as a rendered image of the same text, or as both together. However, little is known about how consistently these surface forms are processed, especially when the evidence conflicts with the model{'}s parametric knowledge. In this work, we study modality robustness under knowledge conflict across 13 MLLMs and two datasets, and find them far from robust: (1) when a single context contradicts parametric knowledge, models tend to favor it more when presented as an image; (2) when a text and an image instead jointly conflict with parametric knowledge, the preferred source becomes essentially arbitrary, fluctuating drastically with input order, model choice, and dataset. We further show this instability affects downstream tasks like multimodal RAG and adversarial attacks: harmful instructions delivered as images, for example, raise attack success rates by up to 27 points. While conflict-aware fine-tuning partially mitigates the issue, we call for broader community attention to MLLMs' modality robustness."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lee-etal-2026-semantics">
<titleInfo>
<title>Same Semantics, Different Outcome: On the Modality Robustness of Multimodal LLMs under Knowledge Conflict</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jungyeon</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yejin</namePart>
<namePart type="family">Yoon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Taeuk</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Multimodal large language models are increasingly given contextual evidence in heterogeneous forms: as a text passage, as a rendered image of the same text, or as both together. However, little is known about how consistently these surface forms are processed, especially when the evidence conflicts with the model’s parametric knowledge. In this work, we study modality robustness under knowledge conflict across 13 MLLMs and two datasets, and find them far from robust: (1) when a single context contradicts parametric knowledge, models tend to favor it more when presented as an image; (2) when a text and an image instead jointly conflict with parametric knowledge, the preferred source becomes essentially arbitrary, fluctuating drastically with input order, model choice, and dataset. We further show this instability affects downstream tasks like multimodal RAG and adversarial attacks: harmful instructions delivered as images, for example, raise attack success rates by up to 27 points. While conflict-aware fine-tuning partially mitigates the issue, we call for broader community attention to MLLMs’ modality robustness.</abstract>
<identifier type="citekey">lee-etal-2026-semantics</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.693/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>14529</start>
<end>14548</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Same Semantics, Different Outcome: On the Modality Robustness of Multimodal LLMs under Knowledge Conflict
%A Lee, Jungyeon
%A Yoon, Yejin
%A Kim, Taeuk
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F lee-etal-2026-semantics
%X Multimodal large language models are increasingly given contextual evidence in heterogeneous forms: as a text passage, as a rendered image of the same text, or as both together. However, little is known about how consistently these surface forms are processed, especially when the evidence conflicts with the model’s parametric knowledge. In this work, we study modality robustness under knowledge conflict across 13 MLLMs and two datasets, and find them far from robust: (1) when a single context contradicts parametric knowledge, models tend to favor it more when presented as an image; (2) when a text and an image instead jointly conflict with parametric knowledge, the preferred source becomes essentially arbitrary, fluctuating drastically with input order, model choice, and dataset. We further show this instability affects downstream tasks like multimodal RAG and adversarial attacks: harmful instructions delivered as images, for example, raise attack success rates by up to 27 points. While conflict-aware fine-tuning partially mitigates the issue, we call for broader community attention to MLLMs’ modality robustness.
%U https://aclanthology.org/2026.findings-emnlp.693/
%P 14529-14548
Markdown (Informal)
[Same Semantics, Different Outcome: On the Modality Robustness of Multimodal LLMs under Knowledge Conflict](https://aclanthology.org/2026.findings-emnlp.693/) (Lee et al., Findings 2026)
ACL