@inproceedings{wang-etal-2026-irrelevant,
title = "When Irrelevant Text Matters: Affine Margin Shifts in Multimodal Large Language Models",
author = "Wang, Yinfeng and
Yao, Zhiyuan and
Fu, Zheren and
Zhang, Lei and
Mao, Zhendong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.884/",
pages = "20395--20417",
ISBN = "979-8-89176-459-0",
abstract = "Multimodal large language models (MLLMs) are frequently exposed to auxiliary textual context, the impact of which on vision-language tasks remains underexplored. In this paper, we investigate the influence of task-irrelevant context by formulating it as a controlled intervention within a binary visual judgment framework. Through controlled paired comparisons with and without auxiliary textual context, we observe that irrelevant text consistently biases model predictions across diverse benchmarks. To move beyond performance metrics, we characterize this sensitivity through ``decision margin'', which is defined by the log-probability difference between candidates. Our analysis reveals an empirical pattern: context-conditioned margins are often well approximated by an affine transformation of their context-free counterparts. We further interpret the fitted affine parameters as metrics for visual commitment preservation and directional answer bias. These findings provide a margin-level diagnostic view of irrelevant-context effects in MLLMs, indicating that irrelevant context does not manifest as stochastic noise, but as an estimable distortion of model preference."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-irrelevant">
<titleInfo>
<title>When Irrelevant Text Matters: Affine Margin Shifts in Multimodal Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yinfeng</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyuan</namePart>
<namePart type="family">Yao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zheren</namePart>
<namePart type="family">Fu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lei</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhendong</namePart>
<namePart type="family">Mao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Multimodal large language models (MLLMs) are frequently exposed to auxiliary textual context, the impact of which on vision-language tasks remains underexplored. In this paper, we investigate the influence of task-irrelevant context by formulating it as a controlled intervention within a binary visual judgment framework. Through controlled paired comparisons with and without auxiliary textual context, we observe that irrelevant text consistently biases model predictions across diverse benchmarks. To move beyond performance metrics, we characterize this sensitivity through “decision margin”, which is defined by the log-probability difference between candidates. Our analysis reveals an empirical pattern: context-conditioned margins are often well approximated by an affine transformation of their context-free counterparts. We further interpret the fitted affine parameters as metrics for visual commitment preservation and directional answer bias. These findings provide a margin-level diagnostic view of irrelevant-context effects in MLLMs, indicating that irrelevant context does not manifest as stochastic noise, but as an estimable distortion of model preference.</abstract>
<identifier type="citekey">wang-etal-2026-irrelevant</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.884/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>20395</start>
<end>20417</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Irrelevant Text Matters: Affine Margin Shifts in Multimodal Large Language Models
%A Wang, Yinfeng
%A Yao, Zhiyuan
%A Fu, Zheren
%A Zhang, Lei
%A Mao, Zhendong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wang-etal-2026-irrelevant
%X Multimodal large language models (MLLMs) are frequently exposed to auxiliary textual context, the impact of which on vision-language tasks remains underexplored. In this paper, we investigate the influence of task-irrelevant context by formulating it as a controlled intervention within a binary visual judgment framework. Through controlled paired comparisons with and without auxiliary textual context, we observe that irrelevant text consistently biases model predictions across diverse benchmarks. To move beyond performance metrics, we characterize this sensitivity through “decision margin”, which is defined by the log-probability difference between candidates. Our analysis reveals an empirical pattern: context-conditioned margins are often well approximated by an affine transformation of their context-free counterparts. We further interpret the fitted affine parameters as metrics for visual commitment preservation and directional answer bias. These findings provide a margin-level diagnostic view of irrelevant-context effects in MLLMs, indicating that irrelevant context does not manifest as stochastic noise, but as an estimable distortion of model preference.
%U https://aclanthology.org/2026.emnlp-main.884/
%P 20395-20417
Markdown (Informal)
[When Irrelevant Text Matters: Affine Margin Shifts in Multimodal Large Language Models](https://aclanthology.org/2026.emnlp-main.884/) (Wang et al., EMNLP 2026)
ACL