@inproceedings{zhang-etal-2026-see-beyond,
title = "See Beyond Words: Joint Visual{--}Language Representations Improve Brain Encoding of Language-Only Stimuli",
author = "Zhang, Yongpeng and
Yin, Congchi and
Xia, Runze and
Wen, Xuyun and
Li, Piji",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.935/",
pages = "21543--21560",
ISBN = "979-8-89176-459-0",
abstract = "Human language processing in the brain involves the coordinated engagement of multiple perceptual pathways. In parallel, recent multimodal large language models (MLLMs) integrate visual and linguistic information into language generation. However, it remains unclear whether and how the injection of visual information alters language representations, and how such changes in turn affect neural predictions during language processing. To address this question, we analyze joint vision{--}language representations extracted from MLLMs using brain encoding models. We visually augment textual stimuli via image generation and use the resulting representations to predict fMRI responses recorded while participants process language-only stimuli. We find that joint vision{--}language representations substantially improve neural prediction performance, with pronounced gains in higher-level visual regions. Further analyses indicate that these improvements reflect both the engagement of visual pathways and the enrichment of linguistic representations through visually grounded information. Together, our results provide evidence for multimodal pathway involvement in human language processing and offer insights into the interpretability of multimodal models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-see-beyond">
<titleInfo>
<title>See Beyond Words: Joint Visual–Language Representations Improve Brain Encoding of Language-Only Stimuli</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yongpeng</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Congchi</namePart>
<namePart type="family">Yin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Runze</namePart>
<namePart type="family">Xia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuyun</namePart>
<namePart type="family">Wen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Piji</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Human language processing in the brain involves the coordinated engagement of multiple perceptual pathways. In parallel, recent multimodal large language models (MLLMs) integrate visual and linguistic information into language generation. However, it remains unclear whether and how the injection of visual information alters language representations, and how such changes in turn affect neural predictions during language processing. To address this question, we analyze joint vision–language representations extracted from MLLMs using brain encoding models. We visually augment textual stimuli via image generation and use the resulting representations to predict fMRI responses recorded while participants process language-only stimuli. We find that joint vision–language representations substantially improve neural prediction performance, with pronounced gains in higher-level visual regions. Further analyses indicate that these improvements reflect both the engagement of visual pathways and the enrichment of linguistic representations through visually grounded information. Together, our results provide evidence for multimodal pathway involvement in human language processing and offer insights into the interpretability of multimodal models.</abstract>
<identifier type="citekey">zhang-etal-2026-see-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.935/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>21543</start>
<end>21560</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T See Beyond Words: Joint Visual–Language Representations Improve Brain Encoding of Language-Only Stimuli
%A Zhang, Yongpeng
%A Yin, Congchi
%A Xia, Runze
%A Wen, Xuyun
%A Li, Piji
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhang-etal-2026-see-beyond
%X Human language processing in the brain involves the coordinated engagement of multiple perceptual pathways. In parallel, recent multimodal large language models (MLLMs) integrate visual and linguistic information into language generation. However, it remains unclear whether and how the injection of visual information alters language representations, and how such changes in turn affect neural predictions during language processing. To address this question, we analyze joint vision–language representations extracted from MLLMs using brain encoding models. We visually augment textual stimuli via image generation and use the resulting representations to predict fMRI responses recorded while participants process language-only stimuli. We find that joint vision–language representations substantially improve neural prediction performance, with pronounced gains in higher-level visual regions. Further analyses indicate that these improvements reflect both the engagement of visual pathways and the enrichment of linguistic representations through visually grounded information. Together, our results provide evidence for multimodal pathway involvement in human language processing and offer insights into the interpretability of multimodal models.
%U https://aclanthology.org/2026.emnlp-main.935/
%P 21543-21560
Markdown (Informal)
[See Beyond Words: Joint Visual–Language Representations Improve Brain Encoding of Language-Only Stimuli](https://aclanthology.org/2026.emnlp-main.935/) (Zhang et al., EMNLP 2026)
ACL