@inproceedings{wang-etal-2026-cross,
title = "Cross-Lingual Activation Steering for Visual-Prior Conflicts",
author = "Wang, Yufei and
Li, Xiang Lorraine and
Kovashka, Adriana",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1280/",
pages = "27124--27135",
ISBN = "979-8-89176-462-0",
abstract = "Multimodal large language models (MLLMs) must often arbitrate between visual evidence and world-knowledge priors when an image conflicts with commonsense expectations. Existing work has demonstrated that activation steering vectors can bias model outputs toward either source of information, but it remains unclear whether this arbitration mechanism generalizes across languages. To investigate this, we construct a multilingual extension of the Visual CounterFact color split and derive activation steering directions that contrast image-grounded answers with prior-based answers. Across 12 languages, we find that English-derived and shared multilingual steering directions consistently shift model outputs on non-English prompts, suggesting that visual-prior arbitration has a substantial cross-lingual component. At the same time, we find that performance gains on common knowledge questions can come at the cost of image-grounded accuracy, a trade-off that is particularly pronounced for lower-resource languages. Code and data are available at https://github.com/elffiona/cross-lingual-visual-prior-steering."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-cross">
<titleInfo>
<title>Cross-Lingual Activation Steering for Visual-Prior Conflicts</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yufei</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiang</namePart>
<namePart type="given">Lorraine</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Adriana</namePart>
<namePart type="family">Kovashka</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Multimodal large language models (MLLMs) must often arbitrate between visual evidence and world-knowledge priors when an image conflicts with commonsense expectations. Existing work has demonstrated that activation steering vectors can bias model outputs toward either source of information, but it remains unclear whether this arbitration mechanism generalizes across languages. To investigate this, we construct a multilingual extension of the Visual CounterFact color split and derive activation steering directions that contrast image-grounded answers with prior-based answers. Across 12 languages, we find that English-derived and shared multilingual steering directions consistently shift model outputs on non-English prompts, suggesting that visual-prior arbitration has a substantial cross-lingual component. At the same time, we find that performance gains on common knowledge questions can come at the cost of image-grounded accuracy, a trade-off that is particularly pronounced for lower-resource languages. Code and data are available at https://github.com/elffiona/cross-lingual-visual-prior-steering.</abstract>
<identifier type="citekey">wang-etal-2026-cross</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1280/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>27124</start>
<end>27135</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Cross-Lingual Activation Steering for Visual-Prior Conflicts
%A Wang, Yufei
%A Li, Xiang Lorraine
%A Kovashka, Adriana
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wang-etal-2026-cross
%X Multimodal large language models (MLLMs) must often arbitrate between visual evidence and world-knowledge priors when an image conflicts with commonsense expectations. Existing work has demonstrated that activation steering vectors can bias model outputs toward either source of information, but it remains unclear whether this arbitration mechanism generalizes across languages. To investigate this, we construct a multilingual extension of the Visual CounterFact color split and derive activation steering directions that contrast image-grounded answers with prior-based answers. Across 12 languages, we find that English-derived and shared multilingual steering directions consistently shift model outputs on non-English prompts, suggesting that visual-prior arbitration has a substantial cross-lingual component. At the same time, we find that performance gains on common knowledge questions can come at the cost of image-grounded accuracy, a trade-off that is particularly pronounced for lower-resource languages. Code and data are available at https://github.com/elffiona/cross-lingual-visual-prior-steering.
%U https://aclanthology.org/2026.findings-emnlp.1280/
%P 27124-27135
Markdown (Informal)
[Cross-Lingual Activation Steering for Visual-Prior Conflicts](https://aclanthology.org/2026.findings-emnlp.1280/) (Wang et al., Findings 2026)
ACL