@inproceedings{xiao-etal-2026-transfer,
title = "Transfer Safety Awareness for Cross-Modal Safety Drift in Multimodal Large Language Models",
author = "Xiao, Tianqi and
Cui, Shiyao and
Zhang, Minghao and
Yang, Junxiao and
Chen, Renmiao",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.638/",
doi = "10.18653/2026.findings-emnlp.638",
pages = "13375--13391",
ISBN = "979-8-89176-462-0",
abstract = "Visual modality enhances the capabilities of Multimodal Large Language Models (MLLMs) but also introduces a safety concern: a benign textual query may convey harmful intent when grounded in a visual image. We term this as cross-modal safety drift, and our pilot studies show that the safety response rate for such requests is substantially lower than that for requests containing explicitly unsafe text. This paper aims to systematically study this issue. First, we conduct an empirical analysis to identify representative unsafe response patterns. Building on these, we interpret model representations and attentions, revealing that visually risky cues receive limited attention and weakly trigger refusal. Motivated by the observation that safety signals from unsafe text processing can be transferred, we propose safety-awareness representation transfer (SRT), a lightweight direction-refinement method that mitigates cross-modal safety drift while freezing the MLLM backbone. Experiments across benchmarks and models show that SRT effectively improves safety in diverse cross-modal settings while preserving utility."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xiao-etal-2026-transfer">
<titleInfo>
<title>Transfer Safety Awareness for Cross-Modal Safety Drift in Multimodal Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tianqi</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shiyao</namePart>
<namePart type="family">Cui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junxiao</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Renmiao</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Visual modality enhances the capabilities of Multimodal Large Language Models (MLLMs) but also introduces a safety concern: a benign textual query may convey harmful intent when grounded in a visual image. We term this as cross-modal safety drift, and our pilot studies show that the safety response rate for such requests is substantially lower than that for requests containing explicitly unsafe text. This paper aims to systematically study this issue. First, we conduct an empirical analysis to identify representative unsafe response patterns. Building on these, we interpret model representations and attentions, revealing that visually risky cues receive limited attention and weakly trigger refusal. Motivated by the observation that safety signals from unsafe text processing can be transferred, we propose safety-awareness representation transfer (SRT), a lightweight direction-refinement method that mitigates cross-modal safety drift while freezing the MLLM backbone. Experiments across benchmarks and models show that SRT effectively improves safety in diverse cross-modal settings while preserving utility.</abstract>
<identifier type="citekey">xiao-etal-2026-transfer</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.638</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.638/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>13375</start>
<end>13391</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Transfer Safety Awareness for Cross-Modal Safety Drift in Multimodal Large Language Models
%A Xiao, Tianqi
%A Cui, Shiyao
%A Zhang, Minghao
%A Yang, Junxiao
%A Chen, Renmiao
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F xiao-etal-2026-transfer
%X Visual modality enhances the capabilities of Multimodal Large Language Models (MLLMs) but also introduces a safety concern: a benign textual query may convey harmful intent when grounded in a visual image. We term this as cross-modal safety drift, and our pilot studies show that the safety response rate for such requests is substantially lower than that for requests containing explicitly unsafe text. This paper aims to systematically study this issue. First, we conduct an empirical analysis to identify representative unsafe response patterns. Building on these, we interpret model representations and attentions, revealing that visually risky cues receive limited attention and weakly trigger refusal. Motivated by the observation that safety signals from unsafe text processing can be transferred, we propose safety-awareness representation transfer (SRT), a lightweight direction-refinement method that mitigates cross-modal safety drift while freezing the MLLM backbone. Experiments across benchmarks and models show that SRT effectively improves safety in diverse cross-modal settings while preserving utility.
%R 10.18653/2026.findings-emnlp.638
%U https://aclanthology.org/2026.findings-emnlp.638/
%U https://doi.org/10.18653/2026.findings-emnlp.638
%P 13375-13391
Markdown (Informal)
[Transfer Safety Awareness for Cross-Modal Safety Drift in Multimodal Large Language Models](https://aclanthology.org/2026.findings-emnlp.638/) (Xiao et al., Findings 2026)
ACL