@inproceedings{zhang-etal-2026-rethinking-cross,
title = "Rethinking Cross-Modality Prompt Transfer: Geometric Priors over Linguistic Semantics",
author = "Zhang, Ningyuan and
Lu, Jie and
Li, Keqiuyin and
Zhang, Guangquan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1218/",
pages = "27815--27833",
ISBN = "979-8-89176-459-0",
abstract = "Cross-Modality Prompt Transfer (CMPT) has been explored to address the initialization instability and data scarcity challenges inherent in prompt tuning. Its empirical success is typically interpreted as the transfer of linguistic semantics acquired during language-based prompt pretraining. In this paper, we revisit this interpretation and explore what actually transfers when language-pretrained prompts are adapted to the visual modality. Through controlled empirical analyses, we show that transfer performance does not consistently correlate with source-task linguistic performance. Instead, transferability is better explained by the geometric priors of prompt vectors, particularly stable rank and spatial direction relative to target features. To further validate this interpretation, we directly construct prompts with favorable geometry without relying on language pretraining. The resulting prompts achieve competitive transfer performance against language-pretrained prompts across diverse visual tasks, supporting prompt geometry as a stronger empirical explanation of transferability."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-rethinking-cross">
<titleInfo>
<title>Rethinking Cross-Modality Prompt Transfer: Geometric Priors over Linguistic Semantics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ningyuan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jie</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keqiuyin</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Guangquan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Cross-Modality Prompt Transfer (CMPT) has been explored to address the initialization instability and data scarcity challenges inherent in prompt tuning. Its empirical success is typically interpreted as the transfer of linguistic semantics acquired during language-based prompt pretraining. In this paper, we revisit this interpretation and explore what actually transfers when language-pretrained prompts are adapted to the visual modality. Through controlled empirical analyses, we show that transfer performance does not consistently correlate with source-task linguistic performance. Instead, transferability is better explained by the geometric priors of prompt vectors, particularly stable rank and spatial direction relative to target features. To further validate this interpretation, we directly construct prompts with favorable geometry without relying on language pretraining. The resulting prompts achieve competitive transfer performance against language-pretrained prompts across diverse visual tasks, supporting prompt geometry as a stronger empirical explanation of transferability.</abstract>
<identifier type="citekey">zhang-etal-2026-rethinking-cross</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1218/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>27815</start>
<end>27833</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Rethinking Cross-Modality Prompt Transfer: Geometric Priors over Linguistic Semantics
%A Zhang, Ningyuan
%A Lu, Jie
%A Li, Keqiuyin
%A Zhang, Guangquan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhang-etal-2026-rethinking-cross
%X Cross-Modality Prompt Transfer (CMPT) has been explored to address the initialization instability and data scarcity challenges inherent in prompt tuning. Its empirical success is typically interpreted as the transfer of linguistic semantics acquired during language-based prompt pretraining. In this paper, we revisit this interpretation and explore what actually transfers when language-pretrained prompts are adapted to the visual modality. Through controlled empirical analyses, we show that transfer performance does not consistently correlate with source-task linguistic performance. Instead, transferability is better explained by the geometric priors of prompt vectors, particularly stable rank and spatial direction relative to target features. To further validate this interpretation, we directly construct prompts with favorable geometry without relying on language pretraining. The resulting prompts achieve competitive transfer performance against language-pretrained prompts across diverse visual tasks, supporting prompt geometry as a stronger empirical explanation of transferability.
%U https://aclanthology.org/2026.emnlp-main.1218/
%P 27815-27833
Markdown (Informal)
[Rethinking Cross-Modality Prompt Transfer: Geometric Priors over Linguistic Semantics](https://aclanthology.org/2026.emnlp-main.1218/) (Zhang et al., EMNLP 2026)
ACL