@inproceedings{qiu-shang-2026-rethinking,
title = "Rethinking Cross-Modal Attention for Visual Token Pruning in Multimodal {LVLM}s",
author = "Qiu, Diego Weng and
Shang, Lin",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.131/",
pages = "2886--2897",
ISBN = "979-8-89176-439-2",
abstract = "Recent studies challenge the reliability of cross-modal attention mechanisms as an indicator for visual token pruning in Large Vision-Language Models (LVLMs). In this work, we demonstrate that such negative conclusions can be substantially influenced by an overlooked phenomenon termed task-bound coupling, wherein cross-modal attention patterns become entangled with task-specific prompts. Through controlled configurations that decouple visual inputs from downstream queries, we observe that attention distributions can be relatively consistent across the evaluated settings. Concurrently, we identify an empirical scaling phenomenon which we term Attention Fidelity Decay, where the downstream effectiveness of attention maps exhibits semantic dilution in higher-capacity models or advanced architectures. Motivated by these insights, we introduce TAP-LLaVA, a simple training-free framework that leverages localized task-agnostic attention for efficient, reusable visual token pruning across multi-turn sessions. Our approach achieves competitive downstream performance while substantially lowering computational overhead. This suggests that attention can provide a reliable structural prior at moderate model scales when isolated from textual confounders."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="qiu-shang-2026-rethinking">
<titleInfo>
<title>Rethinking Cross-Modal Attention for Visual Token Pruning in Multimodal LVLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Diego</namePart>
<namePart type="given">Weng</namePart>
<namePart type="family">Qiu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lin</namePart>
<namePart type="family">Shang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Recent studies challenge the reliability of cross-modal attention mechanisms as an indicator for visual token pruning in Large Vision-Language Models (LVLMs). In this work, we demonstrate that such negative conclusions can be substantially influenced by an overlooked phenomenon termed task-bound coupling, wherein cross-modal attention patterns become entangled with task-specific prompts. Through controlled configurations that decouple visual inputs from downstream queries, we observe that attention distributions can be relatively consistent across the evaluated settings. Concurrently, we identify an empirical scaling phenomenon which we term Attention Fidelity Decay, where the downstream effectiveness of attention maps exhibits semantic dilution in higher-capacity models or advanced architectures. Motivated by these insights, we introduce TAP-LLaVA, a simple training-free framework that leverages localized task-agnostic attention for efficient, reusable visual token pruning across multi-turn sessions. Our approach achieves competitive downstream performance while substantially lowering computational overhead. This suggests that attention can provide a reliable structural prior at moderate model scales when isolated from textual confounders.</abstract>
<identifier type="citekey">qiu-shang-2026-rethinking</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.131/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>2886</start>
<end>2897</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Rethinking Cross-Modal Attention for Visual Token Pruning in Multimodal LVLMs
%A Qiu, Diego Weng
%A Shang, Lin
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F qiu-shang-2026-rethinking
%X Recent studies challenge the reliability of cross-modal attention mechanisms as an indicator for visual token pruning in Large Vision-Language Models (LVLMs). In this work, we demonstrate that such negative conclusions can be substantially influenced by an overlooked phenomenon termed task-bound coupling, wherein cross-modal attention patterns become entangled with task-specific prompts. Through controlled configurations that decouple visual inputs from downstream queries, we observe that attention distributions can be relatively consistent across the evaluated settings. Concurrently, we identify an empirical scaling phenomenon which we term Attention Fidelity Decay, where the downstream effectiveness of attention maps exhibits semantic dilution in higher-capacity models or advanced architectures. Motivated by these insights, we introduce TAP-LLaVA, a simple training-free framework that leverages localized task-agnostic attention for efficient, reusable visual token pruning across multi-turn sessions. Our approach achieves competitive downstream performance while substantially lowering computational overhead. This suggests that attention can provide a reliable structural prior at moderate model scales when isolated from textual confounders.
%U https://aclanthology.org/2026.aacl-long.131/
%P 2886-2897
Markdown (Informal)
[Rethinking Cross-Modal Attention for Visual Token Pruning in Multimodal LVLMs](https://aclanthology.org/2026.aacl-long.131/) (Qiu & Shang, AACL-IJCNLP 2026)
ACL
- Diego Weng Qiu and Lin Shang. 2026. Rethinking Cross-Modal Attention for Visual Token Pruning in Multimodal LVLMs. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 2886–2897, Hengqin, China. Association for Computational Linguistics.