@inproceedings{lee-etal-2026-iwp,
title = "{IWP}: Token Pruning as Implicit Weight Pruning in Large Vision Language Models",
author = "Lee, Dong-Jae and
Baek, Sunghyun and
Kim, Junmo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1778/",
pages = "38025--38047",
ISBN = "979-8-89176-462-0",
abstract = "Large Vision{--}Language Models (LVLMs) show impressive performance across image and video understanding tasks, yet their computational cost grows rapidly with the number of visual tokens. Existing token pruning methods address this by reducing the number of visual tokens, but overlook the internal mechanism of attention. In this paper, we propose a novel training-free token pruning framework grounded in the dual-form perspective of attention. We reformulate attention as an implicit linear layer whose weight matrix is the sum of rank-1 outer products, each generated by a single token{'}s key{--}value pair. Token pruning thus reduces to identifying an optimal subset of these rank-1 updates that retains the essential information of the original dual weight matrix. Extending this perspective to standard softmax attention in LVLMs, we derive a novel importance metric quantifying both a token{'}s information magnitude and information duplication. To efficiently select the subset with the proposed metric, we introduce Progressive Chunked Maximal Marginal Relevance. Extensive experiments demonstrate that our method achieves a better trade-off between performance and efficiency, while providing another perspective on existing pruning approaches."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lee-etal-2026-iwp">
<titleInfo>
<title>IWP: Token Pruning as Implicit Weight Pruning in Large Vision Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dong-Jae</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunghyun</namePart>
<namePart type="family">Baek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junmo</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large Vision–Language Models (LVLMs) show impressive performance across image and video understanding tasks, yet their computational cost grows rapidly with the number of visual tokens. Existing token pruning methods address this by reducing the number of visual tokens, but overlook the internal mechanism of attention. In this paper, we propose a novel training-free token pruning framework grounded in the dual-form perspective of attention. We reformulate attention as an implicit linear layer whose weight matrix is the sum of rank-1 outer products, each generated by a single token’s key–value pair. Token pruning thus reduces to identifying an optimal subset of these rank-1 updates that retains the essential information of the original dual weight matrix. Extending this perspective to standard softmax attention in LVLMs, we derive a novel importance metric quantifying both a token’s information magnitude and information duplication. To efficiently select the subset with the proposed metric, we introduce Progressive Chunked Maximal Marginal Relevance. Extensive experiments demonstrate that our method achieves a better trade-off between performance and efficiency, while providing another perspective on existing pruning approaches.</abstract>
<identifier type="citekey">lee-etal-2026-iwp</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1778/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>38025</start>
<end>38047</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T IWP: Token Pruning as Implicit Weight Pruning in Large Vision Language Models
%A Lee, Dong-Jae
%A Baek, Sunghyun
%A Kim, Junmo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F lee-etal-2026-iwp
%X Large Vision–Language Models (LVLMs) show impressive performance across image and video understanding tasks, yet their computational cost grows rapidly with the number of visual tokens. Existing token pruning methods address this by reducing the number of visual tokens, but overlook the internal mechanism of attention. In this paper, we propose a novel training-free token pruning framework grounded in the dual-form perspective of attention. We reformulate attention as an implicit linear layer whose weight matrix is the sum of rank-1 outer products, each generated by a single token’s key–value pair. Token pruning thus reduces to identifying an optimal subset of these rank-1 updates that retains the essential information of the original dual weight matrix. Extending this perspective to standard softmax attention in LVLMs, we derive a novel importance metric quantifying both a token’s information magnitude and information duplication. To efficiently select the subset with the proposed metric, we introduce Progressive Chunked Maximal Marginal Relevance. Extensive experiments demonstrate that our method achieves a better trade-off between performance and efficiency, while providing another perspective on existing pruning approaches.
%U https://aclanthology.org/2026.findings-emnlp.1778/
%P 38025-38047
Markdown (Informal)
[IWP: Token Pruning as Implicit Weight Pruning in Large Vision Language Models](https://aclanthology.org/2026.findings-emnlp.1778/) (Lee et al., Findings 2026)
ACL