@inproceedings{sanyal-etal-2026-svd,
title = "{SVD}-Attention: Efficiently Analyzing Attention Blocks in Transformers",
author = "Sanyal, Debopam and
Chen, Hongjie and
Kuriyan, Riya Elizabeth and
Tak, Arth and
Kothandaraman, Divya and
Mehra, Akshay and
Kimball, Joshua and
Tumanov, Alexey",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1779/",
doi = "10.18653/2026.findings-emnlp.1779",
pages = "38048--38059",
ISBN = "979-8-89176-462-0",
abstract = "Pretrained transformers spend a large share of their inference compute inside self-attention, yet the per-head query-key interaction is structurally low-rank by construction. We ask a diagnostic question: across layers and architectures, which heads actually use the full rank budget on real data, and which do not? We propose SVD-Attention, a post-hoc method that decomposes the per-head interaction matrix $M_h$ and scores every singular mode by both its spectral magnitude and its activation variance across tokens on a small calibration set. The score keeps the modes the head can amplify and that real data actually activates, and discards the rest; the resulting factorization is exact for the retained modes and needs no finetuning. Across seven pretrained models and five task types, the diagnostic finds a task-dependent pattern: redundancy sits in the early to middle layers for sentiment and image classification and for generation, and in the late layers for dense prediction and extractive question answering. It is also predictive: truncating the layers it flags costs at most 1.75 F1, while truncating the same number of layers elsewhere in the same model costs up to 50 F1. SVD-Attention is an analysis tool rather than a compression method: acting on it reduces inference FLOPs by 0.8-1.5{\%} on its own and by 8-11{\%} when composed with weight pruning, with 0-4{\%} change in task quality. We release the code at https://github.com/riksanyal/SVD-Attention."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sanyal-etal-2026-svd">
<titleInfo>
<title>SVD-Attention: Efficiently Analyzing Attention Blocks in Transformers</title>
</titleInfo>
<name type="personal">
<namePart type="given">Debopam</namePart>
<namePart type="family">Sanyal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Riya</namePart>
<namePart type="given">Elizabeth</namePart>
<namePart type="family">Kuriyan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Arth</namePart>
<namePart type="family">Tak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Divya</namePart>
<namePart type="family">Kothandaraman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akshay</namePart>
<namePart type="family">Mehra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Joshua</namePart>
<namePart type="family">Kimball</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexey</namePart>
<namePart type="family">Tumanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Pretrained transformers spend a large share of their inference compute inside self-attention, yet the per-head query-key interaction is structurally low-rank by construction. We ask a diagnostic question: across layers and architectures, which heads actually use the full rank budget on real data, and which do not? We propose SVD-Attention, a post-hoc method that decomposes the per-head interaction matrix M_h and scores every singular mode by both its spectral magnitude and its activation variance across tokens on a small calibration set. The score keeps the modes the head can amplify and that real data actually activates, and discards the rest; the resulting factorization is exact for the retained modes and needs no finetuning. Across seven pretrained models and five task types, the diagnostic finds a task-dependent pattern: redundancy sits in the early to middle layers for sentiment and image classification and for generation, and in the late layers for dense prediction and extractive question answering. It is also predictive: truncating the layers it flags costs at most 1.75 F1, while truncating the same number of layers elsewhere in the same model costs up to 50 F1. SVD-Attention is an analysis tool rather than a compression method: acting on it reduces inference FLOPs by 0.8-1.5% on its own and by 8-11% when composed with weight pruning, with 0-4% change in task quality. We release the code at https://github.com/riksanyal/SVD-Attention.</abstract>
<identifier type="citekey">sanyal-etal-2026-svd</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1779</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1779/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>38048</start>
<end>38059</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SVD-Attention: Efficiently Analyzing Attention Blocks in Transformers
%A Sanyal, Debopam
%A Chen, Hongjie
%A Kuriyan, Riya Elizabeth
%A Tak, Arth
%A Kothandaraman, Divya
%A Mehra, Akshay
%A Kimball, Joshua
%A Tumanov, Alexey
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F sanyal-etal-2026-svd
%X Pretrained transformers spend a large share of their inference compute inside self-attention, yet the per-head query-key interaction is structurally low-rank by construction. We ask a diagnostic question: across layers and architectures, which heads actually use the full rank budget on real data, and which do not? We propose SVD-Attention, a post-hoc method that decomposes the per-head interaction matrix M_h and scores every singular mode by both its spectral magnitude and its activation variance across tokens on a small calibration set. The score keeps the modes the head can amplify and that real data actually activates, and discards the rest; the resulting factorization is exact for the retained modes and needs no finetuning. Across seven pretrained models and five task types, the diagnostic finds a task-dependent pattern: redundancy sits in the early to middle layers for sentiment and image classification and for generation, and in the late layers for dense prediction and extractive question answering. It is also predictive: truncating the layers it flags costs at most 1.75 F1, while truncating the same number of layers elsewhere in the same model costs up to 50 F1. SVD-Attention is an analysis tool rather than a compression method: acting on it reduces inference FLOPs by 0.8-1.5% on its own and by 8-11% when composed with weight pruning, with 0-4% change in task quality. We release the code at https://github.com/riksanyal/SVD-Attention.
%R 10.18653/2026.findings-emnlp.1779
%U https://aclanthology.org/2026.findings-emnlp.1779/
%U https://doi.org/10.18653/2026.findings-emnlp.1779
%P 38048-38059
Markdown (Informal)
[SVD-Attention: Efficiently Analyzing Attention Blocks in Transformers](https://aclanthology.org/2026.findings-emnlp.1779/) (Sanyal et al., Findings 2026)
ACL
- Debopam Sanyal, Hongjie Chen, Riya Elizabeth Kuriyan, Arth Tak, Divya Kothandaraman, Akshay Mehra, Joshua Kimball, and Alexey Tumanov. 2026. SVD-Attention: Efficiently Analyzing Attention Blocks in Transformers. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 38048–38059, Budapest, Hungary. Association for Computational Linguistics.