@inproceedings{zeng-etal-2026-sparsity,
title = "Sparsity in Linear Attention Models: Bregman Testing Time Learning",
author = "Zeng, Wenqi and
Chen, YuAng and
Chen, Yuxuan and
Sun, Chumin and
Wen, Weihuang and
Liu, Yichuan and
Zhou, Li and
Wang, Tian and
Zhang, Fan and
Yao, Yuan and
Sun, Jie",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2246/",
pages = "48075--48094",
ISBN = "979-8-89176-462-0",
abstract = "Linear attention architectures enable efficient long-context modeling by compressing sequence history into a fixed-size memory state. However, dense key value updates can overload the state as the context grows, causing memory collisions and degraded retrieval. Existing methods rely on memory decay, rank reduction, or heuristic routing to reduce interference, but they largely preserve dense online updates and do not explicitly control the structural complexity of memory. We introduce BregNet, a linear sequence architecture that formulates associative memory updates as a regularized online regression problem. Based on \textit{Linearized Bregman Iteration} (LBI), BregNet decouples memory into a continuous dual accumulator and a structurally sparse primal state. The dual state records incoming information, while the sparse primal state keeps only important associations and removes noise through soft shrinkage. Experiments show that BregNet matches strong linear baselines in language modeling even with high memory sparsity. In this way, BregNet prevents the memory state from being overloaded by dense and interfering updates."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zeng-etal-2026-sparsity">
<titleInfo>
<title>Sparsity in Linear Attention Models: Bregman Testing Time Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wenqi</namePart>
<namePart type="family">Zeng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">YuAng</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuxuan</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chumin</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weihuang</namePart>
<namePart type="family">Wen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yichuan</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Li</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tian</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuan</namePart>
<namePart type="family">Yao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jie</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Linear attention architectures enable efficient long-context modeling by compressing sequence history into a fixed-size memory state. However, dense key value updates can overload the state as the context grows, causing memory collisions and degraded retrieval. Existing methods rely on memory decay, rank reduction, or heuristic routing to reduce interference, but they largely preserve dense online updates and do not explicitly control the structural complexity of memory. We introduce BregNet, a linear sequence architecture that formulates associative memory updates as a regularized online regression problem. Based on Linearized Bregman Iteration (LBI), BregNet decouples memory into a continuous dual accumulator and a structurally sparse primal state. The dual state records incoming information, while the sparse primal state keeps only important associations and removes noise through soft shrinkage. Experiments show that BregNet matches strong linear baselines in language modeling even with high memory sparsity. In this way, BregNet prevents the memory state from being overloaded by dense and interfering updates.</abstract>
<identifier type="citekey">zeng-etal-2026-sparsity</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2246/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>48075</start>
<end>48094</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Sparsity in Linear Attention Models: Bregman Testing Time Learning
%A Zeng, Wenqi
%A Chen, YuAng
%A Chen, Yuxuan
%A Sun, Chumin
%A Wen, Weihuang
%A Liu, Yichuan
%A Zhou, Li
%A Wang, Tian
%A Zhang, Fan
%A Yao, Yuan
%A Sun, Jie
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zeng-etal-2026-sparsity
%X Linear attention architectures enable efficient long-context modeling by compressing sequence history into a fixed-size memory state. However, dense key value updates can overload the state as the context grows, causing memory collisions and degraded retrieval. Existing methods rely on memory decay, rank reduction, or heuristic routing to reduce interference, but they largely preserve dense online updates and do not explicitly control the structural complexity of memory. We introduce BregNet, a linear sequence architecture that formulates associative memory updates as a regularized online regression problem. Based on Linearized Bregman Iteration (LBI), BregNet decouples memory into a continuous dual accumulator and a structurally sparse primal state. The dual state records incoming information, while the sparse primal state keeps only important associations and removes noise through soft shrinkage. Experiments show that BregNet matches strong linear baselines in language modeling even with high memory sparsity. In this way, BregNet prevents the memory state from being overloaded by dense and interfering updates.
%U https://aclanthology.org/2026.findings-emnlp.2246/
%P 48075-48094
Markdown (Informal)
[Sparsity in Linear Attention Models: Bregman Testing Time Learning](https://aclanthology.org/2026.findings-emnlp.2246/) (Zeng et al., Findings 2026)
ACL
- Wenqi Zeng, YuAng Chen, Yuxuan Chen, Chumin Sun, Weihuang Wen, Yichuan Liu, Li Zhou, Tian Wang, Fan Zhang, Yuan Yao, and Jie Sun. 2026. Sparsity in Linear Attention Models: Bregman Testing Time Learning. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 48075–48094, Budapest, Hungary. Association for Computational Linguistics.