@inproceedings{chu-etal-2026-kwai,
title = "Kwai Summary Attention",
author = "Chu, Chenglong and
Peng, Hao and
Wang, Hui and
Cao, Jiangxia and
Ren, Lu and
Liang, Zhiyuan and
Wang, Ziqi and
Cheng, Hongtao and
Su, Yi and
Luo, Xinchen and
Liang, Jian and
Tang, Ruiming and
Zhou, Guorui and
Li, Han",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1953/",
pages = "41635--41655",
ISBN = "979-8-89176-462-0",
abstract = "Long-context capability has become increasingly critical for next-generation Large Language Models (LLMs). However, the quadratic time complexity of standard softmax attention introduces substantial computation and memory overhead as sequence length grows, limiting the scalability of LLMs in long-context scenarios. Beyond existing approaches such as linear recurrent architectures and KV-cache compression methods, we propose \textbf{Kwai Summary Attention (KSA)}, a novel attention mechanism that reduces sequence modeling cost by compressing historical contexts into learnable summary tokens. Specifically, summary tokens are periodically inserted into the input sequence and interleaved with text tokens, partitioning the sequence into distinct chunks while enabling text tokens and summary tokens to operate under different attention scopes. This design preserves dense local expressiveness through full attention among neighboring text tokens, while modeling long-range dependencies efficiently via summary tokens. Empirical results demonstrate that hybrid-KSA consistently outperforms existing hybrid attention variants in long-context modeling. Moreover, KSA is fully orthogonal to GQA and MLA during inference, and combining KSA with these methods achieves up to 8x KV-cache compression while preserving model performance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chu-etal-2026-kwai">
<titleInfo>
<title>Kwai Summary Attention</title>
</titleInfo>
<name type="personal">
<namePart type="given">Chenglong</namePart>
<namePart type="family">Chu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hui</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiangxia</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lu</namePart>
<namePart type="family">Ren</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyuan</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziqi</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongtao</namePart>
<namePart type="family">Cheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="family">Su</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinchen</namePart>
<namePart type="family">Luo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jian</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruiming</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Guorui</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Han</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Long-context capability has become increasingly critical for next-generation Large Language Models (LLMs). However, the quadratic time complexity of standard softmax attention introduces substantial computation and memory overhead as sequence length grows, limiting the scalability of LLMs in long-context scenarios. Beyond existing approaches such as linear recurrent architectures and KV-cache compression methods, we propose Kwai Summary Attention (KSA), a novel attention mechanism that reduces sequence modeling cost by compressing historical contexts into learnable summary tokens. Specifically, summary tokens are periodically inserted into the input sequence and interleaved with text tokens, partitioning the sequence into distinct chunks while enabling text tokens and summary tokens to operate under different attention scopes. This design preserves dense local expressiveness through full attention among neighboring text tokens, while modeling long-range dependencies efficiently via summary tokens. Empirical results demonstrate that hybrid-KSA consistently outperforms existing hybrid attention variants in long-context modeling. Moreover, KSA is fully orthogonal to GQA and MLA during inference, and combining KSA with these methods achieves up to 8x KV-cache compression while preserving model performance.</abstract>
<identifier type="citekey">chu-etal-2026-kwai</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1953/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>41635</start>
<end>41655</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Kwai Summary Attention
%A Chu, Chenglong
%A Peng, Hao
%A Wang, Hui
%A Cao, Jiangxia
%A Ren, Lu
%A Liang, Zhiyuan
%A Wang, Ziqi
%A Cheng, Hongtao
%A Su, Yi
%A Luo, Xinchen
%A Liang, Jian
%A Tang, Ruiming
%A Zhou, Guorui
%A Li, Han
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F chu-etal-2026-kwai
%X Long-context capability has become increasingly critical for next-generation Large Language Models (LLMs). However, the quadratic time complexity of standard softmax attention introduces substantial computation and memory overhead as sequence length grows, limiting the scalability of LLMs in long-context scenarios. Beyond existing approaches such as linear recurrent architectures and KV-cache compression methods, we propose Kwai Summary Attention (KSA), a novel attention mechanism that reduces sequence modeling cost by compressing historical contexts into learnable summary tokens. Specifically, summary tokens are periodically inserted into the input sequence and interleaved with text tokens, partitioning the sequence into distinct chunks while enabling text tokens and summary tokens to operate under different attention scopes. This design preserves dense local expressiveness through full attention among neighboring text tokens, while modeling long-range dependencies efficiently via summary tokens. Empirical results demonstrate that hybrid-KSA consistently outperforms existing hybrid attention variants in long-context modeling. Moreover, KSA is fully orthogonal to GQA and MLA during inference, and combining KSA with these methods achieves up to 8x KV-cache compression while preserving model performance.
%U https://aclanthology.org/2026.findings-emnlp.1953/
%P 41635-41655
Markdown (Informal)
[Kwai Summary Attention](https://aclanthology.org/2026.findings-emnlp.1953/) (Chu et al., Findings 2026)
ACL
- Chenglong Chu, Hao Peng, Hui Wang, Jiangxia Cao, Lu Ren, Zhiyuan Liang, Ziqi Wang, Hongtao Cheng, Yi Su, Xinchen Luo, Jian Liang, Ruiming Tang, Guorui Zhou, and Han Li. 2026. Kwai Summary Attention. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 41635–41655, Budapest, Hungary. Association for Computational Linguistics.