@inproceedings{huang-etal-2026-nosa,
title = "{NOSA}: Native and Offloadable Sparse Attention",
author = "Huang, Yuxiang and
Wang, Pengjie and
Han, Jicheng and
Zhao, Weilin and
su, Zhou and
Sun, Ao and
Lyu, Hongya and
Zhao, Hengyu and
Wang, Yudong and
Xiao, Chaojun and
Han, Xu and
Liu, Zhiyuan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.279/",
pages = "6356--6381",
ISBN = "979-8-89176-459-0",
abstract = "Decoding throughput gains from larger inference batches are often limited by GPU memory, mainly due to the large size of KV cache. Training-free KV cache offloading mitigates this issue by storing redundant context on the CPU and fetching sparse KV subsets, but often harms long-generation quality due to training-inference mismatch of sparse patterns. Trainable sparse attention can improve quality, yet its unconstrained KV accesses are inefficient for offloading and may incur excessive CPU{--}GPU transfers. To this end, we propose NOSA, a trainable sparse attention mechanism designed for KV cache offloading. NOSA explicitly bounds CPU{--}GPU KV transfers, enabling low communication overhead and high decoding throughput. We further develop NOSI, an offloading inference system that realizes these efficiency gains. Experiments on {1,3,8}B LLMs show that NOSA outperforms KV cache offloading baselines on general, long-input, and long-generation tasks, while improving decoding throughput by up to 5.04$\times$, 1.92$\times$, and 1.83$\times$ over FullAttn, InfLLMv2, and ShadowKV, respectively."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="huang-etal-2026-nosa">
<titleInfo>
<title>NOSA: Native and Offloadable Sparse Attention</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuxiang</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pengjie</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jicheng</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weilin</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhou</namePart>
<namePart type="family">su</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ao</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongya</namePart>
<namePart type="family">Lyu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hengyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yudong</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chaojun</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xu</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyuan</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Decoding throughput gains from larger inference batches are often limited by GPU memory, mainly due to the large size of KV cache. Training-free KV cache offloading mitigates this issue by storing redundant context on the CPU and fetching sparse KV subsets, but often harms long-generation quality due to training-inference mismatch of sparse patterns. Trainable sparse attention can improve quality, yet its unconstrained KV accesses are inefficient for offloading and may incur excessive CPU–GPU transfers. To this end, we propose NOSA, a trainable sparse attention mechanism designed for KV cache offloading. NOSA explicitly bounds CPU–GPU KV transfers, enabling low communication overhead and high decoding throughput. We further develop NOSI, an offloading inference system that realizes these efficiency gains. Experiments on 1,3,8B LLMs show that NOSA outperforms KV cache offloading baselines on general, long-input, and long-generation tasks, while improving decoding throughput by up to 5.04\times, 1.92\times, and 1.83\times over FullAttn, InfLLMv2, and ShadowKV, respectively.</abstract>
<identifier type="citekey">huang-etal-2026-nosa</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.279/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>6356</start>
<end>6381</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T NOSA: Native and Offloadable Sparse Attention
%A Huang, Yuxiang
%A Wang, Pengjie
%A Han, Jicheng
%A Zhao, Weilin
%A su, Zhou
%A Sun, Ao
%A Lyu, Hongya
%A Zhao, Hengyu
%A Wang, Yudong
%A Xiao, Chaojun
%A Han, Xu
%A Liu, Zhiyuan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F huang-etal-2026-nosa
%X Decoding throughput gains from larger inference batches are often limited by GPU memory, mainly due to the large size of KV cache. Training-free KV cache offloading mitigates this issue by storing redundant context on the CPU and fetching sparse KV subsets, but often harms long-generation quality due to training-inference mismatch of sparse patterns. Trainable sparse attention can improve quality, yet its unconstrained KV accesses are inefficient for offloading and may incur excessive CPU–GPU transfers. To this end, we propose NOSA, a trainable sparse attention mechanism designed for KV cache offloading. NOSA explicitly bounds CPU–GPU KV transfers, enabling low communication overhead and high decoding throughput. We further develop NOSI, an offloading inference system that realizes these efficiency gains. Experiments on 1,3,8B LLMs show that NOSA outperforms KV cache offloading baselines on general, long-input, and long-generation tasks, while improving decoding throughput by up to 5.04\times, 1.92\times, and 1.83\times over FullAttn, InfLLMv2, and ShadowKV, respectively.
%U https://aclanthology.org/2026.emnlp-main.279/
%P 6356-6381
Markdown (Informal)
[NOSA: Native and Offloadable Sparse Attention](https://aclanthology.org/2026.emnlp-main.279/) (Huang et al., EMNLP 2026)
ACL
- Yuxiang Huang, Pengjie Wang, Jicheng Han, Weilin Zhao, Zhou su, Ao Sun, Hongya Lyu, Hengyu Zhao, Yudong Wang, Chaojun Xiao, Xu Han, and Zhiyuan Liu. 2026. NOSA: Native and Offloadable Sparse Attention. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 6356–6381, Budapest, Hungary. Association for Computational Linguistics.