@inproceedings{hu-etal-2026-seer,
title = "Seer: Efficient {KV} Cache Management for {LLM} Acceleration via Mitigating Semantic Search Inefficiency",
author = "Hu, Yiwei and
Liu, Fangxin and
Liang, Yue and
Wang, Xuan and
Li, Cheng and
Li, Gezi and
Jiang, Li and
Guan, Haibing",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1852/",
pages = "42117--42130",
ISBN = "979-8-89176-459-0",
abstract = "Large language models (LLMs) have made significant progress in various NLP tasks, especially with the rise of long-context models that handle extended inputs. However, the growing size of the key-value (KV) cache in Transformer architectures imposes heavy memory demands during decoding, becoming a major bottleneck. Existing sparse attention methods aim to address this issue but face two main limitations: (1) they introduce high overhead when retrieving important tokens for attention, and (2) they often fail to significantly reduce memory usage in practice. This paper presents \textbf{Seer}, a simple yet effective system for fast and accurate LLM decoding on long-context sequences. Seer leverages semantic space representations to manage the KV cache more efficiently. It dynamically estimates cluster-level importance to reduce both computation and memory access, while preserving output quality. Through offline training, Seer learns semantic centroids to enable fast token importance estimation. To further improve efficiency, it integrates product quantization with sparse attention for KV cache compression and uses an asymmetric distance lookup table to reduce computational cost. Evaluation on a diverse set of LLMs and tasks shows that Seer reduces theoretical attention bandwidth by $40\times$ with minimal accuracy loss. Seer achieves a $5.57\times$ attention speedup and a $2\times$ End-To-End speedup over full attention at 32K sequence length. Our code is available at https://github.com/Acaciasama/Seer."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hu-etal-2026-seer">
<titleInfo>
<title>Seer: Efficient KV Cache Management for LLM Acceleration via Mitigating Semantic Search Inefficiency</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yiwei</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fangxin</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cheng</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gezi</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Li</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haibing</namePart>
<namePart type="family">Guan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) have made significant progress in various NLP tasks, especially with the rise of long-context models that handle extended inputs. However, the growing size of the key-value (KV) cache in Transformer architectures imposes heavy memory demands during decoding, becoming a major bottleneck. Existing sparse attention methods aim to address this issue but face two main limitations: (1) they introduce high overhead when retrieving important tokens for attention, and (2) they often fail to significantly reduce memory usage in practice. This paper presents Seer, a simple yet effective system for fast and accurate LLM decoding on long-context sequences. Seer leverages semantic space representations to manage the KV cache more efficiently. It dynamically estimates cluster-level importance to reduce both computation and memory access, while preserving output quality. Through offline training, Seer learns semantic centroids to enable fast token importance estimation. To further improve efficiency, it integrates product quantization with sparse attention for KV cache compression and uses an asymmetric distance lookup table to reduce computational cost. Evaluation on a diverse set of LLMs and tasks shows that Seer reduces theoretical attention bandwidth by 40\times with minimal accuracy loss. Seer achieves a 5.57\times attention speedup and a 2\times End-To-End speedup over full attention at 32K sequence length. Our code is available at https://github.com/Acaciasama/Seer.</abstract>
<identifier type="citekey">hu-etal-2026-seer</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1852/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>42117</start>
<end>42130</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Seer: Efficient KV Cache Management for LLM Acceleration via Mitigating Semantic Search Inefficiency
%A Hu, Yiwei
%A Liu, Fangxin
%A Liang, Yue
%A Wang, Xuan
%A Li, Cheng
%A Li, Gezi
%A Jiang, Li
%A Guan, Haibing
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F hu-etal-2026-seer
%X Large language models (LLMs) have made significant progress in various NLP tasks, especially with the rise of long-context models that handle extended inputs. However, the growing size of the key-value (KV) cache in Transformer architectures imposes heavy memory demands during decoding, becoming a major bottleneck. Existing sparse attention methods aim to address this issue but face two main limitations: (1) they introduce high overhead when retrieving important tokens for attention, and (2) they often fail to significantly reduce memory usage in practice. This paper presents Seer, a simple yet effective system for fast and accurate LLM decoding on long-context sequences. Seer leverages semantic space representations to manage the KV cache more efficiently. It dynamically estimates cluster-level importance to reduce both computation and memory access, while preserving output quality. Through offline training, Seer learns semantic centroids to enable fast token importance estimation. To further improve efficiency, it integrates product quantization with sparse attention for KV cache compression and uses an asymmetric distance lookup table to reduce computational cost. Evaluation on a diverse set of LLMs and tasks shows that Seer reduces theoretical attention bandwidth by 40\times with minimal accuracy loss. Seer achieves a 5.57\times attention speedup and a 2\times End-To-End speedup over full attention at 32K sequence length. Our code is available at https://github.com/Acaciasama/Seer.
%U https://aclanthology.org/2026.emnlp-main.1852/
%P 42117-42130
Markdown (Informal)
[Seer: Efficient KV Cache Management for LLM Acceleration via Mitigating Semantic Search Inefficiency](https://aclanthology.org/2026.emnlp-main.1852/) (Hu et al., EMNLP 2026)
ACL
- Yiwei Hu, Fangxin Liu, Yue Liang, Xuan Wang, Cheng Li, Gezi Li, Li Jiang, and Haibing Guan. 2026. Seer: Efficient KV Cache Management for LLM Acceleration via Mitigating Semantic Search Inefficiency. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 42117–42130, Budapest, Hungary. Association for Computational Linguistics.