@inproceedings{wang-etal-2026-anchorkv,
title = "{A}nchor{KV}: Training-Free {KV} Cache Compression via Joint Coverage Diversity and Semantic Importance Scoring",
author = "Wang, Mengbo and
Fan, Zhuochen and
Wang, Dayu and
Liu, Zirui and
Zong, Peiyuan and
Xie, Guorui and
Luan, Zeyu and
Li, Qing and
Jiang, Yong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2428/",
pages = "55127--55141",
ISBN = "979-8-89176-459-0",
abstract = "KV cache compression is critical for long-context LLM inference, yet existing training-free methods fall into a dilemma: attention-based eviction clusters retained tokens around semantically ``hot'' regions, creating context holes that degrade long-range reasoning, while diversity-based selection retains directionally broad but semantically irrelevant tokens. We identify the root cause: at prefill time, future decoding queries are unknown, making query-conditioned signals unreliable as a sole selection criterion. The retained KV set must instead provide broad directional coverage of the context{---}so that arbitrary future queries can find supporting keys{---}while filtering out tokens that expand the coverage span without contributing to generation. We propose AnchorKV, which realizes this principle through block-wise partitioning with uniform per-block budgets and anchor-driven cosine dissimilarity scoring for linear-time diversity estimation. A compression-rate adaptive hybrid score dynamically reweights diversity and importance. Experiments on LongBench across three model families show AnchorKV consistently outperforms existing training-free methods, with the largest margins at low retention ratios. A query-subsampled variant reduces prefill latency by up to 2.7{\texttimes} with minimal accuracy loss. We provide theoretical guarantees on approximation error and variance reduction."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-anchorkv">
<titleInfo>
<title>AnchorKV: Training-Free KV Cache Compression via Joint Coverage Diversity and Semantic Importance Scoring</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mengbo</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhuochen</namePart>
<namePart type="family">Fan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dayu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zirui</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peiyuan</namePart>
<namePart type="family">Zong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Guorui</namePart>
<namePart type="family">Xie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zeyu</namePart>
<namePart type="family">Luan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qing</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yong</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>KV cache compression is critical for long-context LLM inference, yet existing training-free methods fall into a dilemma: attention-based eviction clusters retained tokens around semantically “hot” regions, creating context holes that degrade long-range reasoning, while diversity-based selection retains directionally broad but semantically irrelevant tokens. We identify the root cause: at prefill time, future decoding queries are unknown, making query-conditioned signals unreliable as a sole selection criterion. The retained KV set must instead provide broad directional coverage of the context—so that arbitrary future queries can find supporting keys—while filtering out tokens that expand the coverage span without contributing to generation. We propose AnchorKV, which realizes this principle through block-wise partitioning with uniform per-block budgets and anchor-driven cosine dissimilarity scoring for linear-time diversity estimation. A compression-rate adaptive hybrid score dynamically reweights diversity and importance. Experiments on LongBench across three model families show AnchorKV consistently outperforms existing training-free methods, with the largest margins at low retention ratios. A query-subsampled variant reduces prefill latency by up to 2.7× with minimal accuracy loss. We provide theoretical guarantees on approximation error and variance reduction.</abstract>
<identifier type="citekey">wang-etal-2026-anchorkv</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2428/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>55127</start>
<end>55141</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T AnchorKV: Training-Free KV Cache Compression via Joint Coverage Diversity and Semantic Importance Scoring
%A Wang, Mengbo
%A Fan, Zhuochen
%A Wang, Dayu
%A Liu, Zirui
%A Zong, Peiyuan
%A Xie, Guorui
%A Luan, Zeyu
%A Li, Qing
%A Jiang, Yong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wang-etal-2026-anchorkv
%X KV cache compression is critical for long-context LLM inference, yet existing training-free methods fall into a dilemma: attention-based eviction clusters retained tokens around semantically “hot” regions, creating context holes that degrade long-range reasoning, while diversity-based selection retains directionally broad but semantically irrelevant tokens. We identify the root cause: at prefill time, future decoding queries are unknown, making query-conditioned signals unreliable as a sole selection criterion. The retained KV set must instead provide broad directional coverage of the context—so that arbitrary future queries can find supporting keys—while filtering out tokens that expand the coverage span without contributing to generation. We propose AnchorKV, which realizes this principle through block-wise partitioning with uniform per-block budgets and anchor-driven cosine dissimilarity scoring for linear-time diversity estimation. A compression-rate adaptive hybrid score dynamically reweights diversity and importance. Experiments on LongBench across three model families show AnchorKV consistently outperforms existing training-free methods, with the largest margins at low retention ratios. A query-subsampled variant reduces prefill latency by up to 2.7× with minimal accuracy loss. We provide theoretical guarantees on approximation error and variance reduction.
%U https://aclanthology.org/2026.emnlp-main.2428/
%P 55127-55141
Markdown (Informal)
[AnchorKV: Training-Free KV Cache Compression via Joint Coverage Diversity and Semantic Importance Scoring](https://aclanthology.org/2026.emnlp-main.2428/) (Wang et al., EMNLP 2026)
ACL
- Mengbo Wang, Zhuochen Fan, Dayu Wang, Zirui Liu, Peiyuan Zong, Guorui Xie, Zeyu Luan, Qing Li, and Yong Jiang. 2026. AnchorKV: Training-Free KV Cache Compression via Joint Coverage Diversity and Semantic Importance Scoring. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 55127–55141, Budapest, Hungary. Association for Computational Linguistics.