@inproceedings{gao-etal-2026-kv,
title = "{KV}-Retriever: Aligning {KV} Cache Eviction with Downstream Task Loss",
author = "Gao, Wenhao and
Shi, Binqin and
Jia, Wei and
Liu, Aiwei and
Zhang, Linhao and
Yuan, Caixia and
Wang, Xiaojie and
Xiao, Zhou",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2348/",
pages = "53329--53340",
ISBN = "979-8-89176-459-0",
abstract = "The prohibitive memory footprint of the Key-Value (KV) cache imposes a critical bottleneck for efficient long-context LLM serving. Current eviction methods, ranging from attention-based heuristics to recent learnable approaches, rely on scoring criteria decoupled from the downstream task performance that ultimately matters. To address this, we introduce \textbf{KV-Retriever}, a layer-wise learnable importance scoring module directly optimized by the end-to-end downstream task loss. We demonstrate that aligning the scoring objective with the actual task signal{---}rather than intermediate proxies such as attention statistics or oracle labels{---}yields a fundamentally more accurate notion of token importance. Functioning as a lightweight, plug-and-play module with only $\sim$0.01{\%} additional parameters on a frozen 8B backbone, KV-Retriever is compatible with diverse host eviction frameworks. Extensive experiments show that, for each backbone, one trained retriever can be reused unchanged across multiple KV cache compression architectures and delivers consistent improvements on LongBench, LongBench-v2, and RULER. KV-Retriever requires one-time supervised training with golden answers; ``plug-and-play'' refers to deployment into otherwise unchanged host eviction frameworks, not to training-free adaptation. Our code is available at \url{https://github.com/Taylor-Gavel/kv_retriever}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="gao-etal-2026-kv">
<titleInfo>
<title>KV-Retriever: Aligning KV Cache Eviction with Downstream Task Loss</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wenhao</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Binqin</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Jia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aiwei</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Linhao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Caixia</namePart>
<namePart type="family">Yuan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaojie</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhou</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>The prohibitive memory footprint of the Key-Value (KV) cache imposes a critical bottleneck for efficient long-context LLM serving. Current eviction methods, ranging from attention-based heuristics to recent learnable approaches, rely on scoring criteria decoupled from the downstream task performance that ultimately matters. To address this, we introduce KV-Retriever, a layer-wise learnable importance scoring module directly optimized by the end-to-end downstream task loss. We demonstrate that aligning the scoring objective with the actual task signal—rather than intermediate proxies such as attention statistics or oracle labels—yields a fundamentally more accurate notion of token importance. Functioning as a lightweight, plug-and-play module with only \sim0.01% additional parameters on a frozen 8B backbone, KV-Retriever is compatible with diverse host eviction frameworks. Extensive experiments show that, for each backbone, one trained retriever can be reused unchanged across multiple KV cache compression architectures and delivers consistent improvements on LongBench, LongBench-v2, and RULER. KV-Retriever requires one-time supervised training with golden answers; “plug-and-play” refers to deployment into otherwise unchanged host eviction frameworks, not to training-free adaptation. Our code is available at https://github.com/Taylor-Gavel/kv_retriever.</abstract>
<identifier type="citekey">gao-etal-2026-kv</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2348/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>53329</start>
<end>53340</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T KV-Retriever: Aligning KV Cache Eviction with Downstream Task Loss
%A Gao, Wenhao
%A Shi, Binqin
%A Jia, Wei
%A Liu, Aiwei
%A Zhang, Linhao
%A Yuan, Caixia
%A Wang, Xiaojie
%A Xiao, Zhou
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F gao-etal-2026-kv
%X The prohibitive memory footprint of the Key-Value (KV) cache imposes a critical bottleneck for efficient long-context LLM serving. Current eviction methods, ranging from attention-based heuristics to recent learnable approaches, rely on scoring criteria decoupled from the downstream task performance that ultimately matters. To address this, we introduce KV-Retriever, a layer-wise learnable importance scoring module directly optimized by the end-to-end downstream task loss. We demonstrate that aligning the scoring objective with the actual task signal—rather than intermediate proxies such as attention statistics or oracle labels—yields a fundamentally more accurate notion of token importance. Functioning as a lightweight, plug-and-play module with only \sim0.01% additional parameters on a frozen 8B backbone, KV-Retriever is compatible with diverse host eviction frameworks. Extensive experiments show that, for each backbone, one trained retriever can be reused unchanged across multiple KV cache compression architectures and delivers consistent improvements on LongBench, LongBench-v2, and RULER. KV-Retriever requires one-time supervised training with golden answers; “plug-and-play” refers to deployment into otherwise unchanged host eviction frameworks, not to training-free adaptation. Our code is available at https://github.com/Taylor-Gavel/kv_retriever.
%U https://aclanthology.org/2026.emnlp-main.2348/
%P 53329-53340
Markdown (Informal)
[KV-Retriever: Aligning KV Cache Eviction with Downstream Task Loss](https://aclanthology.org/2026.emnlp-main.2348/) (Gao et al., EMNLP 2026)
ACL
- Wenhao Gao, Binqin Shi, Wei Jia, Aiwei Liu, Linhao Zhang, Caixia Yuan, Xiaojie Wang, and Zhou Xiao. 2026. KV-Retriever: Aligning KV Cache Eviction with Downstream Task Loss. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 53329–53340, Budapest, Hungary. Association for Computational Linguistics.