@inproceedings{liu-etal-2026-agentkv,
title = "{A}gent{KV}: Phase-Aware {KV} Eviction for Agentic {LLM}s",
author = "Liu, Taowen Tony and
Wong, Jeffrey T. H. and
Xiao, Can and
Yang, Bowen and
Chen, Hao Mark and
Zhao, Yiren",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2514/",
pages = "57169--57184",
ISBN = "979-8-89176-459-0",
abstract = "Agentic serving can consume orders of magnitude more tokens than chatbot workloads, stressing both KV-cache capacity and decode-time bandwidth. Most KV-eviction methods score cached keys against representative queries drawn from the most recent tokens, assuming future attention resembles recent attention. We show that agentic generation violates this assumption: future queries form a mixture over think, act, tool, and others phases, and principal-angle analysis shows these components occupy measurably different query subspaces, so recency representatives systematically undervalue keys that upcoming phases will need. We propose AgentKV, which maintains a small query buffer per phase and scores cached keys against their union. We further implement AgentKV in a persistent multi-turn serving path that carries compressed KV state across turns and compacts retained KV pages online. Across two models, six task domains, and three KV budgets each, AgentKV improves task score by 5.5 points on average over R-KV and 5.3 over Tri-attention. Relative to upstream full-KV SGLang, AgentKV improves output-token throughput by up to $1.80{\times}$. Code: https://github.com/LiuTaowen-Tony/agentkv."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-agentkv">
<titleInfo>
<title>AgentKV: Phase-Aware KV Eviction for Agentic LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Taowen</namePart>
<namePart type="given">Tony</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jeffrey</namePart>
<namePart type="given">T</namePart>
<namePart type="given">H</namePart>
<namePart type="family">Wong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Can</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bowen</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="given">Mark</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiren</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Agentic serving can consume orders of magnitude more tokens than chatbot workloads, stressing both KV-cache capacity and decode-time bandwidth. Most KV-eviction methods score cached keys against representative queries drawn from the most recent tokens, assuming future attention resembles recent attention. We show that agentic generation violates this assumption: future queries form a mixture over think, act, tool, and others phases, and principal-angle analysis shows these components occupy measurably different query subspaces, so recency representatives systematically undervalue keys that upcoming phases will need. We propose AgentKV, which maintains a small query buffer per phase and scores cached keys against their union. We further implement AgentKV in a persistent multi-turn serving path that carries compressed KV state across turns and compacts retained KV pages online. Across two models, six task domains, and three KV budgets each, AgentKV improves task score by 5.5 points on average over R-KV and 5.3 over Tri-attention. Relative to upstream full-KV SGLang, AgentKV improves output-token throughput by up to 1.80\times. Code: https://github.com/LiuTaowen-Tony/agentkv.</abstract>
<identifier type="citekey">liu-etal-2026-agentkv</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2514/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>57169</start>
<end>57184</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T AgentKV: Phase-Aware KV Eviction for Agentic LLMs
%A Liu, Taowen Tony
%A Wong, Jeffrey T. H.
%A Xiao, Can
%A Yang, Bowen
%A Chen, Hao Mark
%A Zhao, Yiren
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liu-etal-2026-agentkv
%X Agentic serving can consume orders of magnitude more tokens than chatbot workloads, stressing both KV-cache capacity and decode-time bandwidth. Most KV-eviction methods score cached keys against representative queries drawn from the most recent tokens, assuming future attention resembles recent attention. We show that agentic generation violates this assumption: future queries form a mixture over think, act, tool, and others phases, and principal-angle analysis shows these components occupy measurably different query subspaces, so recency representatives systematically undervalue keys that upcoming phases will need. We propose AgentKV, which maintains a small query buffer per phase and scores cached keys against their union. We further implement AgentKV in a persistent multi-turn serving path that carries compressed KV state across turns and compacts retained KV pages online. Across two models, six task domains, and three KV budgets each, AgentKV improves task score by 5.5 points on average over R-KV and 5.3 over Tri-attention. Relative to upstream full-KV SGLang, AgentKV improves output-token throughput by up to 1.80\times. Code: https://github.com/LiuTaowen-Tony/agentkv.
%U https://aclanthology.org/2026.emnlp-main.2514/
%P 57169-57184
Markdown (Informal)
[AgentKV: Phase-Aware KV Eviction for Agentic LLMs](https://aclanthology.org/2026.emnlp-main.2514/) (Liu et al., EMNLP 2026)
ACL
- Taowen Tony Liu, Jeffrey T. H. Wong, Can Xiao, Bowen Yang, Hao Mark Chen, and Yiren Zhao. 2026. AgentKV: Phase-Aware KV Eviction for Agentic LLMs. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 57169–57184, Budapest, Hungary. Association for Computational Linguistics.