@inproceedings{chakrabarti-etal-2026-ultraquant,
title = "{U}ltra{Q}uant: 4-bit {KV} Caching for Context-Heavy Agents",
author = "Chakrabarti, Inesh and
Limpus, David and
Rana, Aditi Ghai and
Bao, Bowen and
Tiwari, Spandan and
Crepaldi, Thiago and
Sirasao, Ashish",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.161/",
pages = "2732--2750",
ISBN = "979-8-89176-460-6",
abstract = "Context-heavy agents place substantial pressure on the key{--}value (KV) cache: long prefixes are reused across many short turns, while concurrency determines whether the serving system can keep GPUs utilized. We study 4-bit KV-cache compression for this setting, using TurboQuant-style rotation and codebook quantization as a quality anchor and vLLM FP8 KV caching as the deployment anchor. We report three contributions. First, we frame 4-bit KV caching around multi-round agent workloads where task quality, cache residency, and serving throughput must be measured jointly. Second, we describe the practical design choices needed to make the 4-bit path robust, including asymmetric K/V treatment, Walsh{--}Hadamard rotation, QJL removal, and block-scale variants. Third, we present serving optimizations on AMD GPUs, including optimized decode-attention kernels and UltraQuant, an FP4 approximation path that uses FP8 queries, FP4 KV tensors, UE8M0 group scales, and native scaled-MFMA support on CDNA4. On an adaptive-SLO replay of production Claude Code traces, UltraQuant sustains $2.71\times$ (MiniMax-M2.5) and $4.38\times$ (Qwen3-235B) the qualified-request throughput of the BF16 baseline, matching or exceeding hardware FP8 KV while using half the KV bytes. UltraQuant delivers its largest gains in long-context, high-concurrency, memory-constrained serving regimes."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chakrabarti-etal-2026-ultraquant">
<titleInfo>
<title>UltraQuant: 4-bit KV Caching for Context-Heavy Agents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Inesh</namePart>
<namePart type="family">Chakrabarti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Limpus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aditi</namePart>
<namePart type="given">Ghai</namePart>
<namePart type="family">Rana</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bowen</namePart>
<namePart type="family">Bao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Spandan</namePart>
<namePart type="family">Tiwari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thiago</namePart>
<namePart type="family">Crepaldi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ashish</namePart>
<namePart type="family">Sirasao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Context-heavy agents place substantial pressure on the key–value (KV) cache: long prefixes are reused across many short turns, while concurrency determines whether the serving system can keep GPUs utilized. We study 4-bit KV-cache compression for this setting, using TurboQuant-style rotation and codebook quantization as a quality anchor and vLLM FP8 KV caching as the deployment anchor. We report three contributions. First, we frame 4-bit KV caching around multi-round agent workloads where task quality, cache residency, and serving throughput must be measured jointly. Second, we describe the practical design choices needed to make the 4-bit path robust, including asymmetric K/V treatment, Walsh–Hadamard rotation, QJL removal, and block-scale variants. Third, we present serving optimizations on AMD GPUs, including optimized decode-attention kernels and UltraQuant, an FP4 approximation path that uses FP8 queries, FP4 KV tensors, UE8M0 group scales, and native scaled-MFMA support on CDNA4. On an adaptive-SLO replay of production Claude Code traces, UltraQuant sustains 2.71\times (MiniMax-M2.5) and 4.38\times (Qwen3-235B) the qualified-request throughput of the BF16 baseline, matching or exceeding hardware FP8 KV while using half the KV bytes. UltraQuant delivers its largest gains in long-context, high-concurrency, memory-constrained serving regimes.</abstract>
<identifier type="citekey">chakrabarti-etal-2026-ultraquant</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.161/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>2732</start>
<end>2750</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T UltraQuant: 4-bit KV Caching for Context-Heavy Agents
%A Chakrabarti, Inesh
%A Limpus, David
%A Rana, Aditi Ghai
%A Bao, Bowen
%A Tiwari, Spandan
%A Crepaldi, Thiago
%A Sirasao, Ashish
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F chakrabarti-etal-2026-ultraquant
%X Context-heavy agents place substantial pressure on the key–value (KV) cache: long prefixes are reused across many short turns, while concurrency determines whether the serving system can keep GPUs utilized. We study 4-bit KV-cache compression for this setting, using TurboQuant-style rotation and codebook quantization as a quality anchor and vLLM FP8 KV caching as the deployment anchor. We report three contributions. First, we frame 4-bit KV caching around multi-round agent workloads where task quality, cache residency, and serving throughput must be measured jointly. Second, we describe the practical design choices needed to make the 4-bit path robust, including asymmetric K/V treatment, Walsh–Hadamard rotation, QJL removal, and block-scale variants. Third, we present serving optimizations on AMD GPUs, including optimized decode-attention kernels and UltraQuant, an FP4 approximation path that uses FP8 queries, FP4 KV tensors, UE8M0 group scales, and native scaled-MFMA support on CDNA4. On an adaptive-SLO replay of production Claude Code traces, UltraQuant sustains 2.71\times (MiniMax-M2.5) and 4.38\times (Qwen3-235B) the qualified-request throughput of the BF16 baseline, matching or exceeding hardware FP8 KV while using half the KV bytes. UltraQuant delivers its largest gains in long-context, high-concurrency, memory-constrained serving regimes.
%U https://aclanthology.org/2026.emnlp-industry.161/
%P 2732-2750
Markdown (Informal)
[UltraQuant: 4-bit KV Caching for Context-Heavy Agents](https://aclanthology.org/2026.emnlp-industry.161/) (Chakrabarti et al., EMNLP 2026)
ACL
- Inesh Chakrabarti, David Limpus, Aditi Ghai Rana, Bowen Bao, Spandan Tiwari, Thiago Crepaldi, and Ashish Sirasao. 2026. UltraQuant: 4-bit KV Caching for Context-Heavy Agents. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track, pages 2732–2750, Budapest, Hungary. Association for Computational Linguistics.