@inproceedings{wang-etal-2026-sampkv,
title = "{S}amp{KV}: Saliency-Aware Mixed-Precision {KV} Cache for Efficient Diffusion Language Model Inference",
author = "Wang, Zhenrui and
Chang, Haodong and
Sui, Yang and
Liang, Rongjian and
Yuan, Bo and
Hu, Jiang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1968/",
pages = "44808--44821",
ISBN = "979-8-89176-459-0",
abstract = "Diffusion language models (dLLMs) have recently shown strong potential in various tasks with competitive decoding efficiency, but their bidirectional attention incurs quadratic compute and memory complexity. Many prior works rely on KV cache optimizations to accelerate inference. However, the full-sequence BF16 KV cache remains the dominant memory bottleneck. We propose SampKV, a Saliency-aware Mixed-Precision KV cache for dLLMs that stores most history in 2 bits while promoting a small budget of attention-salient tokens to high precision. A customized Triton kernel operates directly on the mixed-precision cache and avoids materializing full-precision KV tensors, alleviating the storage and traffic bottleneck. Experiments on LLaDA and Dream models show that the memory bottleneck is substantially alleviated: up to 54{\%} peak memory reduction at matched workload, or up to 2.06x throughput improvement at matched peak memory compared to the BF16 baseline, while maintaining the accuracy."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-sampkv">
<titleInfo>
<title>SampKV: Saliency-Aware Mixed-Precision KV Cache for Efficient Diffusion Language Model Inference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhenrui</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haodong</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yang</namePart>
<namePart type="family">Sui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rongjian</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bo</namePart>
<namePart type="family">Yuan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiang</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Diffusion language models (dLLMs) have recently shown strong potential in various tasks with competitive decoding efficiency, but their bidirectional attention incurs quadratic compute and memory complexity. Many prior works rely on KV cache optimizations to accelerate inference. However, the full-sequence BF16 KV cache remains the dominant memory bottleneck. We propose SampKV, a Saliency-aware Mixed-Precision KV cache for dLLMs that stores most history in 2 bits while promoting a small budget of attention-salient tokens to high precision. A customized Triton kernel operates directly on the mixed-precision cache and avoids materializing full-precision KV tensors, alleviating the storage and traffic bottleneck. Experiments on LLaDA and Dream models show that the memory bottleneck is substantially alleviated: up to 54% peak memory reduction at matched workload, or up to 2.06x throughput improvement at matched peak memory compared to the BF16 baseline, while maintaining the accuracy.</abstract>
<identifier type="citekey">wang-etal-2026-sampkv</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1968/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44808</start>
<end>44821</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SampKV: Saliency-Aware Mixed-Precision KV Cache for Efficient Diffusion Language Model Inference
%A Wang, Zhenrui
%A Chang, Haodong
%A Sui, Yang
%A Liang, Rongjian
%A Yuan, Bo
%A Hu, Jiang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wang-etal-2026-sampkv
%X Diffusion language models (dLLMs) have recently shown strong potential in various tasks with competitive decoding efficiency, but their bidirectional attention incurs quadratic compute and memory complexity. Many prior works rely on KV cache optimizations to accelerate inference. However, the full-sequence BF16 KV cache remains the dominant memory bottleneck. We propose SampKV, a Saliency-aware Mixed-Precision KV cache for dLLMs that stores most history in 2 bits while promoting a small budget of attention-salient tokens to high precision. A customized Triton kernel operates directly on the mixed-precision cache and avoids materializing full-precision KV tensors, alleviating the storage and traffic bottleneck. Experiments on LLaDA and Dream models show that the memory bottleneck is substantially alleviated: up to 54% peak memory reduction at matched workload, or up to 2.06x throughput improvement at matched peak memory compared to the BF16 baseline, while maintaining the accuracy.
%U https://aclanthology.org/2026.emnlp-main.1968/
%P 44808-44821
Markdown (Informal)
[SampKV: Saliency-Aware Mixed-Precision KV Cache for Efficient Diffusion Language Model Inference](https://aclanthology.org/2026.emnlp-main.1968/) (Wang et al., EMNLP 2026)
ACL