@inproceedings{shao-etal-2026-hachiware,
title = "{H}achi{W}are: One-Shot Selective {SVD} in a Dual-Branch Framework for {KV} Cache Compression",
author = "Shao, Hsiao-Ting and
Lin, Xiaolin and
Lu, Yu-Chen and
Zhang, Grace Li and
Wu, Kai-Chiang and
Huang, Ning-Chi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1520/",
pages = "34701--34725",
ISBN = "979-8-89176-459-0",
abstract = "Long-context inference is increasingly limited by the memory and access cost of the key-value (KV) cache. Existing KV cache compression methods face a fundamental trade-off between efficiency and information preservation. Eviction-based methods are efficient but may discard critical tokens, whereas low-rank-based methods preserve all tokens at the cost of substantial computational overhead. We propose HachiWare, a dual-branch framework that applies one-shot selective SVD to semantic-critical cache states while handling less critical states with lightweight eviction. It stores these states in a Semantic-Critical Branch (SCB) using low-rank factors, while assigning the remaining states to an Eviction Branch (EB) for cheaper eviction. During decoding, bounded active-cache refresh selectively reconstructs only active SCB entries, avoiding reconstruction of the full SCB cache. On the RULER benchmark with Llama-3.1-8B-Instruct, HachiWare incurs only a 3{\%} accuracy drop compared with the full KV cache baseline at a KV cache compression ratio of up to 16x, while improving accuracy by 5{\%} over existing compression methods. For decoding efficiency, HachiWare achieves up to 9.44x and 16.50x throughput speedups over Full KV at 64K and 128K context lengths. These results highlight selective allocation of reconstructability as an effective way to improve the accuracy-efficiency trade-off of KV cache compression."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="shao-etal-2026-hachiware">
<titleInfo>
<title>HachiWare: One-Shot Selective SVD in a Dual-Branch Framework for KV Cache Compression</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hsiao-Ting</namePart>
<namePart type="family">Shao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaolin</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yu-Chen</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Grace</namePart>
<namePart type="given">Li</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kai-Chiang</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ning-Chi</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Long-context inference is increasingly limited by the memory and access cost of the key-value (KV) cache. Existing KV cache compression methods face a fundamental trade-off between efficiency and information preservation. Eviction-based methods are efficient but may discard critical tokens, whereas low-rank-based methods preserve all tokens at the cost of substantial computational overhead. We propose HachiWare, a dual-branch framework that applies one-shot selective SVD to semantic-critical cache states while handling less critical states with lightweight eviction. It stores these states in a Semantic-Critical Branch (SCB) using low-rank factors, while assigning the remaining states to an Eviction Branch (EB) for cheaper eviction. During decoding, bounded active-cache refresh selectively reconstructs only active SCB entries, avoiding reconstruction of the full SCB cache. On the RULER benchmark with Llama-3.1-8B-Instruct, HachiWare incurs only a 3% accuracy drop compared with the full KV cache baseline at a KV cache compression ratio of up to 16x, while improving accuracy by 5% over existing compression methods. For decoding efficiency, HachiWare achieves up to 9.44x and 16.50x throughput speedups over Full KV at 64K and 128K context lengths. These results highlight selective allocation of reconstructability as an effective way to improve the accuracy-efficiency trade-off of KV cache compression.</abstract>
<identifier type="citekey">shao-etal-2026-hachiware</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1520/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>34701</start>
<end>34725</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T HachiWare: One-Shot Selective SVD in a Dual-Branch Framework for KV Cache Compression
%A Shao, Hsiao-Ting
%A Lin, Xiaolin
%A Lu, Yu-Chen
%A Zhang, Grace Li
%A Wu, Kai-Chiang
%A Huang, Ning-Chi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F shao-etal-2026-hachiware
%X Long-context inference is increasingly limited by the memory and access cost of the key-value (KV) cache. Existing KV cache compression methods face a fundamental trade-off between efficiency and information preservation. Eviction-based methods are efficient but may discard critical tokens, whereas low-rank-based methods preserve all tokens at the cost of substantial computational overhead. We propose HachiWare, a dual-branch framework that applies one-shot selective SVD to semantic-critical cache states while handling less critical states with lightweight eviction. It stores these states in a Semantic-Critical Branch (SCB) using low-rank factors, while assigning the remaining states to an Eviction Branch (EB) for cheaper eviction. During decoding, bounded active-cache refresh selectively reconstructs only active SCB entries, avoiding reconstruction of the full SCB cache. On the RULER benchmark with Llama-3.1-8B-Instruct, HachiWare incurs only a 3% accuracy drop compared with the full KV cache baseline at a KV cache compression ratio of up to 16x, while improving accuracy by 5% over existing compression methods. For decoding efficiency, HachiWare achieves up to 9.44x and 16.50x throughput speedups over Full KV at 64K and 128K context lengths. These results highlight selective allocation of reconstructability as an effective way to improve the accuracy-efficiency trade-off of KV cache compression.
%U https://aclanthology.org/2026.emnlp-main.1520/
%P 34701-34725
Markdown (Informal)
[HachiWare: One-Shot Selective SVD in a Dual-Branch Framework for KV Cache Compression](https://aclanthology.org/2026.emnlp-main.1520/) (Shao et al., EMNLP 2026)
ACL