@inproceedings{zhan-etal-2026-delta,
title = "{DELTA}: Decoupling Latent Heterogeneity in Asymmetric Low-Rank Compression",
author = "Zhan, Jialin and
Liu, Fangxin and
Wang, Junjie and
Yang, Ning and
Jiang, Li and
Guan, Haibing",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2287/",
pages = "52046--52057",
ISBN = "979-8-89176-459-0",
abstract = "Singular Value Decomposition (SVD) is a promising post-training compression technique for large language models (LLMs), as it is training-free, hardware-friendly, and preserves regular dense computation. However, existing SVD-based methods suffer from severe accuracy degradation under aggressive compression. This is mainly because hard rank truncation discards useful singular directions, while the two resulting SVD factors are usually compressed symmetrically despite their different roles, shapes, and quantization sensitivities. In this paper, we propose DELTA, an asymmetric low-rank compression method that decouples the latent heterogeneity between the two SVD factors. Under a fixed memory budget, the proposed method jointly determines the retained rank, factor-wise quantization bit-widths, and quantization directions, allowing more effective singular directions to be preserved with low-bit representations. To efficiently select compression configurations, we use an activation-aware reconstruction error as a layer-wise proxy objective. Experiments on multiple LLMs and datasets show that DELTA consistently outperforms existing post-training SVD-based compression methods, especially under high compression ratios. Compared with the strongest SVD-based baseline at the same compression ratio, DELTA reduces perplexity on WikiText-2 and C4 by up to 68.2{\%} and 53.2{\%}, respectively, and improves average zero-shot accuracy by up to 11 percentage points. These results demonstrate that asymmetric factor-aware compression can effectively mitigate the accuracy degradation of SVD-based LLM compression at high compression ratios. Code is available at \url{https://github.com/BlacksheepJane/DELTA}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhan-etal-2026-delta">
<titleInfo>
<title>DELTA: Decoupling Latent Heterogeneity in Asymmetric Low-Rank Compression</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jialin</namePart>
<namePart type="family">Zhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fangxin</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junjie</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ning</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Li</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haibing</namePart>
<namePart type="family">Guan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Singular Value Decomposition (SVD) is a promising post-training compression technique for large language models (LLMs), as it is training-free, hardware-friendly, and preserves regular dense computation. However, existing SVD-based methods suffer from severe accuracy degradation under aggressive compression. This is mainly because hard rank truncation discards useful singular directions, while the two resulting SVD factors are usually compressed symmetrically despite their different roles, shapes, and quantization sensitivities. In this paper, we propose DELTA, an asymmetric low-rank compression method that decouples the latent heterogeneity between the two SVD factors. Under a fixed memory budget, the proposed method jointly determines the retained rank, factor-wise quantization bit-widths, and quantization directions, allowing more effective singular directions to be preserved with low-bit representations. To efficiently select compression configurations, we use an activation-aware reconstruction error as a layer-wise proxy objective. Experiments on multiple LLMs and datasets show that DELTA consistently outperforms existing post-training SVD-based compression methods, especially under high compression ratios. Compared with the strongest SVD-based baseline at the same compression ratio, DELTA reduces perplexity on WikiText-2 and C4 by up to 68.2% and 53.2%, respectively, and improves average zero-shot accuracy by up to 11 percentage points. These results demonstrate that asymmetric factor-aware compression can effectively mitigate the accuracy degradation of SVD-based LLM compression at high compression ratios. Code is available at https://github.com/BlacksheepJane/DELTA.</abstract>
<identifier type="citekey">zhan-etal-2026-delta</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2287/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>52046</start>
<end>52057</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DELTA: Decoupling Latent Heterogeneity in Asymmetric Low-Rank Compression
%A Zhan, Jialin
%A Liu, Fangxin
%A Wang, Junjie
%A Yang, Ning
%A Jiang, Li
%A Guan, Haibing
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhan-etal-2026-delta
%X Singular Value Decomposition (SVD) is a promising post-training compression technique for large language models (LLMs), as it is training-free, hardware-friendly, and preserves regular dense computation. However, existing SVD-based methods suffer from severe accuracy degradation under aggressive compression. This is mainly because hard rank truncation discards useful singular directions, while the two resulting SVD factors are usually compressed symmetrically despite their different roles, shapes, and quantization sensitivities. In this paper, we propose DELTA, an asymmetric low-rank compression method that decouples the latent heterogeneity between the two SVD factors. Under a fixed memory budget, the proposed method jointly determines the retained rank, factor-wise quantization bit-widths, and quantization directions, allowing more effective singular directions to be preserved with low-bit representations. To efficiently select compression configurations, we use an activation-aware reconstruction error as a layer-wise proxy objective. Experiments on multiple LLMs and datasets show that DELTA consistently outperforms existing post-training SVD-based compression methods, especially under high compression ratios. Compared with the strongest SVD-based baseline at the same compression ratio, DELTA reduces perplexity on WikiText-2 and C4 by up to 68.2% and 53.2%, respectively, and improves average zero-shot accuracy by up to 11 percentage points. These results demonstrate that asymmetric factor-aware compression can effectively mitigate the accuracy degradation of SVD-based LLM compression at high compression ratios. Code is available at https://github.com/BlacksheepJane/DELTA.
%U https://aclanthology.org/2026.emnlp-main.2287/
%P 52046-52057
Markdown (Informal)
[DELTA: Decoupling Latent Heterogeneity in Asymmetric Low-Rank Compression](https://aclanthology.org/2026.emnlp-main.2287/) (Zhan et al., EMNLP 2026)
ACL