@inproceedings{wu-etal-2026-perturbprune,
title = "{P}erturb{P}rune: Guiding Sparsity Allocation by Probing Layer Stability Along Principal Directions",
author = "Wu, Ningxin and
Hou, Zhengxu and
Zhao, Ziran and
Jing, Tao and
Liu, Pengyuan and
Kang, Chen and
Zhang, Yangshijie and
Xing, Luxi and
Yang, Bin-Bin and
Xiao, Feng and
Liu, Lu and
Yu, Dong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1122/",
pages = "23753--23780",
ISBN = "979-8-89176-462-0",
abstract = "Large language models (LLMs) exhibit significant layer-wise redundancy, motivating non-uniform pruning for efficient compression. However, existing sparsity allocation methods rely on indirect statistical heuristics derived from weight magnitudes or activation distributions, without directly assessing how robustly each layer preserves semantic information under input variation. We propose PerturbPrune, a framework that allocates layer-wise sparsity by probing the intrinsic stability of hidden representations. PerturbPrune estimates the principal subspace of each layer{'}s activations via PCA and injects structured perturbations along these directions. Layer sensitivity is quantified through matrix difference entropy (DiME): stable layers are identified as encoding compact, essential features and thus preserved, while highly sensitive layers are deemed over-parameterized and pruned more aggressively. Extensive experiments across diverse LLM families (LLaMA, LLaMA-2, Qwen, OPT) demonstrate that PerturbPrune broadly matches or outperforms existing non-uniform methods. Compared to uniform pruning, it prevents catastrophic collapse, reducing perplexity by up to 99.7{\%} and improving zero-shot accuracy by up to 22.7{\%}. The method is also highly efficient, completing sparsity allocation on Qwen3.5-9B in 7 minutes{---}achieving 24$\times$ and 17.4$\times$ speedups over AlphaPruning and MRP. Finally, PerturbPrune generalizes to Mixture-of-Experts architectures, achieving up to 19.3{\%} perplexity reduction on Qwen3.5-35B-A3B."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wu-etal-2026-perturbprune">
<titleInfo>
<title>PerturbPrune: Guiding Sparsity Allocation by Probing Layer Stability Along Principal Directions</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ningxin</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhengxu</namePart>
<namePart type="family">Hou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziran</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tao</namePart>
<namePart type="family">Jing</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pengyuan</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chen</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yangshijie</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Luxi</namePart>
<namePart type="family">Xing</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bin-Bin</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Feng</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lu</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dong</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) exhibit significant layer-wise redundancy, motivating non-uniform pruning for efficient compression. However, existing sparsity allocation methods rely on indirect statistical heuristics derived from weight magnitudes or activation distributions, without directly assessing how robustly each layer preserves semantic information under input variation. We propose PerturbPrune, a framework that allocates layer-wise sparsity by probing the intrinsic stability of hidden representations. PerturbPrune estimates the principal subspace of each layer’s activations via PCA and injects structured perturbations along these directions. Layer sensitivity is quantified through matrix difference entropy (DiME): stable layers are identified as encoding compact, essential features and thus preserved, while highly sensitive layers are deemed over-parameterized and pruned more aggressively. Extensive experiments across diverse LLM families (LLaMA, LLaMA-2, Qwen, OPT) demonstrate that PerturbPrune broadly matches or outperforms existing non-uniform methods. Compared to uniform pruning, it prevents catastrophic collapse, reducing perplexity by up to 99.7% and improving zero-shot accuracy by up to 22.7%. The method is also highly efficient, completing sparsity allocation on Qwen3.5-9B in 7 minutes—achieving 24\times and 17.4\times speedups over AlphaPruning and MRP. Finally, PerturbPrune generalizes to Mixture-of-Experts architectures, achieving up to 19.3% perplexity reduction on Qwen3.5-35B-A3B.</abstract>
<identifier type="citekey">wu-etal-2026-perturbprune</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1122/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>23753</start>
<end>23780</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PerturbPrune: Guiding Sparsity Allocation by Probing Layer Stability Along Principal Directions
%A Wu, Ningxin
%A Hou, Zhengxu
%A Zhao, Ziran
%A Jing, Tao
%A Liu, Pengyuan
%A Kang, Chen
%A Zhang, Yangshijie
%A Xing, Luxi
%A Yang, Bin-Bin
%A Xiao, Feng
%A Liu, Lu
%A Yu, Dong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wu-etal-2026-perturbprune
%X Large language models (LLMs) exhibit significant layer-wise redundancy, motivating non-uniform pruning for efficient compression. However, existing sparsity allocation methods rely on indirect statistical heuristics derived from weight magnitudes or activation distributions, without directly assessing how robustly each layer preserves semantic information under input variation. We propose PerturbPrune, a framework that allocates layer-wise sparsity by probing the intrinsic stability of hidden representations. PerturbPrune estimates the principal subspace of each layer’s activations via PCA and injects structured perturbations along these directions. Layer sensitivity is quantified through matrix difference entropy (DiME): stable layers are identified as encoding compact, essential features and thus preserved, while highly sensitive layers are deemed over-parameterized and pruned more aggressively. Extensive experiments across diverse LLM families (LLaMA, LLaMA-2, Qwen, OPT) demonstrate that PerturbPrune broadly matches or outperforms existing non-uniform methods. Compared to uniform pruning, it prevents catastrophic collapse, reducing perplexity by up to 99.7% and improving zero-shot accuracy by up to 22.7%. The method is also highly efficient, completing sparsity allocation on Qwen3.5-9B in 7 minutes—achieving 24\times and 17.4\times speedups over AlphaPruning and MRP. Finally, PerturbPrune generalizes to Mixture-of-Experts architectures, achieving up to 19.3% perplexity reduction on Qwen3.5-35B-A3B.
%U https://aclanthology.org/2026.findings-emnlp.1122/
%P 23753-23780
Markdown (Informal)
[PerturbPrune: Guiding Sparsity Allocation by Probing Layer Stability Along Principal Directions](https://aclanthology.org/2026.findings-emnlp.1122/) (Wu et al., Findings 2026)
ACL
- Ningxin Wu, Zhengxu Hou, Ziran Zhao, Tao Jing, Pengyuan Liu, Chen Kang, Yangshijie Zhang, Luxi Xing, Bin-Bin Yang, Feng Xiao, Lu Liu, and Dong Yu. 2026. PerturbPrune: Guiding Sparsity Allocation by Probing Layer Stability Along Principal Directions. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 23753–23780, Budapest, Hungary. Association for Computational Linguistics.