@inproceedings{yang-etal-2026-rescode,
title = "{R}es{C}ode: Accurate Expert Compression of Sparse {M}o{E} Models via Residual-code Restoration",
author = "Yang, Hyunwoo and
Kim, Minjun and
Kang, U",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2691/",
pages = "61227--61245",
ISBN = "979-8-89176-459-0",
abstract = "How can we accurately compress a sparse mixture-of-experts (SMoE) model? SMoE models are widely adopted in modern LLMs for reducing activated computation, but their deployment remains memory-intensive due to extensive expert parameters. To address this bottleneck, expert compression reduces redundant expert parameters. However, previous methods decrease the number of experts involved in token-dependent routing, inducing output deviations from the original model. Expert decomposition is a promising alternative, but existing approaches suffer from three major limitations: a single shared base, component memory overhead, and suboptimal objectives. In this work, we introduce five desired properties for accurate expert compression. Then, we propose ResCode, an accurate expert decomposition method based on residual-code restoration. ResCode forms compact expert clusters, coalesces residuals into cluster-shared low-rank bases, and learns compact codes to accurately recover the original routed outputs. ResCode provides the state-of-the-art performance for compressing SMoE models on various tasks, achieving up to 2.43 percentage points higher average accuracy than the strongest baseline."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yang-etal-2026-rescode">
<titleInfo>
<title>ResCode: Accurate Expert Compression of Sparse MoE Models via Residual-code Restoration</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hyunwoo</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minjun</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">U</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>How can we accurately compress a sparse mixture-of-experts (SMoE) model? SMoE models are widely adopted in modern LLMs for reducing activated computation, but their deployment remains memory-intensive due to extensive expert parameters. To address this bottleneck, expert compression reduces redundant expert parameters. However, previous methods decrease the number of experts involved in token-dependent routing, inducing output deviations from the original model. Expert decomposition is a promising alternative, but existing approaches suffer from three major limitations: a single shared base, component memory overhead, and suboptimal objectives. In this work, we introduce five desired properties for accurate expert compression. Then, we propose ResCode, an accurate expert decomposition method based on residual-code restoration. ResCode forms compact expert clusters, coalesces residuals into cluster-shared low-rank bases, and learns compact codes to accurately recover the original routed outputs. ResCode provides the state-of-the-art performance for compressing SMoE models on various tasks, achieving up to 2.43 percentage points higher average accuracy than the strongest baseline.</abstract>
<identifier type="citekey">yang-etal-2026-rescode</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2691/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>61227</start>
<end>61245</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ResCode: Accurate Expert Compression of Sparse MoE Models via Residual-code Restoration
%A Yang, Hyunwoo
%A Kim, Minjun
%A Kang, U.
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yang-etal-2026-rescode
%X How can we accurately compress a sparse mixture-of-experts (SMoE) model? SMoE models are widely adopted in modern LLMs for reducing activated computation, but their deployment remains memory-intensive due to extensive expert parameters. To address this bottleneck, expert compression reduces redundant expert parameters. However, previous methods decrease the number of experts involved in token-dependent routing, inducing output deviations from the original model. Expert decomposition is a promising alternative, but existing approaches suffer from three major limitations: a single shared base, component memory overhead, and suboptimal objectives. In this work, we introduce five desired properties for accurate expert compression. Then, we propose ResCode, an accurate expert decomposition method based on residual-code restoration. ResCode forms compact expert clusters, coalesces residuals into cluster-shared low-rank bases, and learns compact codes to accurately recover the original routed outputs. ResCode provides the state-of-the-art performance for compressing SMoE models on various tasks, achieving up to 2.43 percentage points higher average accuracy than the strongest baseline.
%U https://aclanthology.org/2026.emnlp-main.2691/
%P 61227-61245
Markdown (Informal)
[ResCode: Accurate Expert Compression of Sparse MoE Models via Residual-code Restoration](https://aclanthology.org/2026.emnlp-main.2691/) (Yang et al., EMNLP 2026)
ACL