@inproceedings{ning-etal-2026-nimblemoe,
title = "{N}imble{M}o{E}: Collaboration-Aware Expert Merging with Sparse Compensation for Efficient Mixture-of-Experts {LLM}s",
author = "Ning, Zhiyuan and
Liu, Xiaoran and
Xu, Ruge and
Guo, Xinfei and
He, Ziwei and
Qiu, Xipeng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2036/",
pages = "46390--46408",
ISBN = "979-8-89176-459-0",
abstract = "Mixture-of-Experts (MoE) Transformers improve scaling efficiency by activating only a subset of experts per token, but deployed inference remains constrained by memory capacity and bandwidth. Existing expert compression methods face a quality{--}memory{--}latency trilemma: pruning or fully merging degrades model quality, while partial merging increases per-token activated parameters and hurt latency. We propose NimbleMoE, a training-free, collaboration-aware partial expert merging framework that reduces both model size and activated-parameter cost. NimbleMoE clusters experts by routing co-activation so co-activated experts share a low-rank subspace. To compensate for the capacity lost, NimbleMoE adds hardware-efficient Monarch residuals, with an alignment-aware choice between latent-space and output-space compensation. Across multiple MoE backbones, NimbleMoE preserves model quality under 20{--}25{\%} total-parameter compression and outperforms prior pruning and merging baselines, while delivering up to 1.89$\times$ MoE-MLP kernel speedup in bandwidth-bound settings, 1.31$\times$ end-to-end TPOT speedup, and 2.81$\times$ expert-parallel MoE-kernel speedup."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ning-etal-2026-nimblemoe">
<titleInfo>
<title>NimbleMoE: Collaboration-Aware Expert Merging with Sparse Compensation for Efficient Mixture-of-Experts LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhiyuan</namePart>
<namePart type="family">Ning</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaoran</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruge</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinfei</namePart>
<namePart type="family">Guo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziwei</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xipeng</namePart>
<namePart type="family">Qiu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Mixture-of-Experts (MoE) Transformers improve scaling efficiency by activating only a subset of experts per token, but deployed inference remains constrained by memory capacity and bandwidth. Existing expert compression methods face a quality–memory–latency trilemma: pruning or fully merging degrades model quality, while partial merging increases per-token activated parameters and hurt latency. We propose NimbleMoE, a training-free, collaboration-aware partial expert merging framework that reduces both model size and activated-parameter cost. NimbleMoE clusters experts by routing co-activation so co-activated experts share a low-rank subspace. To compensate for the capacity lost, NimbleMoE adds hardware-efficient Monarch residuals, with an alignment-aware choice between latent-space and output-space compensation. Across multiple MoE backbones, NimbleMoE preserves model quality under 20–25% total-parameter compression and outperforms prior pruning and merging baselines, while delivering up to 1.89\times MoE-MLP kernel speedup in bandwidth-bound settings, 1.31\times end-to-end TPOT speedup, and 2.81\times expert-parallel MoE-kernel speedup.</abstract>
<identifier type="citekey">ning-etal-2026-nimblemoe</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2036/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>46390</start>
<end>46408</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T NimbleMoE: Collaboration-Aware Expert Merging with Sparse Compensation for Efficient Mixture-of-Experts LLMs
%A Ning, Zhiyuan
%A Liu, Xiaoran
%A Xu, Ruge
%A Guo, Xinfei
%A He, Ziwei
%A Qiu, Xipeng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F ning-etal-2026-nimblemoe
%X Mixture-of-Experts (MoE) Transformers improve scaling efficiency by activating only a subset of experts per token, but deployed inference remains constrained by memory capacity and bandwidth. Existing expert compression methods face a quality–memory–latency trilemma: pruning or fully merging degrades model quality, while partial merging increases per-token activated parameters and hurt latency. We propose NimbleMoE, a training-free, collaboration-aware partial expert merging framework that reduces both model size and activated-parameter cost. NimbleMoE clusters experts by routing co-activation so co-activated experts share a low-rank subspace. To compensate for the capacity lost, NimbleMoE adds hardware-efficient Monarch residuals, with an alignment-aware choice between latent-space and output-space compensation. Across multiple MoE backbones, NimbleMoE preserves model quality under 20–25% total-parameter compression and outperforms prior pruning and merging baselines, while delivering up to 1.89\times MoE-MLP kernel speedup in bandwidth-bound settings, 1.31\times end-to-end TPOT speedup, and 2.81\times expert-parallel MoE-kernel speedup.
%U https://aclanthology.org/2026.emnlp-main.2036/
%P 46390-46408
Markdown (Informal)
[NimbleMoE: Collaboration-Aware Expert Merging with Sparse Compensation for Efficient Mixture-of-Experts LLMs](https://aclanthology.org/2026.emnlp-main.2036/) (Ning et al., EMNLP 2026)
ACL