@inproceedings{quang-etal-2026-simpa,
title = "{SIMPA}: Restricting What Experts See, Not What They Produce, in {M}o{E}-{PEFT}",
author = "Quang, Pham Tri and
Tran, Khoa and
Do, Giang and
Le, Hung and
Tran, Truyen and
Quan, Tho",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.726/",
pages = "16873--16894",
ISBN = "979-8-89176-459-0",
abstract = "Mixture-of-Experts (MoE) routing collapses in parameter-efficient fine-tuning: on LLaMA-3 8B with MixLoRA, a single expert absorbs 82.5{\%} of tokens. Prior MoE-PEFT remedies intervene at the output: routing reformulations, output-similarity penalties, or shared-expert capacity. All leave each routed expert{'}s input view identical. We propose SIMPA (Sparse-Input Magnitude Partitioning Adaptation), which restricts what experts *see* rather than what they produce: a magnitude-thresholded input drives a sparse routed-LoRA branch, a parallel dense LoRA branch (adapting DeepSeekMoE{'}s (Dai et al., 2024) shared-expert topology to PEFT) provides an information safeguard, and a column-wise gate fuses the two. At a matched 2.9{\%} trainable-parameter budget, SIMPA leads compared baselines on reasoning (86.2{\%} on seven LLaMA-3 8B tasks, +3.9 over MixDoRA), code and arithmetic (38.4 HumanEval Pass@1 and 59.4 GSM8K, matching full fine-tuning under identical data and compute), and NLU (88.7 GLUE / 77.7 SuperGLUE on T5-base); against direct MoE-PEFT counterparts HydraLoRA and MoLoRA at matched setup, SIMPA leads HumanEval by +14.2 and +13.6 and GSM8K by +2.1 and +2.8. Direct measurement returns an expert-input Jaccard of 0.184 versus MixLoRA{'}s 1.000, a 5.4{\texttimes} reduction. The same block-aligned mask drives a Triton kernel measuring 1.46{\texttimes} end-to-end inference speedup on A100. Closed-form partitioning and retention bounds characterize the mechanism, and a tail-index-driven closed form predicts the optimal sparsity $p^*$ from the pretrained backbone. The mechanism, its predicted optimum, and its bound all transfer to a held-out Qwen2.5-7B backbone. Code: https://github.com/triquang26/simpa."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="quang-etal-2026-simpa">
<titleInfo>
<title>SIMPA: Restricting What Experts See, Not What They Produce, in MoE-PEFT</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pham</namePart>
<namePart type="given">Tri</namePart>
<namePart type="family">Quang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Khoa</namePart>
<namePart type="family">Tran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Giang</namePart>
<namePart type="family">Do</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hung</namePart>
<namePart type="family">Le</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Truyen</namePart>
<namePart type="family">Tran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tho</namePart>
<namePart type="family">Quan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Mixture-of-Experts (MoE) routing collapses in parameter-efficient fine-tuning: on LLaMA-3 8B with MixLoRA, a single expert absorbs 82.5% of tokens. Prior MoE-PEFT remedies intervene at the output: routing reformulations, output-similarity penalties, or shared-expert capacity. All leave each routed expert’s input view identical. We propose SIMPA (Sparse-Input Magnitude Partitioning Adaptation), which restricts what experts *see* rather than what they produce: a magnitude-thresholded input drives a sparse routed-LoRA branch, a parallel dense LoRA branch (adapting DeepSeekMoE’s (Dai et al., 2024) shared-expert topology to PEFT) provides an information safeguard, and a column-wise gate fuses the two. At a matched 2.9% trainable-parameter budget, SIMPA leads compared baselines on reasoning (86.2% on seven LLaMA-3 8B tasks, +3.9 over MixDoRA), code and arithmetic (38.4 HumanEval Pass@1 and 59.4 GSM8K, matching full fine-tuning under identical data and compute), and NLU (88.7 GLUE / 77.7 SuperGLUE on T5-base); against direct MoE-PEFT counterparts HydraLoRA and MoLoRA at matched setup, SIMPA leads HumanEval by +14.2 and +13.6 and GSM8K by +2.1 and +2.8. Direct measurement returns an expert-input Jaccard of 0.184 versus MixLoRA’s 1.000, a 5.4× reduction. The same block-aligned mask drives a Triton kernel measuring 1.46× end-to-end inference speedup on A100. Closed-form partitioning and retention bounds characterize the mechanism, and a tail-index-driven closed form predicts the optimal sparsity p^* from the pretrained backbone. The mechanism, its predicted optimum, and its bound all transfer to a held-out Qwen2.5-7B backbone. Code: https://github.com/triquang26/simpa.</abstract>
<identifier type="citekey">quang-etal-2026-simpa</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.726/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>16873</start>
<end>16894</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SIMPA: Restricting What Experts See, Not What They Produce, in MoE-PEFT
%A Quang, Pham Tri
%A Tran, Khoa
%A Do, Giang
%A Le, Hung
%A Tran, Truyen
%A Quan, Tho
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F quang-etal-2026-simpa
%X Mixture-of-Experts (MoE) routing collapses in parameter-efficient fine-tuning: on LLaMA-3 8B with MixLoRA, a single expert absorbs 82.5% of tokens. Prior MoE-PEFT remedies intervene at the output: routing reformulations, output-similarity penalties, or shared-expert capacity. All leave each routed expert’s input view identical. We propose SIMPA (Sparse-Input Magnitude Partitioning Adaptation), which restricts what experts *see* rather than what they produce: a magnitude-thresholded input drives a sparse routed-LoRA branch, a parallel dense LoRA branch (adapting DeepSeekMoE’s (Dai et al., 2024) shared-expert topology to PEFT) provides an information safeguard, and a column-wise gate fuses the two. At a matched 2.9% trainable-parameter budget, SIMPA leads compared baselines on reasoning (86.2% on seven LLaMA-3 8B tasks, +3.9 over MixDoRA), code and arithmetic (38.4 HumanEval Pass@1 and 59.4 GSM8K, matching full fine-tuning under identical data and compute), and NLU (88.7 GLUE / 77.7 SuperGLUE on T5-base); against direct MoE-PEFT counterparts HydraLoRA and MoLoRA at matched setup, SIMPA leads HumanEval by +14.2 and +13.6 and GSM8K by +2.1 and +2.8. Direct measurement returns an expert-input Jaccard of 0.184 versus MixLoRA’s 1.000, a 5.4× reduction. The same block-aligned mask drives a Triton kernel measuring 1.46× end-to-end inference speedup on A100. Closed-form partitioning and retention bounds characterize the mechanism, and a tail-index-driven closed form predicts the optimal sparsity p^* from the pretrained backbone. The mechanism, its predicted optimum, and its bound all transfer to a held-out Qwen2.5-7B backbone. Code: https://github.com/triquang26/simpa.
%U https://aclanthology.org/2026.emnlp-main.726/
%P 16873-16894
Markdown (Informal)
[SIMPA: Restricting What Experts See, Not What They Produce, in MoE-PEFT](https://aclanthology.org/2026.emnlp-main.726/) (Quang et al., EMNLP 2026)
ACL
- Pham Tri Quang, Khoa Tran, Giang Do, Hung Le, Truyen Tran, and Tho Quan. 2026. SIMPA: Restricting What Experts See, Not What They Produce, in MoE-PEFT. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 16873–16894, Budapest, Hungary. Association for Computational Linguistics.