@inproceedings{yao-etal-2026-densemixer,
title = "{D}ense{M}ixer: Improving {M}o{E} Post-Training via Precise Router Gradient",
author = "Yao, Feng and
Cui, Junxia and
Zhang, Ruohan and
Liu, Liyuan and
Hao, Shibo and
Zhang, Li and
Dong, Chengyu and
Zhi, Churan and
Wang, Shuohang and
Shen, Yelong and
Gao, Jianfeng and
Shang, Jingbo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1682/",
pages = "38287--38298",
ISBN = "979-8-89176-459-0",
abstract = "The widely adopted Top-K router in Mixture-of-Experts (MoE) models is mathematically non-differentiable, making them harder to train than dense counterparts. We propose DenseMixer, a simple and effective MoE post-training technique that provides a more precise router gradient estimation by trading in some extra compute. It is universally applicable to TopK routed MoE, operates in a plug-and-play manner, preserves inference-time efficiency, and remains fully compatible with popular training libraries as well as parameter-efficient methods such as LoRA. Extensive experiments demonstrate that DenseMixer consistently outperforms conventional approaches when fine-tuning MoE models across scales (7B{--}30B), architectures (with and without shared experts), pre-training regimes of the underlying MoE models (from scratch or via up-cycling), and post-training data types (instruction tuning and long chain-of-thought). Furthermore, our analysis on efficiency shows that the additional computation introduced by DenseMixer remains modest, providing substantial improvements at an acceptable cost."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yao-etal-2026-densemixer">
<titleInfo>
<title>DenseMixer: Improving MoE Post-Training via Precise Router Gradient</title>
</titleInfo>
<name type="personal">
<namePart type="given">Feng</namePart>
<namePart type="family">Yao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junxia</namePart>
<namePart type="family">Cui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruohan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liyuan</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shibo</namePart>
<namePart type="family">Hao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Li</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chengyu</namePart>
<namePart type="family">Dong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Churan</namePart>
<namePart type="family">Zhi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuohang</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yelong</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianfeng</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingbo</namePart>
<namePart type="family">Shang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>The widely adopted Top-K router in Mixture-of-Experts (MoE) models is mathematically non-differentiable, making them harder to train than dense counterparts. We propose DenseMixer, a simple and effective MoE post-training technique that provides a more precise router gradient estimation by trading in some extra compute. It is universally applicable to TopK routed MoE, operates in a plug-and-play manner, preserves inference-time efficiency, and remains fully compatible with popular training libraries as well as parameter-efficient methods such as LoRA. Extensive experiments demonstrate that DenseMixer consistently outperforms conventional approaches when fine-tuning MoE models across scales (7B–30B), architectures (with and without shared experts), pre-training regimes of the underlying MoE models (from scratch or via up-cycling), and post-training data types (instruction tuning and long chain-of-thought). Furthermore, our analysis on efficiency shows that the additional computation introduced by DenseMixer remains modest, providing substantial improvements at an acceptable cost.</abstract>
<identifier type="citekey">yao-etal-2026-densemixer</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1682/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>38287</start>
<end>38298</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DenseMixer: Improving MoE Post-Training via Precise Router Gradient
%A Yao, Feng
%A Cui, Junxia
%A Zhang, Ruohan
%A Liu, Liyuan
%A Hao, Shibo
%A Zhang, Li
%A Dong, Chengyu
%A Zhi, Churan
%A Wang, Shuohang
%A Shen, Yelong
%A Gao, Jianfeng
%A Shang, Jingbo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yao-etal-2026-densemixer
%X The widely adopted Top-K router in Mixture-of-Experts (MoE) models is mathematically non-differentiable, making them harder to train than dense counterparts. We propose DenseMixer, a simple and effective MoE post-training technique that provides a more precise router gradient estimation by trading in some extra compute. It is universally applicable to TopK routed MoE, operates in a plug-and-play manner, preserves inference-time efficiency, and remains fully compatible with popular training libraries as well as parameter-efficient methods such as LoRA. Extensive experiments demonstrate that DenseMixer consistently outperforms conventional approaches when fine-tuning MoE models across scales (7B–30B), architectures (with and without shared experts), pre-training regimes of the underlying MoE models (from scratch or via up-cycling), and post-training data types (instruction tuning and long chain-of-thought). Furthermore, our analysis on efficiency shows that the additional computation introduced by DenseMixer remains modest, providing substantial improvements at an acceptable cost.
%U https://aclanthology.org/2026.emnlp-main.1682/
%P 38287-38298
Markdown (Informal)
[DenseMixer: Improving MoE Post-Training via Precise Router Gradient](https://aclanthology.org/2026.emnlp-main.1682/) (Yao et al., EMNLP 2026)
ACL
- Feng Yao, Junxia Cui, Ruohan Zhang, Liyuan Liu, Shibo Hao, Li Zhang, Chengyu Dong, Churan Zhi, Shuohang Wang, Yelong Shen, Jianfeng Gao, and Jingbo Shang. 2026. DenseMixer: Improving MoE Post-Training via Precise Router Gradient. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 38287–38298, Budapest, Hungary. Association for Computational Linguistics.