@inproceedings{parvathy-etal-2026-tokenmix,
title = "{T}oken{M}ix: Efficient Batched Inference for Mixture-of-Experts Models with Joint Expert Selection",
author = "Parvathy, Aradhana Mohan and
Krithivasan, Sarada and
Venkataramani, Swagath and
Raghunathan, Anand and
Srinivasan, Viji",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.208/",
pages = "4128--4140",
ISBN = "979-8-89176-444-6",
abstract = "The mixture-of-experts (MoE) architecture has proven indispensable for scaling transformer-based artificial intelligence (AI) models. MoEs use a router network to activate specialized subnetworks, called experts, within each transformer layer on a per-token basis. Thus, MoEs increase model capacity without proportionally increasing the computational requirements of training and inference. However, a key challenge is that MoE inference throughput scales poorly with batching, as each input in a batch activates a different subset of experts, thereby increasing memory intensity.\par To improve batched MoE inference throughput, we propose TokenMix, a novel technique that mixes token representations within the router across inputs in a batch to promote expert reuse. In effect, TokenMix encourages coordinated expert selection across inputs. TokenMix requires neither training nor fine-tuning and can be applied directly to off-the-shelf MoE models. We evaluate TokenMix across five state-of-the-art MoE models with parameter counts ranging from 16B to 235B. TokenMix achieves a 25{--}60{\%} improvement in expert reuse, leading to a 12{--}28{\%} reduction in inter-token latency with negligible or no loss in accuracy."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="parvathy-etal-2026-tokenmix">
<titleInfo>
<title>TokenMix: Efficient Batched Inference for Mixture-of-Experts Models with Joint Expert Selection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Aradhana</namePart>
<namePart type="given">Mohan</namePart>
<namePart type="family">Parvathy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarada</namePart>
<namePart type="family">Krithivasan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swagath</namePart>
<namePart type="family">Venkataramani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anand</namePart>
<namePart type="family">Raghunathan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Viji</namePart>
<namePart type="family">Srinivasan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>The mixture-of-experts (MoE) architecture has proven indispensable for scaling transformer-based artificial intelligence (AI) models. MoEs use a router network to activate specialized subnetworks, called experts, within each transformer layer on a per-token basis. Thus, MoEs increase model capacity without proportionally increasing the computational requirements of training and inference. However, a key challenge is that MoE inference throughput scales poorly with batching, as each input in a batch activates a different subset of experts, thereby increasing memory intensity.\par To improve batched MoE inference throughput, we propose TokenMix, a novel technique that mixes token representations within the router across inputs in a batch to promote expert reuse. In effect, TokenMix encourages coordinated expert selection across inputs. TokenMix requires neither training nor fine-tuning and can be applied directly to off-the-shelf MoE models. We evaluate TokenMix across five state-of-the-art MoE models with parameter counts ranging from 16B to 235B. TokenMix achieves a 25–60% improvement in expert reuse, leading to a 12–28% reduction in inter-token latency with negligible or no loss in accuracy.</abstract>
<identifier type="citekey">parvathy-etal-2026-tokenmix</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.208/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>4128</start>
<end>4140</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T TokenMix: Efficient Batched Inference for Mixture-of-Experts Models with Joint Expert Selection
%A Parvathy, Aradhana Mohan
%A Krithivasan, Sarada
%A Venkataramani, Swagath
%A Raghunathan, Anand
%A Srinivasan, Viji
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F parvathy-etal-2026-tokenmix
%X The mixture-of-experts (MoE) architecture has proven indispensable for scaling transformer-based artificial intelligence (AI) models. MoEs use a router network to activate specialized subnetworks, called experts, within each transformer layer on a per-token basis. Thus, MoEs increase model capacity without proportionally increasing the computational requirements of training and inference. However, a key challenge is that MoE inference throughput scales poorly with batching, as each input in a batch activates a different subset of experts, thereby increasing memory intensity.\par To improve batched MoE inference throughput, we propose TokenMix, a novel technique that mixes token representations within the router across inputs in a batch to promote expert reuse. In effect, TokenMix encourages coordinated expert selection across inputs. TokenMix requires neither training nor fine-tuning and can be applied directly to off-the-shelf MoE models. We evaluate TokenMix across five state-of-the-art MoE models with parameter counts ranging from 16B to 235B. TokenMix achieves a 25–60% improvement in expert reuse, leading to a 12–28% reduction in inter-token latency with negligible or no loss in accuracy.
%U https://aclanthology.org/2026.findings-aacl.208/
%P 4128-4140
Markdown (Informal)
[TokenMix: Efficient Batched Inference for Mixture-of-Experts Models with Joint Expert Selection](https://aclanthology.org/2026.findings-aacl.208/) (Parvathy et al., Findings 2026)
ACL