@inproceedings{mustafa-etal-2026-moep,
title = "{M}o{EP}+: Enhancing Modular Expert Paths for Data-Efficient Language Modeling",
author = "Mustafa, Noor Eidirhn and
Melkersson, Valdemar and
Nugues, Pierre",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.39/",
pages = "504--513",
ISBN = "979-8-89176-475-0",
abstract = "We present MoEP+, an enhanced version of the Modular Expert Paths (MoEP) decoder. MoEP+ retains MoEP{'}s top-k routing over parallel expert paths, simplifies its dimensional mapping layers, and modernizes the backbone with RoPE, RMS normalization, and SwiGLU. It also introduces a projection-specialized routing scheme in which different experts apply nonlinear residual transformations to distinct attention projections. We evaluate MoEP+ on the BabyLM 2026 strict-small track, which is limited to 10 million words, and present it as a combined architectural recipe rather than attributing the observed gains to any single component.\par Under a matched training budget, MoEP+ outperforms the unmodified MoEP baseline on all eight zero-shot benchmarks. On five of these benchmarks, the advantage persists even when the baseline is trained for five times as many epochs. Most notably, MoEP+ is substantially more data-efficient: after only two epochs, it surpasses the unmodified MoEP{'}s ten-epoch BLiMP score. Despite its smaller size, it also remains competitive with the official BabyLM 2026 baselines, with its advantages concentrated in the reading-based measures."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mustafa-etal-2026-moep">
<titleInfo>
<title>MoEP+: Enhancing Modular Expert Paths for Data-Efficient Language Modeling</title>
</titleInfo>
<name type="personal">
<namePart type="given">Noor</namePart>
<namePart type="given">Eidirhn</namePart>
<namePart type="family">Mustafa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Valdemar</namePart>
<namePart type="family">Melkersson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pierre</namePart>
<namePart type="family">Nugues</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>We present MoEP+, an enhanced version of the Modular Expert Paths (MoEP) decoder. MoEP+ retains MoEP’s top-k routing over parallel expert paths, simplifies its dimensional mapping layers, and modernizes the backbone with RoPE, RMS normalization, and SwiGLU. It also introduces a projection-specialized routing scheme in which different experts apply nonlinear residual transformations to distinct attention projections. We evaluate MoEP+ on the BabyLM 2026 strict-small track, which is limited to 10 million words, and present it as a combined architectural recipe rather than attributing the observed gains to any single component.\par Under a matched training budget, MoEP+ outperforms the unmodified MoEP baseline on all eight zero-shot benchmarks. On five of these benchmarks, the advantage persists even when the baseline is trained for five times as many epochs. Most notably, MoEP+ is substantially more data-efficient: after only two epochs, it surpasses the unmodified MoEP’s ten-epoch BLiMP score. Despite its smaller size, it also remains competitive with the official BabyLM 2026 baselines, with its advantages concentrated in the reading-based measures.</abstract>
<identifier type="citekey">mustafa-etal-2026-moep</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.39/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>504</start>
<end>513</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MoEP+: Enhancing Modular Expert Paths for Data-Efficient Language Modeling
%A Mustafa, Noor Eidirhn
%A Melkersson, Valdemar
%A Nugues, Pierre
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F mustafa-etal-2026-moep
%X We present MoEP+, an enhanced version of the Modular Expert Paths (MoEP) decoder. MoEP+ retains MoEP’s top-k routing over parallel expert paths, simplifies its dimensional mapping layers, and modernizes the backbone with RoPE, RMS normalization, and SwiGLU. It also introduces a projection-specialized routing scheme in which different experts apply nonlinear residual transformations to distinct attention projections. We evaluate MoEP+ on the BabyLM 2026 strict-small track, which is limited to 10 million words, and present it as a combined architectural recipe rather than attributing the observed gains to any single component.\par Under a matched training budget, MoEP+ outperforms the unmodified MoEP baseline on all eight zero-shot benchmarks. On five of these benchmarks, the advantage persists even when the baseline is trained for five times as many epochs. Most notably, MoEP+ is substantially more data-efficient: after only two epochs, it surpasses the unmodified MoEP’s ten-epoch BLiMP score. Despite its smaller size, it also remains competitive with the official BabyLM 2026 baselines, with its advantages concentrated in the reading-based measures.
%U https://aclanthology.org/2026.babylm-main.39/
%P 504-513
Markdown (Informal)
[MoEP+: Enhancing Modular Expert Paths for Data-Efficient Language Modeling](https://aclanthology.org/2026.babylm-main.39/) (Mustafa et al., BabyLM 2026)
ACL