@inproceedings{jain-etal-2026-xpertgpt,
title = "{X}pert{GPT}: Multi-Scale Sparse Expert Routing for Data-Constrained Language Modeling",
author = "Jain, Soham and
Singh, Harsh and
Dewan, Divija and
Dev, Atul",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.60/",
pages = "767--778",
ISBN = "979-8-89176-475-0",
abstract = "We introduce XpertGPT, a sparse decoder-only Transformer for data-constrained language modelling that routes tokens to parallel expert pathways operating at multiple contextual scales. Under the BabyLM strict-small budget of roughly ten million words, conventional dense Transformers spend identical computation on every token and impose a single fixed receptive field. Each XpertGPT layer pairs a dense global-attention stream with four sparse sliding-window expert branches, whose attention spans range from 64 down to 4 tokens, chosen by Expert-Choice routing that balances expert load by construction without any auxiliary loss, and applies rotary position embeddings and gated feed-forward networks throughout. Trained on the BabyLM 2026 Strict-Small corpus, our 52.6M-parameter model reaches an overall average of 38.42, surpassing the official GPT-2(98.4M) baseline (37.38) across grammatical acceptability, world-knowledge, and language-understanding benchmarks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jain-etal-2026-xpertgpt">
<titleInfo>
<title>XpertGPT: Multi-Scale Sparse Expert Routing for Data-Constrained Language Modeling</title>
</titleInfo>
<name type="personal">
<namePart type="given">Soham</namePart>
<namePart type="family">Jain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Harsh</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Divija</namePart>
<namePart type="family">Dewan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Atul</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>We introduce XpertGPT, a sparse decoder-only Transformer for data-constrained language modelling that routes tokens to parallel expert pathways operating at multiple contextual scales. Under the BabyLM strict-small budget of roughly ten million words, conventional dense Transformers spend identical computation on every token and impose a single fixed receptive field. Each XpertGPT layer pairs a dense global-attention stream with four sparse sliding-window expert branches, whose attention spans range from 64 down to 4 tokens, chosen by Expert-Choice routing that balances expert load by construction without any auxiliary loss, and applies rotary position embeddings and gated feed-forward networks throughout. Trained on the BabyLM 2026 Strict-Small corpus, our 52.6M-parameter model reaches an overall average of 38.42, surpassing the official GPT-2(98.4M) baseline (37.38) across grammatical acceptability, world-knowledge, and language-understanding benchmarks.</abstract>
<identifier type="citekey">jain-etal-2026-xpertgpt</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.60/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>767</start>
<end>778</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T XpertGPT: Multi-Scale Sparse Expert Routing for Data-Constrained Language Modeling
%A Jain, Soham
%A Singh, Harsh
%A Dewan, Divija
%A Dev, Atul
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F jain-etal-2026-xpertgpt
%X We introduce XpertGPT, a sparse decoder-only Transformer for data-constrained language modelling that routes tokens to parallel expert pathways operating at multiple contextual scales. Under the BabyLM strict-small budget of roughly ten million words, conventional dense Transformers spend identical computation on every token and impose a single fixed receptive field. Each XpertGPT layer pairs a dense global-attention stream with four sparse sliding-window expert branches, whose attention spans range from 64 down to 4 tokens, chosen by Expert-Choice routing that balances expert load by construction without any auxiliary loss, and applies rotary position embeddings and gated feed-forward networks throughout. Trained on the BabyLM 2026 Strict-Small corpus, our 52.6M-parameter model reaches an overall average of 38.42, surpassing the official GPT-2(98.4M) baseline (37.38) across grammatical acceptability, world-knowledge, and language-understanding benchmarks.
%U https://aclanthology.org/2026.babylm-main.60/
%P 767-778
Markdown (Informal)
[XpertGPT: Multi-Scale Sparse Expert Routing for Data-Constrained Language Modeling](https://aclanthology.org/2026.babylm-main.60/) (Jain et al., BabyLM 2026)
ACL