@inproceedings{laitenberger-etal-2026-survives,
title = "What Survives the {B}aby{LM} Budget? A Controlled Study of Token Mixers and Generative Objectives",
author = "Laitenberger, Filipe and
Haller, Patrick and
Golde, Jonas and
Akbik, Alan",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.7/",
pages = "75--92",
ISBN = "979-8-89176-475-0",
abstract = "BabyLM is a pressure test for language-modeling assumptions: which ideas still work when the data budget is closer to a child{'}s linguistic input than to web-scale pretraining? We use the 2026 English tracks to ask this question in a controlled way, holding the corpus, tokenizer, backbone size family, training schedule, and public evaluation wrapper fixed while varying two token mixers and six objectives. Across 432 completed scale-grid runs on the 10M-word Strict-Small and 100M-word Strict tracks, the strongest result is also the most conservative one. Autoregressive Transformers are the best public-evaluation models at every tested scale, reaching peak fine-tuning macro scores of 64.88 and 66.94 and peak BLiMP scores of 64.70 and 75.13. Masked language modeling is the strongest non-autoregressive baseline. The negative results are informative: Gated DeltaNet improves Entity Tracking but loses grammar and broad transfer, while diffusion and flow objectives show local advantages in probes that rarely survive the official scoring interface. The lesson for BabyLM is not that newer objectives or recurrent mixers are unhelpful, but that small-data gains depend on matching the objective, architecture, sampler, and scorer."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="laitenberger-etal-2026-survives">
<titleInfo>
<title>What Survives the BabyLM Budget? A Controlled Study of Token Mixers and Generative Objectives</title>
</titleInfo>
<name type="personal">
<namePart type="given">Filipe</namePart>
<namePart type="family">Laitenberger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Patrick</namePart>
<namePart type="family">Haller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jonas</namePart>
<namePart type="family">Golde</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Akbik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>BabyLM is a pressure test for language-modeling assumptions: which ideas still work when the data budget is closer to a child’s linguistic input than to web-scale pretraining? We use the 2026 English tracks to ask this question in a controlled way, holding the corpus, tokenizer, backbone size family, training schedule, and public evaluation wrapper fixed while varying two token mixers and six objectives. Across 432 completed scale-grid runs on the 10M-word Strict-Small and 100M-word Strict tracks, the strongest result is also the most conservative one. Autoregressive Transformers are the best public-evaluation models at every tested scale, reaching peak fine-tuning macro scores of 64.88 and 66.94 and peak BLiMP scores of 64.70 and 75.13. Masked language modeling is the strongest non-autoregressive baseline. The negative results are informative: Gated DeltaNet improves Entity Tracking but loses grammar and broad transfer, while diffusion and flow objectives show local advantages in probes that rarely survive the official scoring interface. The lesson for BabyLM is not that newer objectives or recurrent mixers are unhelpful, but that small-data gains depend on matching the objective, architecture, sampler, and scorer.</abstract>
<identifier type="citekey">laitenberger-etal-2026-survives</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.7/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>75</start>
<end>92</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Survives the BabyLM Budget? A Controlled Study of Token Mixers and Generative Objectives
%A Laitenberger, Filipe
%A Haller, Patrick
%A Golde, Jonas
%A Akbik, Alan
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F laitenberger-etal-2026-survives
%X BabyLM is a pressure test for language-modeling assumptions: which ideas still work when the data budget is closer to a child’s linguistic input than to web-scale pretraining? We use the 2026 English tracks to ask this question in a controlled way, holding the corpus, tokenizer, backbone size family, training schedule, and public evaluation wrapper fixed while varying two token mixers and six objectives. Across 432 completed scale-grid runs on the 10M-word Strict-Small and 100M-word Strict tracks, the strongest result is also the most conservative one. Autoregressive Transformers are the best public-evaluation models at every tested scale, reaching peak fine-tuning macro scores of 64.88 and 66.94 and peak BLiMP scores of 64.70 and 75.13. Masked language modeling is the strongest non-autoregressive baseline. The negative results are informative: Gated DeltaNet improves Entity Tracking but loses grammar and broad transfer, while diffusion and flow objectives show local advantages in probes that rarely survive the official scoring interface. The lesson for BabyLM is not that newer objectives or recurrent mixers are unhelpful, but that small-data gains depend on matching the objective, architecture, sampler, and scorer.
%U https://aclanthology.org/2026.babylm-main.7/
%P 75-92
Markdown (Informal)
[What Survives the BabyLM Budget? A Controlled Study of Token Mixers and Generative Objectives](https://aclanthology.org/2026.babylm-main.7/) (Laitenberger et al., BabyLM 2026)
ACL