@inproceedings{satheesh-etal-2026-babysteps,
title = "{B}aby{S}teps at {B}aby{LM} 2026: Stronger Optimization and Causal Baselines for Data-Constrained Pretraining",
author = "Satheesh, Siddhi and
Murillo, Jorge C and
Chen, Yubei",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.63/",
pages = "823--830",
ISBN = "979-8-89176-475-0",
abstract = "We present three BabySteps systems that strengthen baselines for data-constrained pretraining in the strict-small (10M words) and strict (100M words) NLP tracks of the 2026 BabyLM Challenge. Across our GPT-BERT experiments, optimizer choice and learning-rate tuning produced our largest gains, highlighting the importance of well-tuned baselines for evaluating progress in sample-efficient pretraining. The curriculum, synthetic-data, teacher-supervision, and corpus-mixing interventions that we tested did not improve the final aggregate score.\par We also test whether the official GPT-2 checkpoint is a sufficiently strong reference for causal language modeling. A 127.6M-parameter Llama-style decoder trained with Muon and Liger-optimized kernels obtains a seven-suite NLP average of 56.18, compared with 52.79 for our completed reevaluation of the official BabyLM GPT-2 strict checkpoint, and scores higher on all seven categories. This is not a controlled architectural ablation: architecture, optimizer, implementation, and hyperparameters change together. Instead, it provides a stronger practical causal baseline. In addition, we report failed data interventions, a numerical failure caused by batches with no supervised target tokens, and substantial variance between small evaluation suites."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="satheesh-etal-2026-babysteps">
<titleInfo>
<title>BabySteps at BabyLM 2026: Stronger Optimization and Causal Baselines for Data-Constrained Pretraining</title>
</titleInfo>
<name type="personal">
<namePart type="given">Siddhi</namePart>
<namePart type="family">Satheesh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jorge</namePart>
<namePart type="given">C</namePart>
<namePart type="family">Murillo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yubei</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>We present three BabySteps systems that strengthen baselines for data-constrained pretraining in the strict-small (10M words) and strict (100M words) NLP tracks of the 2026 BabyLM Challenge. Across our GPT-BERT experiments, optimizer choice and learning-rate tuning produced our largest gains, highlighting the importance of well-tuned baselines for evaluating progress in sample-efficient pretraining. The curriculum, synthetic-data, teacher-supervision, and corpus-mixing interventions that we tested did not improve the final aggregate score.\par We also test whether the official GPT-2 checkpoint is a sufficiently strong reference for causal language modeling. A 127.6M-parameter Llama-style decoder trained with Muon and Liger-optimized kernels obtains a seven-suite NLP average of 56.18, compared with 52.79 for our completed reevaluation of the official BabyLM GPT-2 strict checkpoint, and scores higher on all seven categories. This is not a controlled architectural ablation: architecture, optimizer, implementation, and hyperparameters change together. Instead, it provides a stronger practical causal baseline. In addition, we report failed data interventions, a numerical failure caused by batches with no supervised target tokens, and substantial variance between small evaluation suites.</abstract>
<identifier type="citekey">satheesh-etal-2026-babysteps</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.63/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>823</start>
<end>830</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T BabySteps at BabyLM 2026: Stronger Optimization and Causal Baselines for Data-Constrained Pretraining
%A Satheesh, Siddhi
%A Murillo, Jorge C.
%A Chen, Yubei
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F satheesh-etal-2026-babysteps
%X We present three BabySteps systems that strengthen baselines for data-constrained pretraining in the strict-small (10M words) and strict (100M words) NLP tracks of the 2026 BabyLM Challenge. Across our GPT-BERT experiments, optimizer choice and learning-rate tuning produced our largest gains, highlighting the importance of well-tuned baselines for evaluating progress in sample-efficient pretraining. The curriculum, synthetic-data, teacher-supervision, and corpus-mixing interventions that we tested did not improve the final aggregate score.\par We also test whether the official GPT-2 checkpoint is a sufficiently strong reference for causal language modeling. A 127.6M-parameter Llama-style decoder trained with Muon and Liger-optimized kernels obtains a seven-suite NLP average of 56.18, compared with 52.79 for our completed reevaluation of the official BabyLM GPT-2 strict checkpoint, and scores higher on all seven categories. This is not a controlled architectural ablation: architecture, optimizer, implementation, and hyperparameters change together. Instead, it provides a stronger practical causal baseline. In addition, we report failed data interventions, a numerical failure caused by batches with no supervised target tokens, and substantial variance between small evaluation suites.
%U https://aclanthology.org/2026.babylm-main.63/
%P 823-830
Markdown (Informal)
[BabySteps at BabyLM 2026: Stronger Optimization and Causal Baselines for Data-Constrained Pretraining](https://aclanthology.org/2026.babylm-main.63/) (Satheesh et al., BabyLM 2026)
ACL