@inproceedings{yang-etal-2026-optimizing,
title = "Optimizing Multilingual {B}aby{LM}: The Roles of Tokenization, Data Packing, and Curriculum Learning",
author = "Yang, Fuming and
Yang, Kai-hao and
Lien, Hsuan and
Edman, Lukas and
Fraser, Alexander",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.47/",
pages = "598--610",
ISBN = "979-8-89176-475-0",
abstract = "Training multilingual language models under strict data constraints remains a challenging problem. This paper presents our submission to the BabyLM 2026 Multilingual Track, focusing on English, Dutch, and Chinese. Three key challenges arise under the BabyLM constraints: efficiently sharing limited model capacity, maximizing sequence utilization, and allocating parameters efficiently under a fixed vocabulary budget. To address these challenges, we propose a three-stage curriculum learning strategy that dynamically adjusts language mixing ratios throughout training. Furthermore, we investigate data packing strategies, comparing naive chunking with document wrapping to maximize the learning signal within a strict 100M-token budget. Finally, we design a custom BPE tokenizer with an asymmetric sampling strategy. By reducing the size of the embedding matrix, the saved parameters are reallocated to the Transformer backbone, enabling highly efficient GPT-2 and BERT models. Experimental results show that combining our curriculum strategy with document packing consistently improves GPT-2 training convergence and achieves highly competitive multilingual performance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yang-etal-2026-optimizing">
<titleInfo>
<title>Optimizing Multilingual BabyLM: The Roles of Tokenization, Data Packing, and Curriculum Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Fuming</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kai-hao</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hsuan</namePart>
<namePart type="family">Lien</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lukas</namePart>
<namePart type="family">Edman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexander</namePart>
<namePart type="family">Fraser</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Training multilingual language models under strict data constraints remains a challenging problem. This paper presents our submission to the BabyLM 2026 Multilingual Track, focusing on English, Dutch, and Chinese. Three key challenges arise under the BabyLM constraints: efficiently sharing limited model capacity, maximizing sequence utilization, and allocating parameters efficiently under a fixed vocabulary budget. To address these challenges, we propose a three-stage curriculum learning strategy that dynamically adjusts language mixing ratios throughout training. Furthermore, we investigate data packing strategies, comparing naive chunking with document wrapping to maximize the learning signal within a strict 100M-token budget. Finally, we design a custom BPE tokenizer with an asymmetric sampling strategy. By reducing the size of the embedding matrix, the saved parameters are reallocated to the Transformer backbone, enabling highly efficient GPT-2 and BERT models. Experimental results show that combining our curriculum strategy with document packing consistently improves GPT-2 training convergence and achieves highly competitive multilingual performance.</abstract>
<identifier type="citekey">yang-etal-2026-optimizing</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.47/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>598</start>
<end>610</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Optimizing Multilingual BabyLM: The Roles of Tokenization, Data Packing, and Curriculum Learning
%A Yang, Fuming
%A Yang, Kai-hao
%A Lien, Hsuan
%A Edman, Lukas
%A Fraser, Alexander
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F yang-etal-2026-optimizing
%X Training multilingual language models under strict data constraints remains a challenging problem. This paper presents our submission to the BabyLM 2026 Multilingual Track, focusing on English, Dutch, and Chinese. Three key challenges arise under the BabyLM constraints: efficiently sharing limited model capacity, maximizing sequence utilization, and allocating parameters efficiently under a fixed vocabulary budget. To address these challenges, we propose a three-stage curriculum learning strategy that dynamically adjusts language mixing ratios throughout training. Furthermore, we investigate data packing strategies, comparing naive chunking with document wrapping to maximize the learning signal within a strict 100M-token budget. Finally, we design a custom BPE tokenizer with an asymmetric sampling strategy. By reducing the size of the embedding matrix, the saved parameters are reallocated to the Transformer backbone, enabling highly efficient GPT-2 and BERT models. Experimental results show that combining our curriculum strategy with document packing consistently improves GPT-2 training convergence and achieves highly competitive multilingual performance.
%U https://aclanthology.org/2026.babylm-main.47/
%P 598-610
Markdown (Informal)
[Optimizing Multilingual BabyLM: The Roles of Tokenization, Data Packing, and Curriculum Learning](https://aclanthology.org/2026.babylm-main.47/) (Yang et al., BabyLM 2026)
ACL