@inproceedings{liu-etal-2026-compact,
title = "Compact Multilingual Pre-training under the {B}aby{LM} 100{M}-Word Budget with Regex-Guided Tokenization and Attention Residuals",
author = "Liu, Yihan and
Zhang, Tongjing and
Nugues, Pierre",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.32/",
pages = "403--412",
ISBN = "979-8-89176-475-0",
abstract = "Multilingual pre-training under severe data con straints can suffer from limited representational capacity and uneven tokenization across lan guages. Addressing the BabyLM 2026 Multi lingual Track, we study a compact pre-training recipe under the 100M-word budget. Our main contribution is a Calibrated Regex-Guided Byte-Level BPE (Regex-BBPE) tokenizer that combines script-aware pre-tokenization with byte-premium-adjusted tokenizer training in a 16K vocabulary. We pair this tokenizer with the previously proposed Attention Residuals mechanism and the Muon optimizer, and eval uate three English/Dutch/Chinese mixing ra tios. Among the tested mixtures, the Chinese dominant configuration (20{\%} English, 20{\%} Dutch, 60{\%} Chinese) gives the strongest ag gregate fine-tuning results. Zero-shot results are mixed and remain slightly below the offi cial GPT-2 trilingual baseline on average, while fine-tuning improves the language-level aver ages by 8.55, 9.55, and 10.09 accuracy points for English, Dutch, and Chinese, respectively. The mixing-ratio experiments also show differ ent sensitivity patterns for the two West Ger manic languages and Chinese; we treat this as an observation within the tested language triple rather than a general cross-lingual law."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-compact">
<titleInfo>
<title>Compact Multilingual Pre-training under the BabyLM 100M-Word Budget with Regex-Guided Tokenization and Attention Residuals</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yihan</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tongjing</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pierre</namePart>
<namePart type="family">Nugues</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Multilingual pre-training under severe data con straints can suffer from limited representational capacity and uneven tokenization across lan guages. Addressing the BabyLM 2026 Multi lingual Track, we study a compact pre-training recipe under the 100M-word budget. Our main contribution is a Calibrated Regex-Guided Byte-Level BPE (Regex-BBPE) tokenizer that combines script-aware pre-tokenization with byte-premium-adjusted tokenizer training in a 16K vocabulary. We pair this tokenizer with the previously proposed Attention Residuals mechanism and the Muon optimizer, and eval uate three English/Dutch/Chinese mixing ra tios. Among the tested mixtures, the Chinese dominant configuration (20% English, 20% Dutch, 60% Chinese) gives the strongest ag gregate fine-tuning results. Zero-shot results are mixed and remain slightly below the offi cial GPT-2 trilingual baseline on average, while fine-tuning improves the language-level aver ages by 8.55, 9.55, and 10.09 accuracy points for English, Dutch, and Chinese, respectively. The mixing-ratio experiments also show differ ent sensitivity patterns for the two West Ger manic languages and Chinese; we treat this as an observation within the tested language triple rather than a general cross-lingual law.</abstract>
<identifier type="citekey">liu-etal-2026-compact</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.32/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>403</start>
<end>412</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Compact Multilingual Pre-training under the BabyLM 100M-Word Budget with Regex-Guided Tokenization and Attention Residuals
%A Liu, Yihan
%A Zhang, Tongjing
%A Nugues, Pierre
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F liu-etal-2026-compact
%X Multilingual pre-training under severe data con straints can suffer from limited representational capacity and uneven tokenization across lan guages. Addressing the BabyLM 2026 Multi lingual Track, we study a compact pre-training recipe under the 100M-word budget. Our main contribution is a Calibrated Regex-Guided Byte-Level BPE (Regex-BBPE) tokenizer that combines script-aware pre-tokenization with byte-premium-adjusted tokenizer training in a 16K vocabulary. We pair this tokenizer with the previously proposed Attention Residuals mechanism and the Muon optimizer, and eval uate three English/Dutch/Chinese mixing ra tios. Among the tested mixtures, the Chinese dominant configuration (20% English, 20% Dutch, 60% Chinese) gives the strongest ag gregate fine-tuning results. Zero-shot results are mixed and remain slightly below the offi cial GPT-2 trilingual baseline on average, while fine-tuning improves the language-level aver ages by 8.55, 9.55, and 10.09 accuracy points for English, Dutch, and Chinese, respectively. The mixing-ratio experiments also show differ ent sensitivity patterns for the two West Ger manic languages and Chinese; we treat this as an observation within the tested language triple rather than a general cross-lingual law.
%U https://aclanthology.org/2026.babylm-main.32/
%P 403-412
Markdown (Informal)
[Compact Multilingual Pre-training under the BabyLM 100M-Word Budget with Regex-Guided Tokenization and Attention Residuals](https://aclanthology.org/2026.babylm-main.32/) (Liu et al., BabyLM 2026)
ACL