@inproceedings{rodriguez-2026-tokens,
title = "Tokens of Meaning: Semantically-Informed Morphological Tokenization in Low-Resource Language Modeling",
author = "Rodriguez, Juan Diego",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.67/",
pages = "875--891",
ISBN = "979-8-89176-475-0",
abstract = "Subword tokenizers for language models are typically trained to efficiently encode text into short sequences of reusable units. These often produce tokens which are not linguistically meaningful, such as giraffe {--}{\ensuremath{>}} g ira ffe. We ask whether a tokenizer that instead splits words into semantically meaningful morphemes can help a language model learn under a developmentally plausible data budget, where every training token is scarce and sharing evidence across related word forms should matter most. We build such a tokenizer for English by hand-curating a word-to-morpheme-segmentation dictionary, built on top of MorphyNet, a database of derivational and inflectional morphology. Each word is only split into morphemes when the pieces are productive and the whole word{'}s meaning is semantically related to them, thus splitting unhappiness into un happy ness, but keeping business whole. On the 10M-word BabyLM corpus this holds out-of-vocabulary tokens to 1.0{\%}, with an inventory of {\textasciitilde}37k tokens. We compare the tokenizer intrinsically against both statistical (BPE, Unigram, WordPiece) and morphology-aware (Morfessor, MorphPiece) baselines, and measure its effect on a child-scale language model. In a controlled comparison against an uncurated baseline built from the same morphological source, curation yields +1.4 BLiMP. Our submitted model, which adds masked next-token prediction and vocabulary pruning, reaches 72.53 BLiMP and ranks second on COMPS and third on EWoK on the BabyLM 2026 strict-small leaderboard."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rodriguez-2026-tokens">
<titleInfo>
<title>Tokens of Meaning: Semantically-Informed Morphological Tokenization in Low-Resource Language Modeling</title>
</titleInfo>
<name type="personal">
<namePart type="given">Juan</namePart>
<namePart type="given">Diego</namePart>
<namePart type="family">Rodriguez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Subword tokenizers for language models are typically trained to efficiently encode text into short sequences of reusable units. These often produce tokens which are not linguistically meaningful, such as giraffe –\ensuremath> g ira ffe. We ask whether a tokenizer that instead splits words into semantically meaningful morphemes can help a language model learn under a developmentally plausible data budget, where every training token is scarce and sharing evidence across related word forms should matter most. We build such a tokenizer for English by hand-curating a word-to-morpheme-segmentation dictionary, built on top of MorphyNet, a database of derivational and inflectional morphology. Each word is only split into morphemes when the pieces are productive and the whole word’s meaning is semantically related to them, thus splitting unhappiness into un happy ness, but keeping business whole. On the 10M-word BabyLM corpus this holds out-of-vocabulary tokens to 1.0%, with an inventory of ~37k tokens. We compare the tokenizer intrinsically against both statistical (BPE, Unigram, WordPiece) and morphology-aware (Morfessor, MorphPiece) baselines, and measure its effect on a child-scale language model. In a controlled comparison against an uncurated baseline built from the same morphological source, curation yields +1.4 BLiMP. Our submitted model, which adds masked next-token prediction and vocabulary pruning, reaches 72.53 BLiMP and ranks second on COMPS and third on EWoK on the BabyLM 2026 strict-small leaderboard.</abstract>
<identifier type="citekey">rodriguez-2026-tokens</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.67/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>875</start>
<end>891</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Tokens of Meaning: Semantically-Informed Morphological Tokenization in Low-Resource Language Modeling
%A Rodriguez, Juan Diego
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F rodriguez-2026-tokens
%X Subword tokenizers for language models are typically trained to efficiently encode text into short sequences of reusable units. These often produce tokens which are not linguistically meaningful, such as giraffe –\ensuremath> g ira ffe. We ask whether a tokenizer that instead splits words into semantically meaningful morphemes can help a language model learn under a developmentally plausible data budget, where every training token is scarce and sharing evidence across related word forms should matter most. We build such a tokenizer for English by hand-curating a word-to-morpheme-segmentation dictionary, built on top of MorphyNet, a database of derivational and inflectional morphology. Each word is only split into morphemes when the pieces are productive and the whole word’s meaning is semantically related to them, thus splitting unhappiness into un happy ness, but keeping business whole. On the 10M-word BabyLM corpus this holds out-of-vocabulary tokens to 1.0%, with an inventory of ~37k tokens. We compare the tokenizer intrinsically against both statistical (BPE, Unigram, WordPiece) and morphology-aware (Morfessor, MorphPiece) baselines, and measure its effect on a child-scale language model. In a controlled comparison against an uncurated baseline built from the same morphological source, curation yields +1.4 BLiMP. Our submitted model, which adds masked next-token prediction and vocabulary pruning, reaches 72.53 BLiMP and ranks second on COMPS and third on EWoK on the BabyLM 2026 strict-small leaderboard.
%U https://aclanthology.org/2026.babylm-main.67/
%P 875-891
Markdown (Informal)
[Tokens of Meaning: Semantically-Informed Morphological Tokenization in Low-Resource Language Modeling](https://aclanthology.org/2026.babylm-main.67/) (Rodriguez, BabyLM 2026)
ACL