@inproceedings{yang-etal-2026-learning-combine,
title = "Learning What Can Combine and What Can Substitute: Relational Inductive Biases for {B}aby{LM} Pretraining",
author = "Yang, Yuwei and
Pu, Shihao and
Feng, Gefei and
Yang, Yiming",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.64/",
pages = "831--842",
ISBN = "979-8-89176-475-0",
abstract = "Language can produce many expressions from a limited set of units because linguistic units participate in two basic relations. Syntagmatic relations determine which units can combine, while paradigmatic relations determine which units can substitute for one another in the same position. We study whether explicitly learning these relations helps language models generalize from limited data. We introduce two auxiliary objectives for masked language modeling. The Syntagmatic objective aligns a masked representation with relevant context selected by attention, while the Paradigmatic objective aligns it with the embedding neighbors of the target token. Both objectives are generated by the model itself and require no additional data, annotation, or trainable parameters. Experiments on BabyLM 2026 Strict-Small show that both single-relation systems improve mean Overall performance over Standard MLM, with Paradigmatic giving the most consistent gains across three seeds. Relay Joint combines the two objectives and shows a strong neighborhood-alignment signal at its submitted checkpoint, although its downstream gains vary across seeds. These results suggest that relational objectives can provide useful learning signals under limited data, while stronger internal alignment does not always lead to more stable downstream performance. Code is now publicly available on GitHub."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yang-etal-2026-learning-combine">
<titleInfo>
<title>Learning What Can Combine and What Can Substitute: Relational Inductive Biases for BabyLM Pretraining</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuwei</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shihao</namePart>
<namePart type="family">Pu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gefei</namePart>
<namePart type="family">Feng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiming</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Language can produce many expressions from a limited set of units because linguistic units participate in two basic relations. Syntagmatic relations determine which units can combine, while paradigmatic relations determine which units can substitute for one another in the same position. We study whether explicitly learning these relations helps language models generalize from limited data. We introduce two auxiliary objectives for masked language modeling. The Syntagmatic objective aligns a masked representation with relevant context selected by attention, while the Paradigmatic objective aligns it with the embedding neighbors of the target token. Both objectives are generated by the model itself and require no additional data, annotation, or trainable parameters. Experiments on BabyLM 2026 Strict-Small show that both single-relation systems improve mean Overall performance over Standard MLM, with Paradigmatic giving the most consistent gains across three seeds. Relay Joint combines the two objectives and shows a strong neighborhood-alignment signal at its submitted checkpoint, although its downstream gains vary across seeds. These results suggest that relational objectives can provide useful learning signals under limited data, while stronger internal alignment does not always lead to more stable downstream performance. Code is now publicly available on GitHub.</abstract>
<identifier type="citekey">yang-etal-2026-learning-combine</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.64/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>831</start>
<end>842</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Learning What Can Combine and What Can Substitute: Relational Inductive Biases for BabyLM Pretraining
%A Yang, Yuwei
%A Pu, Shihao
%A Feng, Gefei
%A Yang, Yiming
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F yang-etal-2026-learning-combine
%X Language can produce many expressions from a limited set of units because linguistic units participate in two basic relations. Syntagmatic relations determine which units can combine, while paradigmatic relations determine which units can substitute for one another in the same position. We study whether explicitly learning these relations helps language models generalize from limited data. We introduce two auxiliary objectives for masked language modeling. The Syntagmatic objective aligns a masked representation with relevant context selected by attention, while the Paradigmatic objective aligns it with the embedding neighbors of the target token. Both objectives are generated by the model itself and require no additional data, annotation, or trainable parameters. Experiments on BabyLM 2026 Strict-Small show that both single-relation systems improve mean Overall performance over Standard MLM, with Paradigmatic giving the most consistent gains across three seeds. Relay Joint combines the two objectives and shows a strong neighborhood-alignment signal at its submitted checkpoint, although its downstream gains vary across seeds. These results suggest that relational objectives can provide useful learning signals under limited data, while stronger internal alignment does not always lead to more stable downstream performance. Code is now publicly available on GitHub.
%U https://aclanthology.org/2026.babylm-main.64/
%P 831-842
Markdown (Informal)
[Learning What Can Combine and What Can Substitute: Relational Inductive Biases for BabyLM Pretraining](https://aclanthology.org/2026.babylm-main.64/) (Yang et al., BabyLM 2026)
ACL