@inproceedings{palomino-2026-exploring,
title = "Exploring Auxiliary Objectives through Multi-Task Learning for Small-Data Language Model Pretraining: A {B}aby{LM} 2026 Submission",
author = "Palomino, Alonso",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.38/",
pages = "493--503",
ISBN = "979-8-89176-475-0",
abstract = "Pretraining optimizes a language model with self-supervised objectives over raw text, shaping the representations later measured by diagnostic evaluations. Masked language modeling (MLM) predicts hidden tokens from context, combining the distributional hypothesis with the cloze task. Less is known about alternative objectives under small data budgets. Using BabyLM 2026{'}s 10M-token Strict-Small setting, this paper asks whether auxiliary objectives jointly improve grammar, world-knowledge plausibility, entity tracking, and reading prediction, or instead benefit some areas at the expense of others. The experiments compare an MLM-only baseline with several multi-task objective mixtures while holding the corpus, architecture, tokenizer, training budget, and evaluation pipeline fixed. Results are selective: MLM-only is strongest on grammatical minimal-pair judgments, while auxiliary objectives improve entity-state tracking and reading prediction. Within the controlled fast evaluation, single-mask semantic cloze gives the clearest directional world-knowledge plausibility gain; grammar repair does not recover grammatical performance, and broad plausibility classification produces no positive EWoK delta. The central finding is objective-interface alignment: auxiliary objectives transfer most clearly when their loss format resembles the diagnostic interface used at evaluation time. Auxiliary objectives therefore act as targeted steering signals rather than uniform improvements over MLM."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="palomino-2026-exploring">
<titleInfo>
<title>Exploring Auxiliary Objectives through Multi-Task Learning for Small-Data Language Model Pretraining: A BabyLM 2026 Submission</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alonso</namePart>
<namePart type="family">Palomino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Pretraining optimizes a language model with self-supervised objectives over raw text, shaping the representations later measured by diagnostic evaluations. Masked language modeling (MLM) predicts hidden tokens from context, combining the distributional hypothesis with the cloze task. Less is known about alternative objectives under small data budgets. Using BabyLM 2026’s 10M-token Strict-Small setting, this paper asks whether auxiliary objectives jointly improve grammar, world-knowledge plausibility, entity tracking, and reading prediction, or instead benefit some areas at the expense of others. The experiments compare an MLM-only baseline with several multi-task objective mixtures while holding the corpus, architecture, tokenizer, training budget, and evaluation pipeline fixed. Results are selective: MLM-only is strongest on grammatical minimal-pair judgments, while auxiliary objectives improve entity-state tracking and reading prediction. Within the controlled fast evaluation, single-mask semantic cloze gives the clearest directional world-knowledge plausibility gain; grammar repair does not recover grammatical performance, and broad plausibility classification produces no positive EWoK delta. The central finding is objective-interface alignment: auxiliary objectives transfer most clearly when their loss format resembles the diagnostic interface used at evaluation time. Auxiliary objectives therefore act as targeted steering signals rather than uniform improvements over MLM.</abstract>
<identifier type="citekey">palomino-2026-exploring</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.38/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>493</start>
<end>503</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Exploring Auxiliary Objectives through Multi-Task Learning for Small-Data Language Model Pretraining: A BabyLM 2026 Submission
%A Palomino, Alonso
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F palomino-2026-exploring
%X Pretraining optimizes a language model with self-supervised objectives over raw text, shaping the representations later measured by diagnostic evaluations. Masked language modeling (MLM) predicts hidden tokens from context, combining the distributional hypothesis with the cloze task. Less is known about alternative objectives under small data budgets. Using BabyLM 2026’s 10M-token Strict-Small setting, this paper asks whether auxiliary objectives jointly improve grammar, world-knowledge plausibility, entity tracking, and reading prediction, or instead benefit some areas at the expense of others. The experiments compare an MLM-only baseline with several multi-task objective mixtures while holding the corpus, architecture, tokenizer, training budget, and evaluation pipeline fixed. Results are selective: MLM-only is strongest on grammatical minimal-pair judgments, while auxiliary objectives improve entity-state tracking and reading prediction. Within the controlled fast evaluation, single-mask semantic cloze gives the clearest directional world-knowledge plausibility gain; grammar repair does not recover grammatical performance, and broad plausibility classification produces no positive EWoK delta. The central finding is objective-interface alignment: auxiliary objectives transfer most clearly when their loss format resembles the diagnostic interface used at evaluation time. Auxiliary objectives therefore act as targeted steering signals rather than uniform improvements over MLM.
%U https://aclanthology.org/2026.babylm-main.38/
%P 493-503
Markdown (Informal)
[Exploring Auxiliary Objectives through Multi-Task Learning for Small-Data Language Model Pretraining: A BabyLM 2026 Submission](https://aclanthology.org/2026.babylm-main.38/) (Palomino, BabyLM 2026)
ACL