@inproceedings{benedikt-etal-2026-data2vec,
title = "Does a {D}ata2vec Auxiliary Help Small Masked Language Models? A {B}aby{LM}-2026 Strict-Small Study",
author = "Benedikt, Martin and
Dost{\'a}l, Ivan and
Klubarski, Robin",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.50/",
pages = "631--638",
ISBN = "979-8-89176-475-0",
abstract = "We ask whether a low-weight data2vec-style Exponential Moving Average (EMA) self-distillation auxiliary ($\lambda{=}0.1$, $\tau{=}0.999$) improves a small masked language model at the BabyLM-2026 Strict-Small scale. Our base model is a 33M-parameter LTG-BERT encoder with GPT-BERT-style hybrid masked/causal training, trained from scratch on the official 10M-word corpus in 2.3 GPU-hours on a single NVIDIA L40S. A substantial gain in Entity-Tracking during early development from adding the auxiliary motivated the study, but disappeared in later re-evaluation. In a comparison across 6 seeds, no evaluation task changed significantly (aggregate effect $p{=}0.29$), and the early gain is best explained by seed noise. Several variants intended to strengthen the auxiliary do no better. We attribute the model{'}s competitive scores to its hybrid training recipe and to EMA weight-averaging, both already established."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="benedikt-etal-2026-data2vec">
<titleInfo>
<title>Does a Data2vec Auxiliary Help Small Masked Language Models? A BabyLM-2026 Strict-Small Study</title>
</titleInfo>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Benedikt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ivan</namePart>
<namePart type="family">Dostál</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robin</namePart>
<namePart type="family">Klubarski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>We ask whether a low-weight data2vec-style Exponential Moving Average (EMA) self-distillation auxiliary (łambda=0.1, τ=0.999) improves a small masked language model at the BabyLM-2026 Strict-Small scale. Our base model is a 33M-parameter LTG-BERT encoder with GPT-BERT-style hybrid masked/causal training, trained from scratch on the official 10M-word corpus in 2.3 GPU-hours on a single NVIDIA L40S. A substantial gain in Entity-Tracking during early development from adding the auxiliary motivated the study, but disappeared in later re-evaluation. In a comparison across 6 seeds, no evaluation task changed significantly (aggregate effect p=0.29), and the early gain is best explained by seed noise. Several variants intended to strengthen the auxiliary do no better. We attribute the model’s competitive scores to its hybrid training recipe and to EMA weight-averaging, both already established.</abstract>
<identifier type="citekey">benedikt-etal-2026-data2vec</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.50/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>631</start>
<end>638</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Does a Data2vec Auxiliary Help Small Masked Language Models? A BabyLM-2026 Strict-Small Study
%A Benedikt, Martin
%A Dostál, Ivan
%A Klubarski, Robin
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F benedikt-etal-2026-data2vec
%X We ask whether a low-weight data2vec-style Exponential Moving Average (EMA) self-distillation auxiliary (łambda=0.1, τ=0.999) improves a small masked language model at the BabyLM-2026 Strict-Small scale. Our base model is a 33M-parameter LTG-BERT encoder with GPT-BERT-style hybrid masked/causal training, trained from scratch on the official 10M-word corpus in 2.3 GPU-hours on a single NVIDIA L40S. A substantial gain in Entity-Tracking during early development from adding the auxiliary motivated the study, but disappeared in later re-evaluation. In a comparison across 6 seeds, no evaluation task changed significantly (aggregate effect p=0.29), and the early gain is best explained by seed noise. Several variants intended to strengthen the auxiliary do no better. We attribute the model’s competitive scores to its hybrid training recipe and to EMA weight-averaging, both already established.
%U https://aclanthology.org/2026.babylm-main.50/
%P 631-638
Markdown (Informal)
[Does a Data2vec Auxiliary Help Small Masked Language Models? A BabyLM-2026 Strict-Small Study](https://aclanthology.org/2026.babylm-main.50/) (Benedikt et al., BabyLM 2026)
ACL