@inproceedings{mikat-fahim-2026-dense,
title = "Dense Supervision, Gentle Decay: Curriculum-Guided Rebalancing of {ELECTRA}-Style {RTD} and {MLM} for Small-Data Pretraining",
author = "Mikat, Aftab Hossain and
Fahim, Tamjid Hasan",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.66/",
pages = "859--874",
ISBN = "979-8-89176-475-0",
abstract = "When training data is very limited, the way different training signals are weighted over time can strongly affect what a model learns. We study hybrid replaced-token detection (RTD) and masked language modeling (MLM) pretraining under the BabyLM 2026 Strict-Small budget. We train five single-seed DeBERTa-v2 models along three curriculum axes: objective scheduling, masking policy, and data order. The primary system, bgb-decay, retains generator MLM, discriminator RTD, and auxiliary discriminator MLM throughout training while reducing the RTD coefficient in stages. We compare it with a matched fixed-RTD reference, a variant that adds easy-to-hard data ordering and adaptive auxiliary masking, and two stronger RTD-to-MLM handoff schedules with whole-word and adaptive masking. Among the five runs, bgb-decay achieves the best overall balance, while the more aggressive variants gain more on some tasks but drop more on others. Training diagnostics reveal a trade-off between replacement discrimination and auxiliary vocabulary prediction: under fixed high RTD weighting, RTD gradients are larger and nearly orthogonal to auxiliary-MLM gradients, while staged decay preserves RTD but shifts late optimization toward MLM. These results suggest that gradual objective rebalancing is a promising direction for small-data pretraining."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mikat-fahim-2026-dense">
<titleInfo>
<title>Dense Supervision, Gentle Decay: Curriculum-Guided Rebalancing of ELECTRA-Style RTD and MLM for Small-Data Pretraining</title>
</titleInfo>
<name type="personal">
<namePart type="given">Aftab</namePart>
<namePart type="given">Hossain</namePart>
<namePart type="family">Mikat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tamjid</namePart>
<namePart type="given">Hasan</namePart>
<namePart type="family">Fahim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>When training data is very limited, the way different training signals are weighted over time can strongly affect what a model learns. We study hybrid replaced-token detection (RTD) and masked language modeling (MLM) pretraining under the BabyLM 2026 Strict-Small budget. We train five single-seed DeBERTa-v2 models along three curriculum axes: objective scheduling, masking policy, and data order. The primary system, bgb-decay, retains generator MLM, discriminator RTD, and auxiliary discriminator MLM throughout training while reducing the RTD coefficient in stages. We compare it with a matched fixed-RTD reference, a variant that adds easy-to-hard data ordering and adaptive auxiliary masking, and two stronger RTD-to-MLM handoff schedules with whole-word and adaptive masking. Among the five runs, bgb-decay achieves the best overall balance, while the more aggressive variants gain more on some tasks but drop more on others. Training diagnostics reveal a trade-off between replacement discrimination and auxiliary vocabulary prediction: under fixed high RTD weighting, RTD gradients are larger and nearly orthogonal to auxiliary-MLM gradients, while staged decay preserves RTD but shifts late optimization toward MLM. These results suggest that gradual objective rebalancing is a promising direction for small-data pretraining.</abstract>
<identifier type="citekey">mikat-fahim-2026-dense</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.66/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>859</start>
<end>874</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Dense Supervision, Gentle Decay: Curriculum-Guided Rebalancing of ELECTRA-Style RTD and MLM for Small-Data Pretraining
%A Mikat, Aftab Hossain
%A Fahim, Tamjid Hasan
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F mikat-fahim-2026-dense
%X When training data is very limited, the way different training signals are weighted over time can strongly affect what a model learns. We study hybrid replaced-token detection (RTD) and masked language modeling (MLM) pretraining under the BabyLM 2026 Strict-Small budget. We train five single-seed DeBERTa-v2 models along three curriculum axes: objective scheduling, masking policy, and data order. The primary system, bgb-decay, retains generator MLM, discriminator RTD, and auxiliary discriminator MLM throughout training while reducing the RTD coefficient in stages. We compare it with a matched fixed-RTD reference, a variant that adds easy-to-hard data ordering and adaptive auxiliary masking, and two stronger RTD-to-MLM handoff schedules with whole-word and adaptive masking. Among the five runs, bgb-decay achieves the best overall balance, while the more aggressive variants gain more on some tasks but drop more on others. Training diagnostics reveal a trade-off between replacement discrimination and auxiliary vocabulary prediction: under fixed high RTD weighting, RTD gradients are larger and nearly orthogonal to auxiliary-MLM gradients, while staged decay preserves RTD but shifts late optimization toward MLM. These results suggest that gradual objective rebalancing is a promising direction for small-data pretraining.
%U https://aclanthology.org/2026.babylm-main.66/
%P 859-874
Markdown (Informal)
[Dense Supervision, Gentle Decay: Curriculum-Guided Rebalancing of ELECTRA-Style RTD and MLM for Small-Data Pretraining](https://aclanthology.org/2026.babylm-main.66/) (Mikat & Fahim, BabyLM 2026)
ACL