@inproceedings{miles-2026-weight,
title = "Weight-Tied Looped Depth for Data-Constrained Pretraining",
author = "Miles, Sebastian",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.41/",
pages = "525--538",
ISBN = "979-8-89176-475-0",
abstract = "The BabyLM Strict-Small track fixes the training corpus at 10M words and the budget at 10 epochs. Data cannot be added, so any gain has to come from the architecture. We ask whether weight-tied looped depth, applying the same transformer block stack twice per forward pass, helps under that constraint. On a 50M-parameter model built from \texttt{nanochat} it does. Held-out bits-per-byte (\texttt{val{\_}bpb}, a per-byte cross-entropy that is comparable across tokenizers) falls from 1.3222 to 1.3175 as a three-seed mean, and every looped run lands below every baseline run, at no parameter cost. Downstream the effect points the same way, but we cannot resolve it: averaged over five zero-shot benchmarks under the official harness, the looped mean is 50.00 against the baseline{'}s 49.90, a gap of 0.10 points against a standard error of 0.17. The second pass is paid for in computation. Training takes $1.71\times$ as long, and a 100M-parameter model trained for a comparable wall-clock time still scores better on both metrics. What looping buys here is parameter efficiency, not compute efficiency.\par The baseline these experiments run on was tuned by a human-directed agentic loop. The researcher proposed each hypothesis; an agent implemented it, trained for the fixed budget around the clock on one GPU, and kept or reverted the change under a fixed rule on \texttt{val{\_}bpb}. We treat the loop as methodology and report its rejected hypotheses alongside the accepted ones."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="miles-2026-weight">
<titleInfo>
<title>Weight-Tied Looped Depth for Data-Constrained Pretraining</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sebastian</namePart>
<namePart type="family">Miles</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>The BabyLM Strict-Small track fixes the training corpus at 10M words and the budget at 10 epochs. Data cannot be added, so any gain has to come from the architecture. We ask whether weight-tied looped depth, applying the same transformer block stack twice per forward pass, helps under that constraint. On a 50M-parameter model built from nanochat it does. Held-out bits-per-byte (val_bpb, a per-byte cross-entropy that is comparable across tokenizers) falls from 1.3222 to 1.3175 as a three-seed mean, and every looped run lands below every baseline run, at no parameter cost. Downstream the effect points the same way, but we cannot resolve it: averaged over five zero-shot benchmarks under the official harness, the looped mean is 50.00 against the baseline’s 49.90, a gap of 0.10 points against a standard error of 0.17. The second pass is paid for in computation. Training takes 1.71\times as long, and a 100M-parameter model trained for a comparable wall-clock time still scores better on both metrics. What looping buys here is parameter efficiency, not compute efficiency.\par The baseline these experiments run on was tuned by a human-directed agentic loop. The researcher proposed each hypothesis; an agent implemented it, trained for the fixed budget around the clock on one GPU, and kept or reverted the change under a fixed rule on val_bpb. We treat the loop as methodology and report its rejected hypotheses alongside the accepted ones.</abstract>
<identifier type="citekey">miles-2026-weight</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.41/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>525</start>
<end>538</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Weight-Tied Looped Depth for Data-Constrained Pretraining
%A Miles, Sebastian
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F miles-2026-weight
%X The BabyLM Strict-Small track fixes the training corpus at 10M words and the budget at 10 epochs. Data cannot be added, so any gain has to come from the architecture. We ask whether weight-tied looped depth, applying the same transformer block stack twice per forward pass, helps under that constraint. On a 50M-parameter model built from nanochat it does. Held-out bits-per-byte (val_bpb, a per-byte cross-entropy that is comparable across tokenizers) falls from 1.3222 to 1.3175 as a three-seed mean, and every looped run lands below every baseline run, at no parameter cost. Downstream the effect points the same way, but we cannot resolve it: averaged over five zero-shot benchmarks under the official harness, the looped mean is 50.00 against the baseline’s 49.90, a gap of 0.10 points against a standard error of 0.17. The second pass is paid for in computation. Training takes 1.71\times as long, and a 100M-parameter model trained for a comparable wall-clock time still scores better on both metrics. What looping buys here is parameter efficiency, not compute efficiency.\par The baseline these experiments run on was tuned by a human-directed agentic loop. The researcher proposed each hypothesis; an agent implemented it, trained for the fixed budget around the clock on one GPU, and kept or reverted the change under a fixed rule on val_bpb. We treat the loop as methodology and report its rejected hypotheses alongside the accepted ones.
%U https://aclanthology.org/2026.babylm-main.41/
%P 525-538
Markdown (Informal)
[Weight-Tied Looped Depth for Data-Constrained Pretraining](https://aclanthology.org/2026.babylm-main.41/) (Miles, BabyLM 2026)
ACL