@inproceedings{haller-etal-2026-puzzle,
title = "From Puzzle-Solver to Language Model: Recurrent Inductive Bias, Data Scale, and Benchmark Validity in {B}aby{LM}",
author = "Haller, Patrick and
Laitenberger, Filipe and
Golde, Jonas and
Akbik, Alan",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.57/",
pages = "722--734",
ISBN = "979-8-89176-475-0",
abstract = "In the small-data regime of the BabyLM Challenge, a model{'}s architecture still shapes what it learns. We ask how much an inductive bias toward recurrence helps, and whether current benchmarks can detect that help. Our model adapts the Hierarchical Reasoning Model, a small recurrent puzzle-solver that reuses one shared block many times per token instead of stacking more layers. We make three changes to turn it into a language model: a sequence-mixing layer that can carry order-dependent state, a gate on one of the two connections between its modules, and a tunable number of recurrent cycles. The result matches a parameter-matched GPT-2, and we use it to study three questions. \textbf{Q1:} recurrence buys depth without adding parameters. On a controlled state-tracking task, running more cycles of the same weights tracks longer sequences, even when the extra cycles are added only at inference. \textbf{Q2:} the advantage is largest where data is scarcest. leads GPT-2 on BLiMP at words, ties at , and, trained to convergence, still leads at tokens (out of budget, where data and tokenizer also change). \textbf{Q3:} the benchmark meant to reward state-tracking cannot see it. A null model that never reads the puzzle outscores every trained model on \texttt{entity{\_}tracking}, because 28.5{\%} of its items reveal the answer in the option list and every trained model is at chance on the rest. Gains on that task measure how well a model has learned the frequency of one answer. Because the suite misses a capability we can demonstrate directly, we recommend pairing benchmarks with controlled architectural probes."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="haller-etal-2026-puzzle">
<titleInfo>
<title>From Puzzle-Solver to Language Model: Recurrent Inductive Bias, Data Scale, and Benchmark Validity in BabyLM</title>
</titleInfo>
<name type="personal">
<namePart type="given">Patrick</namePart>
<namePart type="family">Haller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Filipe</namePart>
<namePart type="family">Laitenberger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jonas</namePart>
<namePart type="family">Golde</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Akbik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>In the small-data regime of the BabyLM Challenge, a model’s architecture still shapes what it learns. We ask how much an inductive bias toward recurrence helps, and whether current benchmarks can detect that help. Our model adapts the Hierarchical Reasoning Model, a small recurrent puzzle-solver that reuses one shared block many times per token instead of stacking more layers. We make three changes to turn it into a language model: a sequence-mixing layer that can carry order-dependent state, a gate on one of the two connections between its modules, and a tunable number of recurrent cycles. The result matches a parameter-matched GPT-2, and we use it to study three questions. Q1: recurrence buys depth without adding parameters. On a controlled state-tracking task, running more cycles of the same weights tracks longer sequences, even when the extra cycles are added only at inference. Q2: the advantage is largest where data is scarcest. leads GPT-2 on BLiMP at words, ties at , and, trained to convergence, still leads at tokens (out of budget, where data and tokenizer also change). Q3: the benchmark meant to reward state-tracking cannot see it. A null model that never reads the puzzle outscores every trained model on entity_tracking, because 28.5% of its items reveal the answer in the option list and every trained model is at chance on the rest. Gains on that task measure how well a model has learned the frequency of one answer. Because the suite misses a capability we can demonstrate directly, we recommend pairing benchmarks with controlled architectural probes.</abstract>
<identifier type="citekey">haller-etal-2026-puzzle</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.57/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>722</start>
<end>734</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T From Puzzle-Solver to Language Model: Recurrent Inductive Bias, Data Scale, and Benchmark Validity in BabyLM
%A Haller, Patrick
%A Laitenberger, Filipe
%A Golde, Jonas
%A Akbik, Alan
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F haller-etal-2026-puzzle
%X In the small-data regime of the BabyLM Challenge, a model’s architecture still shapes what it learns. We ask how much an inductive bias toward recurrence helps, and whether current benchmarks can detect that help. Our model adapts the Hierarchical Reasoning Model, a small recurrent puzzle-solver that reuses one shared block many times per token instead of stacking more layers. We make three changes to turn it into a language model: a sequence-mixing layer that can carry order-dependent state, a gate on one of the two connections between its modules, and a tunable number of recurrent cycles. The result matches a parameter-matched GPT-2, and we use it to study three questions. Q1: recurrence buys depth without adding parameters. On a controlled state-tracking task, running more cycles of the same weights tracks longer sequences, even when the extra cycles are added only at inference. Q2: the advantage is largest where data is scarcest. leads GPT-2 on BLiMP at words, ties at , and, trained to convergence, still leads at tokens (out of budget, where data and tokenizer also change). Q3: the benchmark meant to reward state-tracking cannot see it. A null model that never reads the puzzle outscores every trained model on entity_tracking, because 28.5% of its items reveal the answer in the option list and every trained model is at chance on the rest. Gains on that task measure how well a model has learned the frequency of one answer. Because the suite misses a capability we can demonstrate directly, we recommend pairing benchmarks with controlled architectural probes.
%U https://aclanthology.org/2026.babylm-main.57/
%P 722-734
Markdown (Informal)
[From Puzzle-Solver to Language Model: Recurrent Inductive Bias, Data Scale, and Benchmark Validity in BabyLM](https://aclanthology.org/2026.babylm-main.57/) (Haller et al., BabyLM 2026)
ACL