@inproceedings{furr-2026-four,
title = "Four Kilobytes of Memory: A 10{M}-Parameter Fixed-State Recurrent {LM} for {B}aby{LM} Strict-Small",
author = "Furr, Sam",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.40/",
pages = "514--524",
ISBN = "979-8-89176-475-0",
abstract = "How much language competence can a small recurrent model acquire when it must compress its context into a fixed-size state? We study this question with StateHead, a gated recurrent language model without attention or positional embeddings. Our BabyLM 2026 Strict-Small submission has 9.97M parameters and 2,160 recurrent state values, equivalent to 4.2 KiB in bf16. After 1.1 hours of training on a laptop, it scores 68.2 on BLiMP, compared with 65.2 for the official 98.4M-parameter GPT-2 baseline on the same benchmark suite. This advantage is specific to grammatical discrimination: StateHead is weaker on reading-time prediction and scores near or below chance on several semantic and entity-tracking evaluations. We characterize its computation and training trajectory, and compare recipes that reset or carry state across training chunks. The apparent entity-tracking gain is confined to items whose correct answer is ``nothing''; both recipes score below chance on the revised benchmark. These results document both the useful capabilities and the limits of a language model with a very small recurrent memory."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="furr-2026-four">
<titleInfo>
<title>Four Kilobytes of Memory: A 10M-Parameter Fixed-State Recurrent LM for BabyLM Strict-Small</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sam</namePart>
<namePart type="family">Furr</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>How much language competence can a small recurrent model acquire when it must compress its context into a fixed-size state? We study this question with StateHead, a gated recurrent language model without attention or positional embeddings. Our BabyLM 2026 Strict-Small submission has 9.97M parameters and 2,160 recurrent state values, equivalent to 4.2 KiB in bf16. After 1.1 hours of training on a laptop, it scores 68.2 on BLiMP, compared with 65.2 for the official 98.4M-parameter GPT-2 baseline on the same benchmark suite. This advantage is specific to grammatical discrimination: StateHead is weaker on reading-time prediction and scores near or below chance on several semantic and entity-tracking evaluations. We characterize its computation and training trajectory, and compare recipes that reset or carry state across training chunks. The apparent entity-tracking gain is confined to items whose correct answer is “nothing”; both recipes score below chance on the revised benchmark. These results document both the useful capabilities and the limits of a language model with a very small recurrent memory.</abstract>
<identifier type="citekey">furr-2026-four</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.40/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>514</start>
<end>524</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Four Kilobytes of Memory: A 10M-Parameter Fixed-State Recurrent LM for BabyLM Strict-Small
%A Furr, Sam
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F furr-2026-four
%X How much language competence can a small recurrent model acquire when it must compress its context into a fixed-size state? We study this question with StateHead, a gated recurrent language model without attention or positional embeddings. Our BabyLM 2026 Strict-Small submission has 9.97M parameters and 2,160 recurrent state values, equivalent to 4.2 KiB in bf16. After 1.1 hours of training on a laptop, it scores 68.2 on BLiMP, compared with 65.2 for the official 98.4M-parameter GPT-2 baseline on the same benchmark suite. This advantage is specific to grammatical discrimination: StateHead is weaker on reading-time prediction and scores near or below chance on several semantic and entity-tracking evaluations. We characterize its computation and training trajectory, and compare recipes that reset or carry state across training chunks. The apparent entity-tracking gain is confined to items whose correct answer is “nothing”; both recipes score below chance on the revised benchmark. These results document both the useful capabilities and the limits of a language model with a very small recurrent memory.
%U https://aclanthology.org/2026.babylm-main.40/
%P 514-524
Markdown (Informal)
[Four Kilobytes of Memory: A 10M-Parameter Fixed-State Recurrent LM for BabyLM Strict-Small](https://aclanthology.org/2026.babylm-main.40/) (Furr, BabyLM 2026)
ACL