@inproceedings{taylor-2026-hard,
title = "Hard-Wiring the Induction Head: The Surprising Effectiveness of Copy Mechanisms in Data-Efficient Language Modelling",
author = "Taylor, Josh",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.3/",
pages = "24--32",
ISBN = "979-8-89176-475-0",
abstract = "Large language models need several orders of magnitude more text than a child hears to reach a comparable level of linguistic skill. One reason is that a particularly useful capability, the induction head (a circuit that completes a pattern of the form [A] [B] ... [A] by copying [B]), is not built in but must emerge during training, and does so only after the model has read a great deal of text. We ask whether supplying this capability directly, as a fixed part of the architecture, improves learning when data is scarce. We present a three-layer, attention-free language model of about 12M parameters in which all long-range mixing is performed by a non-learned, sub-quadratic induction operator (its cost grows more slowly than the square of the sequence length): for each position it finds recent earlier occurrences of the same token and copies the continuation of the best match. Trained on the 10M-word BabyLM Strict-Small corpus, the model learns markedly faster than a parameter-matched self-attention baseline and stays competitive at convergence; on the public BabyLM evaluation it is competitive with the challenge{'}s GPT-2 baselines on grammatical minimal pairs with a tenth of the parameters. Targeted interventions show the operator is used as intended: predictions travel along its links, the links are necessary, and stacking layers composes them into two-step retrieval. These results motivate hybrid designs that interleave exact look-back with attention."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="taylor-2026-hard">
<titleInfo>
<title>Hard-Wiring the Induction Head: The Surprising Effectiveness of Copy Mechanisms in Data-Efficient Language Modelling</title>
</titleInfo>
<name type="personal">
<namePart type="given">Josh</namePart>
<namePart type="family">Taylor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Large language models need several orders of magnitude more text than a child hears to reach a comparable level of linguistic skill. One reason is that a particularly useful capability, the induction head (a circuit that completes a pattern of the form [A] [B] ... [A] by copying [B]), is not built in but must emerge during training, and does so only after the model has read a great deal of text. We ask whether supplying this capability directly, as a fixed part of the architecture, improves learning when data is scarce. We present a three-layer, attention-free language model of about 12M parameters in which all long-range mixing is performed by a non-learned, sub-quadratic induction operator (its cost grows more slowly than the square of the sequence length): for each position it finds recent earlier occurrences of the same token and copies the continuation of the best match. Trained on the 10M-word BabyLM Strict-Small corpus, the model learns markedly faster than a parameter-matched self-attention baseline and stays competitive at convergence; on the public BabyLM evaluation it is competitive with the challenge’s GPT-2 baselines on grammatical minimal pairs with a tenth of the parameters. Targeted interventions show the operator is used as intended: predictions travel along its links, the links are necessary, and stacking layers composes them into two-step retrieval. These results motivate hybrid designs that interleave exact look-back with attention.</abstract>
<identifier type="citekey">taylor-2026-hard</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.3/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>24</start>
<end>32</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Hard-Wiring the Induction Head: The Surprising Effectiveness of Copy Mechanisms in Data-Efficient Language Modelling
%A Taylor, Josh
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F taylor-2026-hard
%X Large language models need several orders of magnitude more text than a child hears to reach a comparable level of linguistic skill. One reason is that a particularly useful capability, the induction head (a circuit that completes a pattern of the form [A] [B] ... [A] by copying [B]), is not built in but must emerge during training, and does so only after the model has read a great deal of text. We ask whether supplying this capability directly, as a fixed part of the architecture, improves learning when data is scarce. We present a three-layer, attention-free language model of about 12M parameters in which all long-range mixing is performed by a non-learned, sub-quadratic induction operator (its cost grows more slowly than the square of the sequence length): for each position it finds recent earlier occurrences of the same token and copies the continuation of the best match. Trained on the 10M-word BabyLM Strict-Small corpus, the model learns markedly faster than a parameter-matched self-attention baseline and stays competitive at convergence; on the public BabyLM evaluation it is competitive with the challenge’s GPT-2 baselines on grammatical minimal pairs with a tenth of the parameters. Targeted interventions show the operator is used as intended: predictions travel along its links, the links are necessary, and stacking layers composes them into two-step retrieval. These results motivate hybrid designs that interleave exact look-back with attention.
%U https://aclanthology.org/2026.babylm-main.3/
%P 24-32
Markdown (Informal)
[Hard-Wiring the Induction Head: The Surprising Effectiveness of Copy Mechanisms in Data-Efficient Language Modelling](https://aclanthology.org/2026.babylm-main.3/) (Taylor, BabyLM 2026)
ACL