@inproceedings{oytac-2026-trlm,
title = "{TRLM}: Shared-Weight Recursive Reasoning for Non-Autoregressive Language Generation",
author = "Oyta{\c{c}}, Do{\u{g}}a",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.29/",
pages = "358--368",
ISBN = "979-8-89176-475-0",
abstract = "Large language models achieve strong reasoning but require substantial parameters and memory, limiting deployment in resource-constrained settings. We introduce the Tiny Recursive Language Model (TRLM), a small masked non-autoregressive model that increases effective computation depth through shared-weight recursion. TRLM represents a masked sequence with input, answer, and latent reasoning states. A shared Transformer repeatedly refines the latent reasoning state, conditioning each update on trajectory time and step size via adaptive layer normalization. Training combines long- and short-trajectory answer-prediction losses with representation-level consistency. At inference, TRLM recursively refines latent states and fills all masked answer tokens in a single parallel decoding pass. We pretrain TRLM on approximately one billion OpenWebText and FineWeb-Edu tokens, then fine-tune and evaluate it on GSM8K and SVAMP against scratch-pretrained GPT-2-style, Gemma-4-Text-style, Diffusion-Gemma-style, and Mamba-2-style baselines. TRLM matches the best observed GSM8K result with 2.88{\%} exact-match accuracy and achieves the highest SVAMP point estimate in our controlled comparison at 9.00{\%}. Performance is non-monotonic with recursion depth: the highest observed GSM8K accuracy occurs at eight steps, whereas SVAMP peaks at four, suggesting that the preferred recursion depth may depend on the task in this setting."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="oytac-2026-trlm">
<titleInfo>
<title>TRLM: Shared-Weight Recursive Reasoning for Non-Autoregressive Language Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Doğa</namePart>
<namePart type="family">Oytaç</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Large language models achieve strong reasoning but require substantial parameters and memory, limiting deployment in resource-constrained settings. We introduce the Tiny Recursive Language Model (TRLM), a small masked non-autoregressive model that increases effective computation depth through shared-weight recursion. TRLM represents a masked sequence with input, answer, and latent reasoning states. A shared Transformer repeatedly refines the latent reasoning state, conditioning each update on trajectory time and step size via adaptive layer normalization. Training combines long- and short-trajectory answer-prediction losses with representation-level consistency. At inference, TRLM recursively refines latent states and fills all masked answer tokens in a single parallel decoding pass. We pretrain TRLM on approximately one billion OpenWebText and FineWeb-Edu tokens, then fine-tune and evaluate it on GSM8K and SVAMP against scratch-pretrained GPT-2-style, Gemma-4-Text-style, Diffusion-Gemma-style, and Mamba-2-style baselines. TRLM matches the best observed GSM8K result with 2.88% exact-match accuracy and achieves the highest SVAMP point estimate in our controlled comparison at 9.00%. Performance is non-monotonic with recursion depth: the highest observed GSM8K accuracy occurs at eight steps, whereas SVAMP peaks at four, suggesting that the preferred recursion depth may depend on the task in this setting.</abstract>
<identifier type="citekey">oytac-2026-trlm</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.29/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>358</start>
<end>368</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T TRLM: Shared-Weight Recursive Reasoning for Non-Autoregressive Language Generation
%A Oytaç, Doğa
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F oytac-2026-trlm
%X Large language models achieve strong reasoning but require substantial parameters and memory, limiting deployment in resource-constrained settings. We introduce the Tiny Recursive Language Model (TRLM), a small masked non-autoregressive model that increases effective computation depth through shared-weight recursion. TRLM represents a masked sequence with input, answer, and latent reasoning states. A shared Transformer repeatedly refines the latent reasoning state, conditioning each update on trajectory time and step size via adaptive layer normalization. Training combines long- and short-trajectory answer-prediction losses with representation-level consistency. At inference, TRLM recursively refines latent states and fills all masked answer tokens in a single parallel decoding pass. We pretrain TRLM on approximately one billion OpenWebText and FineWeb-Edu tokens, then fine-tune and evaluate it on GSM8K and SVAMP against scratch-pretrained GPT-2-style, Gemma-4-Text-style, Diffusion-Gemma-style, and Mamba-2-style baselines. TRLM matches the best observed GSM8K result with 2.88% exact-match accuracy and achieves the highest SVAMP point estimate in our controlled comparison at 9.00%. Performance is non-monotonic with recursion depth: the highest observed GSM8K accuracy occurs at eight steps, whereas SVAMP peaks at four, suggesting that the preferred recursion depth may depend on the task in this setting.
%U https://aclanthology.org/2026.babylm-main.29/
%P 358-368
Markdown (Informal)
[TRLM: Shared-Weight Recursive Reasoning for Non-Autoregressive Language Generation](https://aclanthology.org/2026.babylm-main.29/) (Oytaç, BabyLM 2026)
ACL