@inproceedings{ouyang-etal-2026-test,
title = "Test-Time Training with Next-Token Prediction",
author = "Ouyang, Xuan and
Cai, Zefan and
Hu, Junjie",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.936/",
pages = "19750--19764",
ISBN = "979-8-89176-462-0",
abstract = "Next-token prediction is the self-supervised signal that trains language models, and every observed prompt token provides the same signal at test time. We study whether this signal can define the inner-loop objective for test-time training (TTT) in pretrained long-context language models. Existing in-place TTT methods store prompt information in fast weights, but they train each write to match a learned shortcut target: an internal pattern that may be easy to predict, yet is not the trajectory the model uses to form its next-token distribution. We introduce Test-Time Training with Next-Token Prediction (TTT-NTP), a drop-in fast-weight adaptation method for pretrained LLMs that instead supervises updates using the model{'}s own next contextual hidden state. This makes each local write follow the same causal computation that supports next-token prediction, without aggregating over a neighborhood of positions: the value target is a pointwise linear projection of a single next-position contextual state. Across four backbones from three model families, Llama-3.1-8B, Mistral-7B-v0.3, Qwen3-4B, and Qwen3-0.6B, TTT-NTP is the only method that consistently improves RULER Full-13 over the released backbone, by 2.9 to 4.1 points, while aggregate commonsense and knowledge performance is preserved. Our code is publicly available at https://github.com/yancyou/TTT-NTP."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ouyang-etal-2026-test">
<titleInfo>
<title>Test-Time Training with Next-Token Prediction</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xuan</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zefan</namePart>
<namePart type="family">Cai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junjie</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Next-token prediction is the self-supervised signal that trains language models, and every observed prompt token provides the same signal at test time. We study whether this signal can define the inner-loop objective for test-time training (TTT) in pretrained long-context language models. Existing in-place TTT methods store prompt information in fast weights, but they train each write to match a learned shortcut target: an internal pattern that may be easy to predict, yet is not the trajectory the model uses to form its next-token distribution. We introduce Test-Time Training with Next-Token Prediction (TTT-NTP), a drop-in fast-weight adaptation method for pretrained LLMs that instead supervises updates using the model’s own next contextual hidden state. This makes each local write follow the same causal computation that supports next-token prediction, without aggregating over a neighborhood of positions: the value target is a pointwise linear projection of a single next-position contextual state. Across four backbones from three model families, Llama-3.1-8B, Mistral-7B-v0.3, Qwen3-4B, and Qwen3-0.6B, TTT-NTP is the only method that consistently improves RULER Full-13 over the released backbone, by 2.9 to 4.1 points, while aggregate commonsense and knowledge performance is preserved. Our code is publicly available at https://github.com/yancyou/TTT-NTP.</abstract>
<identifier type="citekey">ouyang-etal-2026-test</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.936/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>19750</start>
<end>19764</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Test-Time Training with Next-Token Prediction
%A Ouyang, Xuan
%A Cai, Zefan
%A Hu, Junjie
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F ouyang-etal-2026-test
%X Next-token prediction is the self-supervised signal that trains language models, and every observed prompt token provides the same signal at test time. We study whether this signal can define the inner-loop objective for test-time training (TTT) in pretrained long-context language models. Existing in-place TTT methods store prompt information in fast weights, but they train each write to match a learned shortcut target: an internal pattern that may be easy to predict, yet is not the trajectory the model uses to form its next-token distribution. We introduce Test-Time Training with Next-Token Prediction (TTT-NTP), a drop-in fast-weight adaptation method for pretrained LLMs that instead supervises updates using the model’s own next contextual hidden state. This makes each local write follow the same causal computation that supports next-token prediction, without aggregating over a neighborhood of positions: the value target is a pointwise linear projection of a single next-position contextual state. Across four backbones from three model families, Llama-3.1-8B, Mistral-7B-v0.3, Qwen3-4B, and Qwen3-0.6B, TTT-NTP is the only method that consistently improves RULER Full-13 over the released backbone, by 2.9 to 4.1 points, while aggregate commonsense and knowledge performance is preserved. Our code is publicly available at https://github.com/yancyou/TTT-NTP.
%U https://aclanthology.org/2026.findings-emnlp.936/
%P 19750-19764
Markdown (Informal)
[Test-Time Training with Next-Token Prediction](https://aclanthology.org/2026.findings-emnlp.936/) (Ouyang et al., Findings 2026)
ACL
- Xuan Ouyang, Zefan Cai, and Junjie Hu. 2026. Test-Time Training with Next-Token Prediction. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 19750–19764, Budapest, Hungary. Association for Computational Linguistics.