@inproceedings{ozturk-etal-2026-sentence,
title = "Sentence-Level Foresight in Small Language Models: A Diagnostic Study of Learning Signals and Calibration",
author = {{\"O}zt{\"u}rk, Ahmet R{\i}fat and
Dalar, Ya{\u{g}}{\i}z Ekrem and
Sezer, Nedim Mutlu and
Saliho{\u{g}}lu, Feyzi Arda and
Aksoy, {\"O}mer Faruk},
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.2/",
pages = "15--23",
ISBN = "979-8-89176-475-0",
abstract = "Small language models provide useful diagnostic settings for studying which learning signals can be acquired under limited computation and whether those signals translate into observable behavior. We study whether a DistilGPT-2-scale autoregressive language model can learn a sentence-level future signal from next-sentence prediction and self-loop hard negatives. Across three seeds on both WikiText-103 and WritingPrompts, ForesightLM-v2 consistently learns positive held-out hard-negative separation, indicating that the auxiliary future signal is learnable. Generation experiments, however, reveal a calibration gap between representation learning and behavioral utility. Five-candidate reranking substantially improves several automatic diagnostics, but comparable gains are obtained with generic reranking, random and shuffled future-score controls, and a matched training-free base-model reranking baseline. On the fixed five-candidate pool, removing the learned future term changes only 2 of 120 selected continuations. These results show that a small autoregressive model can acquire a measurable sentence-level future representation, while most observed generation gains are explained by candidate selection rather than by the learned future term itself. ForesightLM-v2 therefore serves as a controlled case study of the distinction between learning an auxiliary signal and making that signal behaviorally effective."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ozturk-etal-2026-sentence">
<titleInfo>
<title>Sentence-Level Foresight in Small Language Models: A Diagnostic Study of Learning Signals and Calibration</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ahmet</namePart>
<namePart type="given">Rıfat</namePart>
<namePart type="family">Öztürk</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yağız</namePart>
<namePart type="given">Ekrem</namePart>
<namePart type="family">Dalar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nedim</namePart>
<namePart type="given">Mutlu</namePart>
<namePart type="family">Sezer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Feyzi</namePart>
<namePart type="given">Arda</namePart>
<namePart type="family">Salihoğlu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ömer</namePart>
<namePart type="given">Faruk</namePart>
<namePart type="family">Aksoy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Small language models provide useful diagnostic settings for studying which learning signals can be acquired under limited computation and whether those signals translate into observable behavior. We study whether a DistilGPT-2-scale autoregressive language model can learn a sentence-level future signal from next-sentence prediction and self-loop hard negatives. Across three seeds on both WikiText-103 and WritingPrompts, ForesightLM-v2 consistently learns positive held-out hard-negative separation, indicating that the auxiliary future signal is learnable. Generation experiments, however, reveal a calibration gap between representation learning and behavioral utility. Five-candidate reranking substantially improves several automatic diagnostics, but comparable gains are obtained with generic reranking, random and shuffled future-score controls, and a matched training-free base-model reranking baseline. On the fixed five-candidate pool, removing the learned future term changes only 2 of 120 selected continuations. These results show that a small autoregressive model can acquire a measurable sentence-level future representation, while most observed generation gains are explained by candidate selection rather than by the learned future term itself. ForesightLM-v2 therefore serves as a controlled case study of the distinction between learning an auxiliary signal and making that signal behaviorally effective.</abstract>
<identifier type="citekey">ozturk-etal-2026-sentence</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.2/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>15</start>
<end>23</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Sentence-Level Foresight in Small Language Models: A Diagnostic Study of Learning Signals and Calibration
%A Öztürk, Ahmet Rıfat
%A Dalar, Yağız Ekrem
%A Sezer, Nedim Mutlu
%A Salihoğlu, Feyzi Arda
%A Aksoy, Ömer Faruk
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F ozturk-etal-2026-sentence
%X Small language models provide useful diagnostic settings for studying which learning signals can be acquired under limited computation and whether those signals translate into observable behavior. We study whether a DistilGPT-2-scale autoregressive language model can learn a sentence-level future signal from next-sentence prediction and self-loop hard negatives. Across three seeds on both WikiText-103 and WritingPrompts, ForesightLM-v2 consistently learns positive held-out hard-negative separation, indicating that the auxiliary future signal is learnable. Generation experiments, however, reveal a calibration gap between representation learning and behavioral utility. Five-candidate reranking substantially improves several automatic diagnostics, but comparable gains are obtained with generic reranking, random and shuffled future-score controls, and a matched training-free base-model reranking baseline. On the fixed five-candidate pool, removing the learned future term changes only 2 of 120 selected continuations. These results show that a small autoregressive model can acquire a measurable sentence-level future representation, while most observed generation gains are explained by candidate selection rather than by the learned future term itself. ForesightLM-v2 therefore serves as a controlled case study of the distinction between learning an auxiliary signal and making that signal behaviorally effective.
%U https://aclanthology.org/2026.babylm-main.2/
%P 15-23
Markdown (Informal)
[Sentence-Level Foresight in Small Language Models: A Diagnostic Study of Learning Signals and Calibration](https://aclanthology.org/2026.babylm-main.2/) (Öztürk et al., BabyLM 2026)
ACL