@inproceedings{rahman-etal-2026-language,
title = "What Does a Language Model Know If It Has Never Seen the Future? Temporal Isolation as a Natural Experiment in Knowledge Attribution",
author = "Rahman, Azizur and
Hiray, Rushikesh and
Srivastava, Shubham and
Hijjawi, Mohammad and
Hasan, Laraib",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2325/",
pages = "52814--52828",
ISBN = "979-8-89176-459-0",
abstract = "How much of a language model{'}s apparent knowledge reflects general linguistic competence, and how much reflects memorisation of facts seen during pretraining? Disentangling the two is normally confounded by data contamination, because modern corpora overlap with the benchmarks used to evaluate them. We exploit a natural experiment that removes exact benchmark exposure by construction. TALKIE-1930 is a 13B-parameter transformer trained exclusively on approximately 260 billion tokens of English published before 1931; TALKIE-WEB is an architecture-, tokeniser-, and compute-matched twin trained on an equivalent volume of FineWeb. Because every modern NLP benchmark postdates 1930, any post-1930 capability TALKIE-1930 exhibits cannot arise from benchmark contamination. We study this model pair along three axes. First, syntactic competence transfers across eras with only a modest gap (BLiMP: 76.9{\%} for TALKIE-1930 vs. 82.0{\%} for TALKIE-WEB), and an OCR-noise ablation shows that this gap is not an artefact of character-level corruption in the historical corpus. Second, in-context learning is largely preserved by temporal isolation: TALKIE-1930 matches TALKIE-WEB on SST-2 (91.2{\%} vs. 90.9{\%} at k=32), exceeds it on MNLI (47.3{\%} vs. 37.8{\%}), and learns post-2006 genres it cannot have seen (tweet sentiment and emotion) with learning-rate slopes statistically indistinguishable from TALKIE-WEB{'}s. Third, linear probes recover the temporal provenance of a factual statement from hidden states at 98.2{\%} accuracy (TALKIE-1930) and 99.5{\%} (TALKIE-WEB); a three-way probe on TALKIE-1930 that adds a suspected-leaked class reaches 96.0{\%} accuracy (chance approximately 33{\%}), well above lexical and permutation baselines, though this figure is carried mainly by the temporal contrast and the leaked class is only partially recovered. Finally, we document a dissociation between representation and behaviour: although TALKIE-1930{'}s hidden states linearly encode temporal-boundary information, its greedy decodings do not, confabulating post-1930 facts with fluent, confident anachronisms such as dating the iPhone to 1876. We frame these findings as a controlled case study of a rare temporally isolated model pair, providing evidence that exact benchmark exposure is not necessary for in-context learning on modern tasks, and that base language models contain linearly decodable representations of the temporal provenance and, in part, the epistemic status of their knowledge that need not surface in their outputs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rahman-etal-2026-language">
<titleInfo>
<title>What Does a Language Model Know If It Has Never Seen the Future? Temporal Isolation as a Natural Experiment in Knowledge Attribution</title>
</titleInfo>
<name type="personal">
<namePart type="given">Azizur</namePart>
<namePart type="family">Rahman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rushikesh</namePart>
<namePart type="family">Hiray</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shubham</namePart>
<namePart type="family">Srivastava</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohammad</namePart>
<namePart type="family">Hijjawi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Laraib</namePart>
<namePart type="family">Hasan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>How much of a language model’s apparent knowledge reflects general linguistic competence, and how much reflects memorisation of facts seen during pretraining? Disentangling the two is normally confounded by data contamination, because modern corpora overlap with the benchmarks used to evaluate them. We exploit a natural experiment that removes exact benchmark exposure by construction. TALKIE-1930 is a 13B-parameter transformer trained exclusively on approximately 260 billion tokens of English published before 1931; TALKIE-WEB is an architecture-, tokeniser-, and compute-matched twin trained on an equivalent volume of FineWeb. Because every modern NLP benchmark postdates 1930, any post-1930 capability TALKIE-1930 exhibits cannot arise from benchmark contamination. We study this model pair along three axes. First, syntactic competence transfers across eras with only a modest gap (BLiMP: 76.9% for TALKIE-1930 vs. 82.0% for TALKIE-WEB), and an OCR-noise ablation shows that this gap is not an artefact of character-level corruption in the historical corpus. Second, in-context learning is largely preserved by temporal isolation: TALKIE-1930 matches TALKIE-WEB on SST-2 (91.2% vs. 90.9% at k=32), exceeds it on MNLI (47.3% vs. 37.8%), and learns post-2006 genres it cannot have seen (tweet sentiment and emotion) with learning-rate slopes statistically indistinguishable from TALKIE-WEB’s. Third, linear probes recover the temporal provenance of a factual statement from hidden states at 98.2% accuracy (TALKIE-1930) and 99.5% (TALKIE-WEB); a three-way probe on TALKIE-1930 that adds a suspected-leaked class reaches 96.0% accuracy (chance approximately 33%), well above lexical and permutation baselines, though this figure is carried mainly by the temporal contrast and the leaked class is only partially recovered. Finally, we document a dissociation between representation and behaviour: although TALKIE-1930’s hidden states linearly encode temporal-boundary information, its greedy decodings do not, confabulating post-1930 facts with fluent, confident anachronisms such as dating the iPhone to 1876. We frame these findings as a controlled case study of a rare temporally isolated model pair, providing evidence that exact benchmark exposure is not necessary for in-context learning on modern tasks, and that base language models contain linearly decodable representations of the temporal provenance and, in part, the epistemic status of their knowledge that need not surface in their outputs.</abstract>
<identifier type="citekey">rahman-etal-2026-language</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2325/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>52814</start>
<end>52828</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Does a Language Model Know If It Has Never Seen the Future? Temporal Isolation as a Natural Experiment in Knowledge Attribution
%A Rahman, Azizur
%A Hiray, Rushikesh
%A Srivastava, Shubham
%A Hijjawi, Mohammad
%A Hasan, Laraib
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F rahman-etal-2026-language
%X How much of a language model’s apparent knowledge reflects general linguistic competence, and how much reflects memorisation of facts seen during pretraining? Disentangling the two is normally confounded by data contamination, because modern corpora overlap with the benchmarks used to evaluate them. We exploit a natural experiment that removes exact benchmark exposure by construction. TALKIE-1930 is a 13B-parameter transformer trained exclusively on approximately 260 billion tokens of English published before 1931; TALKIE-WEB is an architecture-, tokeniser-, and compute-matched twin trained on an equivalent volume of FineWeb. Because every modern NLP benchmark postdates 1930, any post-1930 capability TALKIE-1930 exhibits cannot arise from benchmark contamination. We study this model pair along three axes. First, syntactic competence transfers across eras with only a modest gap (BLiMP: 76.9% for TALKIE-1930 vs. 82.0% for TALKIE-WEB), and an OCR-noise ablation shows that this gap is not an artefact of character-level corruption in the historical corpus. Second, in-context learning is largely preserved by temporal isolation: TALKIE-1930 matches TALKIE-WEB on SST-2 (91.2% vs. 90.9% at k=32), exceeds it on MNLI (47.3% vs. 37.8%), and learns post-2006 genres it cannot have seen (tweet sentiment and emotion) with learning-rate slopes statistically indistinguishable from TALKIE-WEB’s. Third, linear probes recover the temporal provenance of a factual statement from hidden states at 98.2% accuracy (TALKIE-1930) and 99.5% (TALKIE-WEB); a three-way probe on TALKIE-1930 that adds a suspected-leaked class reaches 96.0% accuracy (chance approximately 33%), well above lexical and permutation baselines, though this figure is carried mainly by the temporal contrast and the leaked class is only partially recovered. Finally, we document a dissociation between representation and behaviour: although TALKIE-1930’s hidden states linearly encode temporal-boundary information, its greedy decodings do not, confabulating post-1930 facts with fluent, confident anachronisms such as dating the iPhone to 1876. We frame these findings as a controlled case study of a rare temporally isolated model pair, providing evidence that exact benchmark exposure is not necessary for in-context learning on modern tasks, and that base language models contain linearly decodable representations of the temporal provenance and, in part, the epistemic status of their knowledge that need not surface in their outputs.
%U https://aclanthology.org/2026.emnlp-main.2325/
%P 52814-52828
Markdown (Informal)
[What Does a Language Model Know If It Has Never Seen the Future? Temporal Isolation as a Natural Experiment in Knowledge Attribution](https://aclanthology.org/2026.emnlp-main.2325/) (Rahman et al., EMNLP 2026)
ACL