@inproceedings{li-li-2026-language,
title = "Language Models Detokenize, But Only for Themselves",
author = "Li, Qingyun and
Li, Ruoyun",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.123/",
pages = "2703--2719",
ISBN = "979-8-89176-439-2",
abstract = "Language models often appear to become insensitive to how text is tokenized. Models trained with different tokenizers can reach similar language-modeling loss, and prior work shows that they recover word-level information from the raw subword segmentations (a phenomenon referred to as ``detokenization''). This raises a natural question: once that information has been recovered, do the resulting hidden states become independent of the tokenizer that produced them? We study this using *tokenizer twins*, pairs of models matched in architecture, training data, and random seed but trained with different tokenizers. We learn small translators between their hidden states at positions where the two tokenizers agree, and then test the same mapping where their tokenization paths differ. Transfer works well at agreement positions, adding less than one nat of loss, but breaks sharply at disagreement positions, where the loss increases by more than seven nats. This gap appears from the first hidden layer and persists through the network. At the same time, word-identity probes can still recover much of the underlying content from these representations. In other words, language models can abstract beyond their individual subword tokens while still carrying a lasting imprint of how the text was tokenized."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-li-2026-language">
<titleInfo>
<title>Language Models Detokenize, But Only for Themselves</title>
</titleInfo>
<name type="personal">
<namePart type="given">Qingyun</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruoyun</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Language models often appear to become insensitive to how text is tokenized. Models trained with different tokenizers can reach similar language-modeling loss, and prior work shows that they recover word-level information from the raw subword segmentations (a phenomenon referred to as “detokenization”). This raises a natural question: once that information has been recovered, do the resulting hidden states become independent of the tokenizer that produced them? We study this using *tokenizer twins*, pairs of models matched in architecture, training data, and random seed but trained with different tokenizers. We learn small translators between their hidden states at positions where the two tokenizers agree, and then test the same mapping where their tokenization paths differ. Transfer works well at agreement positions, adding less than one nat of loss, but breaks sharply at disagreement positions, where the loss increases by more than seven nats. This gap appears from the first hidden layer and persists through the network. At the same time, word-identity probes can still recover much of the underlying content from these representations. In other words, language models can abstract beyond their individual subword tokens while still carrying a lasting imprint of how the text was tokenized.</abstract>
<identifier type="citekey">li-li-2026-language</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.123/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>2703</start>
<end>2719</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Language Models Detokenize, But Only for Themselves
%A Li, Qingyun
%A Li, Ruoyun
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F li-li-2026-language
%X Language models often appear to become insensitive to how text is tokenized. Models trained with different tokenizers can reach similar language-modeling loss, and prior work shows that they recover word-level information from the raw subword segmentations (a phenomenon referred to as “detokenization”). This raises a natural question: once that information has been recovered, do the resulting hidden states become independent of the tokenizer that produced them? We study this using *tokenizer twins*, pairs of models matched in architecture, training data, and random seed but trained with different tokenizers. We learn small translators between their hidden states at positions where the two tokenizers agree, and then test the same mapping where their tokenization paths differ. Transfer works well at agreement positions, adding less than one nat of loss, but breaks sharply at disagreement positions, where the loss increases by more than seven nats. This gap appears from the first hidden layer and persists through the network. At the same time, word-identity probes can still recover much of the underlying content from these representations. In other words, language models can abstract beyond their individual subword tokens while still carrying a lasting imprint of how the text was tokenized.
%U https://aclanthology.org/2026.aacl-long.123/
%P 2703-2719
Markdown (Informal)
[Language Models Detokenize, But Only for Themselves](https://aclanthology.org/2026.aacl-long.123/) (Li & Li, AACL-IJCNLP 2026)
ACL
- Qingyun Li and Ruoyun Li. 2026. Language Models Detokenize, But Only for Themselves. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 2703–2719, Hengqin, China. Association for Computational Linguistics.