@inproceedings{shimizu-etal-2026-revisiting,
title = "Revisiting the Effectiveness of Word-level Tokenization in Language Model Pretraining",
author = "Shimizu, Takahiro and
Wang, Tianqi and
Suzuki, Jun",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.44/",
pages = "542--559",
ISBN = "979-8-89176-469-9",
abstract = "Subword tokenization is widely used in modern large language models for open-vocabulary coverage and token efficiency. We revisit word-level tokenization by comparing it with SentencePiece Unigram under matched byte fallback settings, allowing both tokenizers to represent out-of-vocabulary units without unknown tokens. Using language-appropriate word-like segmentation, we evaluate models pretrained from scratch across six model configurations in English and, using a common model configuration, across four languages (English, French, Chinese, and Japanese), with vocabulary sizes of 10k, 50k, and 100k. Although subword tokenization is generally more token-efficient, raw accuracy varies across model configurations, languages, and vocabulary sizes, with word-level tokenization remaining competitive across the evaluated settings. Word-level tokenization also tends to achieve higher character-length-normalized accuracy, although this complementary metric does not isolate the effects of tokenization on model scoring or establish a universal advantage in language modeling quality. These results show that word-level tokenization can remain competitive with subword tokenization when open-vocabulary coverage and language-appropriate segmentation are provided. Our code and tokenizers are publicly available."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="shimizu-etal-2026-revisiting">
<titleInfo>
<title>Revisiting the Effectiveness of Word-level Tokenization in Language Model Pretraining</title>
</titleInfo>
<name type="personal">
<namePart type="given">Takahiro</namePart>
<namePart type="family">Shimizu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianqi</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>Subword tokenization is widely used in modern large language models for open-vocabulary coverage and token efficiency. We revisit word-level tokenization by comparing it with SentencePiece Unigram under matched byte fallback settings, allowing both tokenizers to represent out-of-vocabulary units without unknown tokens. Using language-appropriate word-like segmentation, we evaluate models pretrained from scratch across six model configurations in English and, using a common model configuration, across four languages (English, French, Chinese, and Japanese), with vocabulary sizes of 10k, 50k, and 100k. Although subword tokenization is generally more token-efficient, raw accuracy varies across model configurations, languages, and vocabulary sizes, with word-level tokenization remaining competitive across the evaluated settings. Word-level tokenization also tends to achieve higher character-length-normalized accuracy, although this complementary metric does not isolate the effects of tokenization on model scoring or establish a universal advantage in language modeling quality. These results show that word-level tokenization can remain competitive with subword tokenization when open-vocabulary coverage and language-appropriate segmentation are provided. Our code and tokenizers are publicly available.</abstract>
<identifier type="citekey">shimizu-etal-2026-revisiting</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.44/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>542</start>
<end>559</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Revisiting the Effectiveness of Word-level Tokenization in Language Model Pretraining
%A Shimizu, Takahiro
%A Wang, Tianqi
%A Suzuki, Jun
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F shimizu-etal-2026-revisiting
%X Subword tokenization is widely used in modern large language models for open-vocabulary coverage and token efficiency. We revisit word-level tokenization by comparing it with SentencePiece Unigram under matched byte fallback settings, allowing both tokenizers to represent out-of-vocabulary units without unknown tokens. Using language-appropriate word-like segmentation, we evaluate models pretrained from scratch across six model configurations in English and, using a common model configuration, across four languages (English, French, Chinese, and Japanese), with vocabulary sizes of 10k, 50k, and 100k. Although subword tokenization is generally more token-efficient, raw accuracy varies across model configurations, languages, and vocabulary sizes, with word-level tokenization remaining competitive across the evaluated settings. Word-level tokenization also tends to achieve higher character-length-normalized accuracy, although this complementary metric does not isolate the effects of tokenization on model scoring or establish a universal advantage in language modeling quality. These results show that word-level tokenization can remain competitive with subword tokenization when open-vocabulary coverage and language-appropriate segmentation are provided. Our code and tokenizers are publicly available.
%U https://aclanthology.org/2026.mrl-1.44/
%P 542-559
Markdown (Informal)
[Revisiting the Effectiveness of Word-level Tokenization in Language Model Pretraining](https://aclanthology.org/2026.mrl-1.44/) (Shimizu et al., MRL 2026)
ACL