@inproceedings{deng-2026-language,
title = "Language Coverage Is Not Variety Coverage: Lexical Underrepresentation of Classical {C}hinese in Multilingual Tokenizers",
author = "Deng, Lifan",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.5/",
pages = "53--57",
ISBN = "979-8-89176-469-9",
abstract = "Multilingual models are commonly described as supporting Chinese, but language-level coverage may conceal disparities between historical and modern varieties. We audit five public or widely deployed tokenizer families on Classical Chinese prose and Modern Chinese newspaper text. XLM-R, a canonical multilingual representation model, requires 1.070 tokens per Classical character versus 0.688 for Modern Chinese. More importantly, only 17.4{\%} of annotated two-character Classical words receive a single XLM-R token, compared with 66.4{\%} of Modern words. After controlling for word length and corpus frequency, the odds of atomic coverage are 0.158 of the Modern odds. Four general-purpose LLM tokenizers reproduce the same asymmetry. Exact sentence-length matching, frequency controls, Traditional-Simplified pairs, and leave-one-source-out tests rule out several simple artifacts. Multilingual tokenizers can therefore cover a language{'}s script while underrepresenting the lexical units of its historical varieties."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="deng-2026-language">
<titleInfo>
<title>Language Coverage Is Not Variety Coverage: Lexical Underrepresentation of Classical Chinese in Multilingual Tokenizers</title>
</titleInfo>
<name type="personal">
<namePart type="given">Lifan</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>Multilingual models are commonly described as supporting Chinese, but language-level coverage may conceal disparities between historical and modern varieties. We audit five public or widely deployed tokenizer families on Classical Chinese prose and Modern Chinese newspaper text. XLM-R, a canonical multilingual representation model, requires 1.070 tokens per Classical character versus 0.688 for Modern Chinese. More importantly, only 17.4% of annotated two-character Classical words receive a single XLM-R token, compared with 66.4% of Modern words. After controlling for word length and corpus frequency, the odds of atomic coverage are 0.158 of the Modern odds. Four general-purpose LLM tokenizers reproduce the same asymmetry. Exact sentence-length matching, frequency controls, Traditional-Simplified pairs, and leave-one-source-out tests rule out several simple artifacts. Multilingual tokenizers can therefore cover a language’s script while underrepresenting the lexical units of its historical varieties.</abstract>
<identifier type="citekey">deng-2026-language</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.5/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>53</start>
<end>57</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Language Coverage Is Not Variety Coverage: Lexical Underrepresentation of Classical Chinese in Multilingual Tokenizers
%A Deng, Lifan
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F deng-2026-language
%X Multilingual models are commonly described as supporting Chinese, but language-level coverage may conceal disparities between historical and modern varieties. We audit five public or widely deployed tokenizer families on Classical Chinese prose and Modern Chinese newspaper text. XLM-R, a canonical multilingual representation model, requires 1.070 tokens per Classical character versus 0.688 for Modern Chinese. More importantly, only 17.4% of annotated two-character Classical words receive a single XLM-R token, compared with 66.4% of Modern words. After controlling for word length and corpus frequency, the odds of atomic coverage are 0.158 of the Modern odds. Four general-purpose LLM tokenizers reproduce the same asymmetry. Exact sentence-length matching, frequency controls, Traditional-Simplified pairs, and leave-one-source-out tests rule out several simple artifacts. Multilingual tokenizers can therefore cover a language’s script while underrepresenting the lexical units of its historical varieties.
%U https://aclanthology.org/2026.mrl-1.5/
%P 53-57
Markdown (Informal)
[Language Coverage Is Not Variety Coverage: Lexical Underrepresentation of Classical Chinese in Multilingual Tokenizers](https://aclanthology.org/2026.mrl-1.5/) (Deng, MRL 2026)
ACL