@inproceedings{dwivedi-etal-2026-performance,
title = "Performance of Grapheme-Based Tokenizers on Word-level and Natural Language Understanding tasks for {I}ndic Languages",
author = "Dwivedi, Shagun and
Saquib, Mohammad and
Gopalan, Kaushik",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1093/",
pages = "23149--23161",
ISBN = "979-8-89176-462-0",
abstract = "Tokenization is one of the factors which influences the multilingual downstream performance of a language model. This paper demonstrates that vocabulary constrained grapheme cluster based tokenization leads to improved performance on Natural Language Understanding and Word-level tasks. We assess three different grapheme cluster based tokenizers, which are the Grapheme Cluster tokenizer, the Grapheme Pair Encoding, and a vocabulary constrained version of Grapheme Pair Encoding, across eleven Indic languages, namely Assamese, Bengali, Gujarati, Hindi, Kannada, Malayalam, Marathi, Odia, Punjabi, Tamil, and Telugu. We present the intrinsic performance of three grapheme cluster based tokenizers using Fertility, R{\'e}nyi Efficiency, and Percentile Frequency. We also compare the performance of these grapheme based tokenizers with seven popular multilingual and Indic-focused tokenizers on downstream tasks. We find that the vocabulary constrained Grapheme Pair Encoding tokenizer exhibits competitive and sometimes superior downstream performance in comparison to other grapheme cluster based tokenizers."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dwivedi-etal-2026-performance">
<titleInfo>
<title>Performance of Grapheme-Based Tokenizers on Word-level and Natural Language Understanding tasks for Indic Languages</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shagun</namePart>
<namePart type="family">Dwivedi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohammad</namePart>
<namePart type="family">Saquib</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kaushik</namePart>
<namePart type="family">Gopalan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Tokenization is one of the factors which influences the multilingual downstream performance of a language model. This paper demonstrates that vocabulary constrained grapheme cluster based tokenization leads to improved performance on Natural Language Understanding and Word-level tasks. We assess three different grapheme cluster based tokenizers, which are the Grapheme Cluster tokenizer, the Grapheme Pair Encoding, and a vocabulary constrained version of Grapheme Pair Encoding, across eleven Indic languages, namely Assamese, Bengali, Gujarati, Hindi, Kannada, Malayalam, Marathi, Odia, Punjabi, Tamil, and Telugu. We present the intrinsic performance of three grapheme cluster based tokenizers using Fertility, Rényi Efficiency, and Percentile Frequency. We also compare the performance of these grapheme based tokenizers with seven popular multilingual and Indic-focused tokenizers on downstream tasks. We find that the vocabulary constrained Grapheme Pair Encoding tokenizer exhibits competitive and sometimes superior downstream performance in comparison to other grapheme cluster based tokenizers.</abstract>
<identifier type="citekey">dwivedi-etal-2026-performance</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1093/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>23149</start>
<end>23161</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Performance of Grapheme-Based Tokenizers on Word-level and Natural Language Understanding tasks for Indic Languages
%A Dwivedi, Shagun
%A Saquib, Mohammad
%A Gopalan, Kaushik
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F dwivedi-etal-2026-performance
%X Tokenization is one of the factors which influences the multilingual downstream performance of a language model. This paper demonstrates that vocabulary constrained grapheme cluster based tokenization leads to improved performance on Natural Language Understanding and Word-level tasks. We assess three different grapheme cluster based tokenizers, which are the Grapheme Cluster tokenizer, the Grapheme Pair Encoding, and a vocabulary constrained version of Grapheme Pair Encoding, across eleven Indic languages, namely Assamese, Bengali, Gujarati, Hindi, Kannada, Malayalam, Marathi, Odia, Punjabi, Tamil, and Telugu. We present the intrinsic performance of three grapheme cluster based tokenizers using Fertility, Rényi Efficiency, and Percentile Frequency. We also compare the performance of these grapheme based tokenizers with seven popular multilingual and Indic-focused tokenizers on downstream tasks. We find that the vocabulary constrained Grapheme Pair Encoding tokenizer exhibits competitive and sometimes superior downstream performance in comparison to other grapheme cluster based tokenizers.
%U https://aclanthology.org/2026.findings-emnlp.1093/
%P 23149-23161
Markdown (Informal)
[Performance of Grapheme-Based Tokenizers on Word-level and Natural Language Understanding tasks for Indic Languages](https://aclanthology.org/2026.findings-emnlp.1093/) (Dwivedi et al., Findings 2026)
ACL