@inproceedings{kashyap-etal-2026-far,
title = "How Far Can Current Tokenizers Go in {B}angla? A Benchmark and a Morphology-Aware Solution",
author = "Kashyap, Gautam Siddharth and
Hoque, MD Azizul and
Srivastava, Kinjalk and
Jalan, Pratik and
Kar, Subhashis and
Naseem, Usman",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2436/",
pages = "52275--52284",
ISBN = "979-8-89176-462-0",
abstract = "Bangla (Bengali), spoken by over 300 million people worldwide, presents unique tokenization challenges due to its rich morphological structure. Despite its global importance, Bangla remains underrepresented in mainstream NLP, particularly in low-resource tokenization research. Existing tokenizers such as Byte Pair Encoding (BPE), Unigram, WordPiece, and character-level were originally developed for high-resource languages. These tokenizers struggle with morphologically rich languages and show two main limitations: (i) unnecessary splitting of morphemes and (ii) fragmentation of compound words. Based on these limitations, we hypothesize that morphology-aware tokenization is necessary for handling Bangla effectively. To address this, we first introduce a benchmark that systematically evaluates how widely used tokenizers perform on morphology-related tasks. Motivated by these findings, we then propose BANGLATOKENIZER, a morphology-aware tokenizer that alleviates these limitations using a novel bond-strength framework inspired by molecular structures. Empirically, BANGLATOKENIZER yields consistent downstream improvements, achieving absolute accuracy gains of 2.8{\%}-3.4{\%}, macro-F1 gains of 2.5{\%}-3.6{\%}, and perplexity reductions of up to 20{\%} across diverse language models and tasks. These improvements are accompanied by 5{\%}-10{\%} higher inference throughput, showing that morphology-aware tokenization improves both effectiveness and efficiency"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kashyap-etal-2026-far">
<titleInfo>
<title>How Far Can Current Tokenizers Go in Bangla? A Benchmark and a Morphology-Aware Solution</title>
</titleInfo>
<name type="personal">
<namePart type="given">Gautam</namePart>
<namePart type="given">Siddharth</namePart>
<namePart type="family">Kashyap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">MD</namePart>
<namePart type="given">Azizul</namePart>
<namePart type="family">Hoque</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kinjalk</namePart>
<namePart type="family">Srivastava</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pratik</namePart>
<namePart type="family">Jalan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Subhashis</namePart>
<namePart type="family">Kar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Usman</namePart>
<namePart type="family">Naseem</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Bangla (Bengali), spoken by over 300 million people worldwide, presents unique tokenization challenges due to its rich morphological structure. Despite its global importance, Bangla remains underrepresented in mainstream NLP, particularly in low-resource tokenization research. Existing tokenizers such as Byte Pair Encoding (BPE), Unigram, WordPiece, and character-level were originally developed for high-resource languages. These tokenizers struggle with morphologically rich languages and show two main limitations: (i) unnecessary splitting of morphemes and (ii) fragmentation of compound words. Based on these limitations, we hypothesize that morphology-aware tokenization is necessary for handling Bangla effectively. To address this, we first introduce a benchmark that systematically evaluates how widely used tokenizers perform on morphology-related tasks. Motivated by these findings, we then propose BANGLATOKENIZER, a morphology-aware tokenizer that alleviates these limitations using a novel bond-strength framework inspired by molecular structures. Empirically, BANGLATOKENIZER yields consistent downstream improvements, achieving absolute accuracy gains of 2.8%-3.4%, macro-F1 gains of 2.5%-3.6%, and perplexity reductions of up to 20% across diverse language models and tasks. These improvements are accompanied by 5%-10% higher inference throughput, showing that morphology-aware tokenization improves both effectiveness and efficiency</abstract>
<identifier type="citekey">kashyap-etal-2026-far</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2436/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>52275</start>
<end>52284</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T How Far Can Current Tokenizers Go in Bangla? A Benchmark and a Morphology-Aware Solution
%A Kashyap, Gautam Siddharth
%A Hoque, MD Azizul
%A Srivastava, Kinjalk
%A Jalan, Pratik
%A Kar, Subhashis
%A Naseem, Usman
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F kashyap-etal-2026-far
%X Bangla (Bengali), spoken by over 300 million people worldwide, presents unique tokenization challenges due to its rich morphological structure. Despite its global importance, Bangla remains underrepresented in mainstream NLP, particularly in low-resource tokenization research. Existing tokenizers such as Byte Pair Encoding (BPE), Unigram, WordPiece, and character-level were originally developed for high-resource languages. These tokenizers struggle with morphologically rich languages and show two main limitations: (i) unnecessary splitting of morphemes and (ii) fragmentation of compound words. Based on these limitations, we hypothesize that morphology-aware tokenization is necessary for handling Bangla effectively. To address this, we first introduce a benchmark that systematically evaluates how widely used tokenizers perform on morphology-related tasks. Motivated by these findings, we then propose BANGLATOKENIZER, a morphology-aware tokenizer that alleviates these limitations using a novel bond-strength framework inspired by molecular structures. Empirically, BANGLATOKENIZER yields consistent downstream improvements, achieving absolute accuracy gains of 2.8%-3.4%, macro-F1 gains of 2.5%-3.6%, and perplexity reductions of up to 20% across diverse language models and tasks. These improvements are accompanied by 5%-10% higher inference throughput, showing that morphology-aware tokenization improves both effectiveness and efficiency
%U https://aclanthology.org/2026.findings-emnlp.2436/
%P 52275-52284
Markdown (Informal)
[How Far Can Current Tokenizers Go in Bangla? A Benchmark and a Morphology-Aware Solution](https://aclanthology.org/2026.findings-emnlp.2436/) (Kashyap et al., Findings 2026)
ACL