@inproceedings{hofmann-etal-2026-tokenization,
title = "Tokenization in Large Language Models",
author = "Hofmann, Valentin and
Kallini, Julie and
Kumar, Sachin and
Libovick{\'y}, Jind{\v{r}}ich and
Limisiewicz, Tomasz and
Salesky, Elizabeth",
editor = "Marasovi{\'c}, Ana and
Glava{\v{s}}, Goran",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-tutorials.3/",
pages = "4--5",
ISBN = "979-8-89176-463-7",
abstract = "Tokenization defines how data is represented as input and output for machine learning systems, including large language models (LLMs). While tokenization has long been a foundational component of NLP systems, it has historically remained a niche research area, often considered solved. The recent proliferation of LLMs has changed this: as these models are deployed across diverse languages and domains, tokenization-related issues have become increasingly apparent{---}from cross-lingual representation disparities to efficiency bottlenecks. This tutorial will provide a comprehensive overview of tokenization methods in text, starting from classical word-based and morphological approaches, to the current standard of statistical subword tokenizers, and emerging methods on end-to-end learnable tokenizers. We will also discuss the implications of tokenization choices on model performance, fairness, and efficiency, and present best practices for evaluating and designing tokenizers in multilingual and domain-specific settings."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hofmann-etal-2026-tokenization">
<titleInfo>
<title>Tokenization in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Valentin</namePart>
<namePart type="family">Hofmann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julie</namePart>
<namePart type="family">Kallini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sachin</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jindřich</namePart>
<namePart type="family">Libovický</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tomasz</namePart>
<namePart type="family">Limisiewicz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elizabeth</namePart>
<namePart type="family">Salesky</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ana</namePart>
<namePart type="family">Marasović</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Goran</namePart>
<namePart type="family">Glavaš</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-463-7</identifier>
</relatedItem>
<abstract>Tokenization defines how data is represented as input and output for machine learning systems, including large language models (LLMs). While tokenization has long been a foundational component of NLP systems, it has historically remained a niche research area, often considered solved. The recent proliferation of LLMs has changed this: as these models are deployed across diverse languages and domains, tokenization-related issues have become increasingly apparent—from cross-lingual representation disparities to efficiency bottlenecks. This tutorial will provide a comprehensive overview of tokenization methods in text, starting from classical word-based and morphological approaches, to the current standard of statistical subword tokenizers, and emerging methods on end-to-end learnable tokenizers. We will also discuss the implications of tokenization choices on model performance, fairness, and efficiency, and present best practices for evaluating and designing tokenizers in multilingual and domain-specific settings.</abstract>
<identifier type="citekey">hofmann-etal-2026-tokenization</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-tutorials.3/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>4</start>
<end>5</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Tokenization in Large Language Models
%A Hofmann, Valentin
%A Kallini, Julie
%A Kumar, Sachin
%A Libovický, Jindřich
%A Limisiewicz, Tomasz
%A Salesky, Elizabeth
%Y Marasović, Ana
%Y Glavaš, Goran
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-463-7
%F hofmann-etal-2026-tokenization
%X Tokenization defines how data is represented as input and output for machine learning systems, including large language models (LLMs). While tokenization has long been a foundational component of NLP systems, it has historically remained a niche research area, often considered solved. The recent proliferation of LLMs has changed this: as these models are deployed across diverse languages and domains, tokenization-related issues have become increasingly apparent—from cross-lingual representation disparities to efficiency bottlenecks. This tutorial will provide a comprehensive overview of tokenization methods in text, starting from classical word-based and morphological approaches, to the current standard of statistical subword tokenizers, and emerging methods on end-to-end learnable tokenizers. We will also discuss the implications of tokenization choices on model performance, fairness, and efficiency, and present best practices for evaluating and designing tokenizers in multilingual and domain-specific settings.
%U https://aclanthology.org/2026.emnlp-tutorials.3/
%P 4-5
Markdown (Informal)
[Tokenization in Large Language Models](https://aclanthology.org/2026.emnlp-tutorials.3/) (Hofmann et al., EMNLP 2026)
ACL
- Valentin Hofmann, Julie Kallini, Sachin Kumar, Jindřich Libovický, Tomasz Limisiewicz, and Elizabeth Salesky. 2026. Tokenization in Large Language Models. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts, pages 4–5, Budapest, Hungary. Association for Computational Linguistics.