@inproceedings{braunschweiler-etal-2026-tokenization,
title = "Tokenization Risk as a Control Signal for Adaptive {RAG}",
author = "Braunschweiler, Norbert and
Doddipatla, Rama and
Collier, Nigel",
editor = {Aziz, Wilker and
Berant, Jonathan and
Eikema, Bryan and
Ilia, Evgenia and
de Marneffe, Marie-Catherine and
Plank, Barbara and
Shelmanov, Artem and
Swayamdipta, Swabha and
Tiedemann, J{\"o}rg and
Vazhentsev, Artem and
V{\'a}zquez, Ra{\'u}l and
Zerva, Chrysoula},
booktitle = "The Proceedings of the Third Workshop on Uncertainty-Aware {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.uncertainlp-main.2/",
pages = "19--29",
ISBN = "979-8-89176-486-6",
abstract = "Retrieval-Augmented Generation (RAG) systems can degrade on inputs containing specialized, low-frequency vocabulary, where subword tokenization introduces fragmentation that disrupts both retrieval and generation. We introduce tokenization risk, a lightweight metric that predicts these failures. We show that (i) tokenization risk strongly correlates with degradation in retrieval and answer quality, and (ii) static mitigations (e.g., hybrid retrieval) provide only partial gains. To address this, we propose risk-based routing, which adaptively switches retrieval and generation strategies based on risk. Experiments on SpecialVocabBench, a risk-stratified benchmark of technical documents, demonstrate that this approach consistently improves retrieval accuracy and answer correctness with small system changes, establishing tokenization risk as an effective control signal for adaptive RAG."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="braunschweiler-etal-2026-tokenization">
<titleInfo>
<title>Tokenization Risk as a Control Signal for Adaptive RAG</title>
</titleInfo>
<name type="personal">
<namePart type="given">Norbert</namePart>
<namePart type="family">Braunschweiler</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rama</namePart>
<namePart type="family">Doddipatla</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nigel</namePart>
<namePart type="family">Collier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of the Third Workshop on Uncertainty-Aware NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wilker</namePart>
<namePart type="family">Aziz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jonathan</namePart>
<namePart type="family">Berant</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bryan</namePart>
<namePart type="family">Eikema</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Evgenia</namePart>
<namePart type="family">Ilia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marie-Catherine</namePart>
<namePart type="family">de Marneffe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Shelmanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swabha</namePart>
<namePart type="family">Swayamdipta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jörg</namePart>
<namePart type="family">Tiedemann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Vazhentsev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raúl</namePart>
<namePart type="family">Vázquez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chrysoula</namePart>
<namePart type="family">Zerva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-486-6</identifier>
</relatedItem>
<abstract>Retrieval-Augmented Generation (RAG) systems can degrade on inputs containing specialized, low-frequency vocabulary, where subword tokenization introduces fragmentation that disrupts both retrieval and generation. We introduce tokenization risk, a lightweight metric that predicts these failures. We show that (i) tokenization risk strongly correlates with degradation in retrieval and answer quality, and (ii) static mitigations (e.g., hybrid retrieval) provide only partial gains. To address this, we propose risk-based routing, which adaptively switches retrieval and generation strategies based on risk. Experiments on SpecialVocabBench, a risk-stratified benchmark of technical documents, demonstrate that this approach consistently improves retrieval accuracy and answer correctness with small system changes, establishing tokenization risk as an effective control signal for adaptive RAG.</abstract>
<identifier type="citekey">braunschweiler-etal-2026-tokenization</identifier>
<location>
<url>https://aclanthology.org/2026.uncertainlp-main.2/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>19</start>
<end>29</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Tokenization Risk as a Control Signal for Adaptive RAG
%A Braunschweiler, Norbert
%A Doddipatla, Rama
%A Collier, Nigel
%Y Aziz, Wilker
%Y Berant, Jonathan
%Y Eikema, Bryan
%Y Ilia, Evgenia
%Y de Marneffe, Marie-Catherine
%Y Plank, Barbara
%Y Shelmanov, Artem
%Y Swayamdipta, Swabha
%Y Tiedemann, Jörg
%Y Vazhentsev, Artem
%Y Vázquez, Raúl
%Y Zerva, Chrysoula
%S The Proceedings of the Third Workshop on Uncertainty-Aware NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-486-6
%F braunschweiler-etal-2026-tokenization
%X Retrieval-Augmented Generation (RAG) systems can degrade on inputs containing specialized, low-frequency vocabulary, where subword tokenization introduces fragmentation that disrupts both retrieval and generation. We introduce tokenization risk, a lightweight metric that predicts these failures. We show that (i) tokenization risk strongly correlates with degradation in retrieval and answer quality, and (ii) static mitigations (e.g., hybrid retrieval) provide only partial gains. To address this, we propose risk-based routing, which adaptively switches retrieval and generation strategies based on risk. Experiments on SpecialVocabBench, a risk-stratified benchmark of technical documents, demonstrate that this approach consistently improves retrieval accuracy and answer correctness with small system changes, establishing tokenization risk as an effective control signal for adaptive RAG.
%U https://aclanthology.org/2026.uncertainlp-main.2/
%P 19-29
Markdown (Informal)
[Tokenization Risk as a Control Signal for Adaptive RAG](https://aclanthology.org/2026.uncertainlp-main.2/) (Braunschweiler et al., UncertaiNLP 2026)
ACL