@inproceedings{malashenko-etal-2026-loss,
title = "Loss-Guided Vocabulary Discovery for Multilingual Static Encoders",
author = "Malashenko, Boris and
Jarsky, Ivan and
Stumpf, Sviatoslav A. and
Efimova, Valeria",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.15/",
pages = "196--205",
ISBN = "979-8-89176-469-9",
abstract = "Static sentence encoders offer efficient multilingual inference, but their vocabularies are fixed independently of the sentence-level objective. We introduce loss-guided vocabulary discovery, a train-time procedure that uses contrastive-loss utility to identify useful adjacent token pairs, assigns each selected pair its own row, and keeps the selected pair set unchanged during continued training. In our main run, the method selects 56,071 pairs, increasing the number of embedding rows by 21.74{\%} without changing the tokenizer or introducing contextual inference. We train the resulting encoder on 176 languages. Over three training seeds, loss-guided discovery outperforms an independently collected equal-cardinality frequency selector by 1.70{\ensuremath{\pm}}0.07, 0.12{\ensuremath{\pm}}0.05, and 0.62{\ensuremath{\pm}}0.02 points on Tatoeba, BUCC, and Flores, and the margin widens with continued training. Under matched update budgets, it also improves all 12 comparisons across four additional base-model families. The selected-pair model improves the no-pair control in all four training-resource quartiles on Tatoeba, including by 0.86 points in the lowest-resource quartile. On MTEB, our encoder outperforms the strongest competing static model by 2.92 points. Loss-guided discovery thus lets the sentence-level training signal allocate multilingual vocabulary capacity within a static encoder."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="malashenko-etal-2026-loss">
<titleInfo>
<title>Loss-Guided Vocabulary Discovery for Multilingual Static Encoders</title>
</titleInfo>
<name type="personal">
<namePart type="given">Boris</namePart>
<namePart type="family">Malashenko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ivan</namePart>
<namePart type="family">Jarsky</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sviatoslav</namePart>
<namePart type="given">A</namePart>
<namePart type="family">Stumpf</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Valeria</namePart>
<namePart type="family">Efimova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>Static sentence encoders offer efficient multilingual inference, but their vocabularies are fixed independently of the sentence-level objective. We introduce loss-guided vocabulary discovery, a train-time procedure that uses contrastive-loss utility to identify useful adjacent token pairs, assigns each selected pair its own row, and keeps the selected pair set unchanged during continued training. In our main run, the method selects 56,071 pairs, increasing the number of embedding rows by 21.74% without changing the tokenizer or introducing contextual inference. We train the resulting encoder on 176 languages. Over three training seeds, loss-guided discovery outperforms an independently collected equal-cardinality frequency selector by 1.70\ensuremath\pm0.07, 0.12\ensuremath\pm0.05, and 0.62\ensuremath\pm0.02 points on Tatoeba, BUCC, and Flores, and the margin widens with continued training. Under matched update budgets, it also improves all 12 comparisons across four additional base-model families. The selected-pair model improves the no-pair control in all four training-resource quartiles on Tatoeba, including by 0.86 points in the lowest-resource quartile. On MTEB, our encoder outperforms the strongest competing static model by 2.92 points. Loss-guided discovery thus lets the sentence-level training signal allocate multilingual vocabulary capacity within a static encoder.</abstract>
<identifier type="citekey">malashenko-etal-2026-loss</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.15/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>196</start>
<end>205</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Loss-Guided Vocabulary Discovery for Multilingual Static Encoders
%A Malashenko, Boris
%A Jarsky, Ivan
%A Stumpf, Sviatoslav A.
%A Efimova, Valeria
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F malashenko-etal-2026-loss
%X Static sentence encoders offer efficient multilingual inference, but their vocabularies are fixed independently of the sentence-level objective. We introduce loss-guided vocabulary discovery, a train-time procedure that uses contrastive-loss utility to identify useful adjacent token pairs, assigns each selected pair its own row, and keeps the selected pair set unchanged during continued training. In our main run, the method selects 56,071 pairs, increasing the number of embedding rows by 21.74% without changing the tokenizer or introducing contextual inference. We train the resulting encoder on 176 languages. Over three training seeds, loss-guided discovery outperforms an independently collected equal-cardinality frequency selector by 1.70\ensuremath\pm0.07, 0.12\ensuremath\pm0.05, and 0.62\ensuremath\pm0.02 points on Tatoeba, BUCC, and Flores, and the margin widens with continued training. Under matched update budgets, it also improves all 12 comparisons across four additional base-model families. The selected-pair model improves the no-pair control in all four training-resource quartiles on Tatoeba, including by 0.86 points in the lowest-resource quartile. On MTEB, our encoder outperforms the strongest competing static model by 2.92 points. Loss-guided discovery thus lets the sentence-level training signal allocate multilingual vocabulary capacity within a static encoder.
%U https://aclanthology.org/2026.mrl-1.15/
%P 196-205
Markdown (Informal)
[Loss-Guided Vocabulary Discovery for Multilingual Static Encoders](https://aclanthology.org/2026.mrl-1.15/) (Malashenko et al., MRL 2026)
ACL