@inproceedings{ries-venkatesh-2026-evaluating,
title = "Evaluating and Debiasing Multilingual Backpack Language Model Representations",
author = "Ries, Jennifer Jane and
Venkatesh, Kavya",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.24/",
pages = "300--309",
ISBN = "979-8-89176-469-9",
abstract = "Backpack language model embeddings represent each word as a weighted sum of sense vectors, offering more interpretable and intervenable lexical representations than standard contextual embeddings. Backpack sense vector representations show strong performance on lexical similarity tasks in a monolingual setting, as well as cross-lingual transfer through finetuning on aligned parallel multilingual data. Our work investigates whether representations from Backpack models trained from scratch on parallel aligned multilingual data allow for interpretable control without sacrificing translational and cross-lingual representation learning ability. We train matched 131M parameter Transformer and 132M parameter Backpack models from scratch on parallel aligned English-French EuroParl and evaluate their expressivity using validation perplexity, their learned monolingual and cross-lingual lexical relationships using Multi-SimLex and their translational ability using sentence-level cosine similarity. While the Transformer attains higher English and cross-lingual Multi-SimLex, the Backpack outperforms the Transformer on French Multi-SimLex and shows stronger translated-vs-random sentence separation. We finally compare sense-level gender bias ablations on the Backpack model with nullspace projection debiasing on the Transformer, demonstrating the effectiveness of sense vector control for debiasing multilingual representations."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ries-venkatesh-2026-evaluating">
<titleInfo>
<title>Evaluating and Debiasing Multilingual Backpack Language Model Representations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jennifer</namePart>
<namePart type="given">Jane</namePart>
<namePart type="family">Ries</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kavya</namePart>
<namePart type="family">Venkatesh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>Backpack language model embeddings represent each word as a weighted sum of sense vectors, offering more interpretable and intervenable lexical representations than standard contextual embeddings. Backpack sense vector representations show strong performance on lexical similarity tasks in a monolingual setting, as well as cross-lingual transfer through finetuning on aligned parallel multilingual data. Our work investigates whether representations from Backpack models trained from scratch on parallel aligned multilingual data allow for interpretable control without sacrificing translational and cross-lingual representation learning ability. We train matched 131M parameter Transformer and 132M parameter Backpack models from scratch on parallel aligned English-French EuroParl and evaluate their expressivity using validation perplexity, their learned monolingual and cross-lingual lexical relationships using Multi-SimLex and their translational ability using sentence-level cosine similarity. While the Transformer attains higher English and cross-lingual Multi-SimLex, the Backpack outperforms the Transformer on French Multi-SimLex and shows stronger translated-vs-random sentence separation. We finally compare sense-level gender bias ablations on the Backpack model with nullspace projection debiasing on the Transformer, demonstrating the effectiveness of sense vector control for debiasing multilingual representations.</abstract>
<identifier type="citekey">ries-venkatesh-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.24/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>300</start>
<end>309</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating and Debiasing Multilingual Backpack Language Model Representations
%A Ries, Jennifer Jane
%A Venkatesh, Kavya
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F ries-venkatesh-2026-evaluating
%X Backpack language model embeddings represent each word as a weighted sum of sense vectors, offering more interpretable and intervenable lexical representations than standard contextual embeddings. Backpack sense vector representations show strong performance on lexical similarity tasks in a monolingual setting, as well as cross-lingual transfer through finetuning on aligned parallel multilingual data. Our work investigates whether representations from Backpack models trained from scratch on parallel aligned multilingual data allow for interpretable control without sacrificing translational and cross-lingual representation learning ability. We train matched 131M parameter Transformer and 132M parameter Backpack models from scratch on parallel aligned English-French EuroParl and evaluate their expressivity using validation perplexity, their learned monolingual and cross-lingual lexical relationships using Multi-SimLex and their translational ability using sentence-level cosine similarity. While the Transformer attains higher English and cross-lingual Multi-SimLex, the Backpack outperforms the Transformer on French Multi-SimLex and shows stronger translated-vs-random sentence separation. We finally compare sense-level gender bias ablations on the Backpack model with nullspace projection debiasing on the Transformer, demonstrating the effectiveness of sense vector control for debiasing multilingual representations.
%U https://aclanthology.org/2026.mrl-1.24/
%P 300-309
Markdown (Informal)
[Evaluating and Debiasing Multilingual Backpack Language Model Representations](https://aclanthology.org/2026.mrl-1.24/) (Ries & Venkatesh, MRL 2026)
ACL