@inproceedings{adari-2026-vaaniclip,
title = "{V}aani{CLIP}: Cross-Lingual Speech-Text Retrieval for {I}ndic Languages",
author = "Adari, Vamsi",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.16/",
pages = "206--220",
ISBN = "979-8-89176-469-9",
abstract = "Cross-lingual speech-text retrieval for Indic languages (retrieving English text using vernacular voice queries) remains an open problem despite recent progress in multilingual speech and text representation learning. We introduce VaaniCLIP and establish a systematic benchmark on BhasaAnuvaad across Telugu, Tamil, and Hindi, demonstrating that lightweight task-specific contrastive alignment achieves strong performance where established baselines fail. Cascade pipelines (Whisper ASR {\textrightarrow} NLLB translation {\textrightarrow} LaBSE retrieval) collapse on low-resource Indic languages: Telugu cascade retrieval reaches only 3.1{\%} R@1, roughly 18{\texttimes} below VaaniCLIP, and SeamlessM4T v2 (2.3B parameters) achieves near-random performance zero-shot. By contrast, VaaniCLIP aligns frozen Whisper and LaBSE encoders with partial Whisper fine-tuning (15M trainable parameters, against a 2.3B-parameter generalist model used zero-shot) and achieves R@1 of 86.4 on Hindi, 55.5 on Telugu, and 54.8 on Tamil at a normalized retrieval pool of N = 2,869, capturing 76{--}89{\%} of the LaBSE text-text upper bound. A frozen Whisper-medium encoder with only 787K trainable parameters reaches 95{--}99{\%} of per-language partial fine-tuning performance, confirming encoder representation quality as the primary bottleneck. We further show that the joint model transfers zero-shot to Kannada (an unseen Dravidian language), achieving R@1 of 25.1 ({\textasciitilde}700{\texttimes} above random) with no Kannada training data, providing evidence that VaaniCLIP learns generalizable Dravidian audio-text alignment."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="adari-2026-vaaniclip">
<titleInfo>
<title>VaaniCLIP: Cross-Lingual Speech-Text Retrieval for Indic Languages</title>
</titleInfo>
<name type="personal">
<namePart type="given">Vamsi</namePart>
<namePart type="family">Adari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>Cross-lingual speech-text retrieval for Indic languages (retrieving English text using vernacular voice queries) remains an open problem despite recent progress in multilingual speech and text representation learning. We introduce VaaniCLIP and establish a systematic benchmark on BhasaAnuvaad across Telugu, Tamil, and Hindi, demonstrating that lightweight task-specific contrastive alignment achieves strong performance where established baselines fail. Cascade pipelines (Whisper ASR → NLLB translation → LaBSE retrieval) collapse on low-resource Indic languages: Telugu cascade retrieval reaches only 3.1% R@1, roughly 18× below VaaniCLIP, and SeamlessM4T v2 (2.3B parameters) achieves near-random performance zero-shot. By contrast, VaaniCLIP aligns frozen Whisper and LaBSE encoders with partial Whisper fine-tuning (15M trainable parameters, against a 2.3B-parameter generalist model used zero-shot) and achieves R@1 of 86.4 on Hindi, 55.5 on Telugu, and 54.8 on Tamil at a normalized retrieval pool of N = 2,869, capturing 76–89% of the LaBSE text-text upper bound. A frozen Whisper-medium encoder with only 787K trainable parameters reaches 95–99% of per-language partial fine-tuning performance, confirming encoder representation quality as the primary bottleneck. We further show that the joint model transfers zero-shot to Kannada (an unseen Dravidian language), achieving R@1 of 25.1 (~700× above random) with no Kannada training data, providing evidence that VaaniCLIP learns generalizable Dravidian audio-text alignment.</abstract>
<identifier type="citekey">adari-2026-vaaniclip</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.16/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>206</start>
<end>220</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T VaaniCLIP: Cross-Lingual Speech-Text Retrieval for Indic Languages
%A Adari, Vamsi
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F adari-2026-vaaniclip
%X Cross-lingual speech-text retrieval for Indic languages (retrieving English text using vernacular voice queries) remains an open problem despite recent progress in multilingual speech and text representation learning. We introduce VaaniCLIP and establish a systematic benchmark on BhasaAnuvaad across Telugu, Tamil, and Hindi, demonstrating that lightweight task-specific contrastive alignment achieves strong performance where established baselines fail. Cascade pipelines (Whisper ASR → NLLB translation → LaBSE retrieval) collapse on low-resource Indic languages: Telugu cascade retrieval reaches only 3.1% R@1, roughly 18× below VaaniCLIP, and SeamlessM4T v2 (2.3B parameters) achieves near-random performance zero-shot. By contrast, VaaniCLIP aligns frozen Whisper and LaBSE encoders with partial Whisper fine-tuning (15M trainable parameters, against a 2.3B-parameter generalist model used zero-shot) and achieves R@1 of 86.4 on Hindi, 55.5 on Telugu, and 54.8 on Tamil at a normalized retrieval pool of N = 2,869, capturing 76–89% of the LaBSE text-text upper bound. A frozen Whisper-medium encoder with only 787K trainable parameters reaches 95–99% of per-language partial fine-tuning performance, confirming encoder representation quality as the primary bottleneck. We further show that the joint model transfers zero-shot to Kannada (an unseen Dravidian language), achieving R@1 of 25.1 (~700× above random) with no Kannada training data, providing evidence that VaaniCLIP learns generalizable Dravidian audio-text alignment.
%U https://aclanthology.org/2026.mrl-1.16/
%P 206-220
Markdown (Informal)
[VaaniCLIP: Cross-Lingual Speech-Text Retrieval for Indic Languages](https://aclanthology.org/2026.mrl-1.16/) (Adari, MRL 2026)
ACL