@inproceedings{koudounas-etal-2026-benchmarking,
title = "Benchmarking Speech-to-Speech Translation Models",
author = "Koudounas, Alkis and
Futami, Hayato and
Jodelet, Quentin and
Take, Osamu and
Watanabe, Shinji and
Tsunoo, Emiru",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2194/",
pages = "46973--47008",
ISBN = "979-8-89176-462-0",
abstract = "Speech-to-speech translation (S2ST) has advanced rapidly, but offline evaluation lacks a unified protocol: studies report non-overlapping metric subsets, preventing direct comparisons. While prior toolkits target subsets of metrics or simultaneous and text-only translation (e.g., stopes-eval, SimulEval, IWSLT), we introduce COMPASS, the first offline S2ST benchmarking framework to unify 46 metrics across eight dimensions with domain-aware validation, and deploy it on 1,248 model-language configurations from FLEURS and CVSS, spanning cascaded and end-to-end architectures over ten language pairs. Architectures exhibit complementary strengths: best-vs-worst gaps exceed 30{\%} on naturalness and speaker preservation but remain within a few points on translation quality, so single-metric rankings systematically misrepresent system quality. Correlation filtering reduces 46 metrics to 10 per direction, with three axes requiring different metrics across X$\to$EN and EN$\to$X (e.g., TER/UTMOS vs. ChrF++/NISQA-MOS); these subsets preserve rankings (Spearman{'}s $\rho>0.80$) while cutting evaluation time by $\approx 2.5\times$. Human validation across dubbing, podcasts, and medical domains shows standalone MOS predictors fail to predict listener preference, while top domain-specific metrics correlate with human judgment ($\rho \geq 0.90$). We release COMPASS as a foundation for domain-aware S2ST evaluation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="koudounas-etal-2026-benchmarking">
<titleInfo>
<title>Benchmarking Speech-to-Speech Translation Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alkis</namePart>
<namePart type="family">Koudounas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hayato</namePart>
<namePart type="family">Futami</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Quentin</namePart>
<namePart type="family">Jodelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Osamu</namePart>
<namePart type="family">Take</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shinji</namePart>
<namePart type="family">Watanabe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Emiru</namePart>
<namePart type="family">Tsunoo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Speech-to-speech translation (S2ST) has advanced rapidly, but offline evaluation lacks a unified protocol: studies report non-overlapping metric subsets, preventing direct comparisons. While prior toolkits target subsets of metrics or simultaneous and text-only translation (e.g., stopes-eval, SimulEval, IWSLT), we introduce COMPASS, the first offline S2ST benchmarking framework to unify 46 metrics across eight dimensions with domain-aware validation, and deploy it on 1,248 model-language configurations from FLEURS and CVSS, spanning cascaded and end-to-end architectures over ten language pairs. Architectures exhibit complementary strengths: best-vs-worst gaps exceed 30% on naturalness and speaker preservation but remain within a few points on translation quality, so single-metric rankings systematically misrepresent system quality. Correlation filtering reduces 46 metrics to 10 per direction, with three axes requiring different metrics across XEN and ENX (e.g., TER/UTMOS vs. ChrF++/NISQA-MOS); these subsets preserve rankings (Spearman’s ρ>0.80) while cutting evaluation time by \approx 2.5\times. Human validation across dubbing, podcasts, and medical domains shows standalone MOS predictors fail to predict listener preference, while top domain-specific metrics correlate with human judgment (ρ \geq 0.90). We release COMPASS as a foundation for domain-aware S2ST evaluation.</abstract>
<identifier type="citekey">koudounas-etal-2026-benchmarking</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2194/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>46973</start>
<end>47008</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Benchmarking Speech-to-Speech Translation Models
%A Koudounas, Alkis
%A Futami, Hayato
%A Jodelet, Quentin
%A Take, Osamu
%A Watanabe, Shinji
%A Tsunoo, Emiru
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F koudounas-etal-2026-benchmarking
%X Speech-to-speech translation (S2ST) has advanced rapidly, but offline evaluation lacks a unified protocol: studies report non-overlapping metric subsets, preventing direct comparisons. While prior toolkits target subsets of metrics or simultaneous and text-only translation (e.g., stopes-eval, SimulEval, IWSLT), we introduce COMPASS, the first offline S2ST benchmarking framework to unify 46 metrics across eight dimensions with domain-aware validation, and deploy it on 1,248 model-language configurations from FLEURS and CVSS, spanning cascaded and end-to-end architectures over ten language pairs. Architectures exhibit complementary strengths: best-vs-worst gaps exceed 30% on naturalness and speaker preservation but remain within a few points on translation quality, so single-metric rankings systematically misrepresent system quality. Correlation filtering reduces 46 metrics to 10 per direction, with three axes requiring different metrics across XEN and ENX (e.g., TER/UTMOS vs. ChrF++/NISQA-MOS); these subsets preserve rankings (Spearman’s ρ>0.80) while cutting evaluation time by \approx 2.5\times. Human validation across dubbing, podcasts, and medical domains shows standalone MOS predictors fail to predict listener preference, while top domain-specific metrics correlate with human judgment (ρ \geq 0.90). We release COMPASS as a foundation for domain-aware S2ST evaluation.
%U https://aclanthology.org/2026.findings-emnlp.2194/
%P 46973-47008
Markdown (Informal)
[Benchmarking Speech-to-Speech Translation Models](https://aclanthology.org/2026.findings-emnlp.2194/) (Koudounas et al., Findings 2026)
ACL
- Alkis Koudounas, Hayato Futami, Quentin Jodelet, Osamu Take, Shinji Watanabe, and Emiru Tsunoo. 2026. Benchmarking Speech-to-Speech Translation Models. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 46973–47008, Budapest, Hungary. Association for Computational Linguistics.