@inproceedings{bamgbose-etal-2026-beyond,
title = "Beyond Naturalness: Probing Automated {TTS} Evaluators on Linguistically Grounded Dimensions",
author = "Bamgbose, Oluwanifemi and
Shah, Jash and
Rosen, Simon and
Brin, Lindsay Devon and
Nguyen, Hoang H and
Koelzer, Anke and
Hansen, Rachel and
Bogavelli, Tara and
Riols, Fanny",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1857/",
pages = "42215--42243",
ISBN = "979-8-89176-459-0",
abstract = "Automated Text-to-Speech (TTS) evaluation methods (Mean Opinion Score (MOS) predictors and Audio-LLM Judges) are expected to reflect human perception, yet it is unclear how well they capture the distinct aspects of speech that listeners actually perceive. We deconstruct the under-specified ``naturalness'' into a linguistically grounded annotation schema spanning 10 distinct perceptual dimensions, and use it to construct the first dimension-level meta-evaluation benchmark for TTS, comprising 860 utterances annotated by trained linguist raters (Krippendorff{'}s up to 0.821). Results from benchmarking four MOS predictors and four Audio-LLM Judges reveals that MOS predictors collapse onto acoustic signal quality, while Audio-LLM Judges show selective, prompt-dependent detection that doesnt generalise across the dimension set. Neither class reliably captures linguistically structured speech errors. Our dataset, annotation schema, and evaluation code are publicly released to support more targeted and interpretable TTS evaluation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="bamgbose-etal-2026-beyond">
<titleInfo>
<title>Beyond Naturalness: Probing Automated TTS Evaluators on Linguistically Grounded Dimensions</title>
</titleInfo>
<name type="personal">
<namePart type="given">Oluwanifemi</namePart>
<namePart type="family">Bamgbose</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jash</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Rosen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lindsay</namePart>
<namePart type="given">Devon</namePart>
<namePart type="family">Brin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hoang</namePart>
<namePart type="given">H</namePart>
<namePart type="family">Nguyen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anke</namePart>
<namePart type="family">Koelzer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rachel</namePart>
<namePart type="family">Hansen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tara</namePart>
<namePart type="family">Bogavelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fanny</namePart>
<namePart type="family">Riols</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Automated Text-to-Speech (TTS) evaluation methods (Mean Opinion Score (MOS) predictors and Audio-LLM Judges) are expected to reflect human perception, yet it is unclear how well they capture the distinct aspects of speech that listeners actually perceive. We deconstruct the under-specified “naturalness” into a linguistically grounded annotation schema spanning 10 distinct perceptual dimensions, and use it to construct the first dimension-level meta-evaluation benchmark for TTS, comprising 860 utterances annotated by trained linguist raters (Krippendorff’s up to 0.821). Results from benchmarking four MOS predictors and four Audio-LLM Judges reveals that MOS predictors collapse onto acoustic signal quality, while Audio-LLM Judges show selective, prompt-dependent detection that doesnt generalise across the dimension set. Neither class reliably captures linguistically structured speech errors. Our dataset, annotation schema, and evaluation code are publicly released to support more targeted and interpretable TTS evaluation.</abstract>
<identifier type="citekey">bamgbose-etal-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1857/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>42215</start>
<end>42243</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Naturalness: Probing Automated TTS Evaluators on Linguistically Grounded Dimensions
%A Bamgbose, Oluwanifemi
%A Shah, Jash
%A Rosen, Simon
%A Brin, Lindsay Devon
%A Nguyen, Hoang H.
%A Koelzer, Anke
%A Hansen, Rachel
%A Bogavelli, Tara
%A Riols, Fanny
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F bamgbose-etal-2026-beyond
%X Automated Text-to-Speech (TTS) evaluation methods (Mean Opinion Score (MOS) predictors and Audio-LLM Judges) are expected to reflect human perception, yet it is unclear how well they capture the distinct aspects of speech that listeners actually perceive. We deconstruct the under-specified “naturalness” into a linguistically grounded annotation schema spanning 10 distinct perceptual dimensions, and use it to construct the first dimension-level meta-evaluation benchmark for TTS, comprising 860 utterances annotated by trained linguist raters (Krippendorff’s up to 0.821). Results from benchmarking four MOS predictors and four Audio-LLM Judges reveals that MOS predictors collapse onto acoustic signal quality, while Audio-LLM Judges show selective, prompt-dependent detection that doesnt generalise across the dimension set. Neither class reliably captures linguistically structured speech errors. Our dataset, annotation schema, and evaluation code are publicly released to support more targeted and interpretable TTS evaluation.
%U https://aclanthology.org/2026.emnlp-main.1857/
%P 42215-42243
Markdown (Informal)
[Beyond Naturalness: Probing Automated TTS Evaluators on Linguistically Grounded Dimensions](https://aclanthology.org/2026.emnlp-main.1857/) (Bamgbose et al., EMNLP 2026)
ACL
- Oluwanifemi Bamgbose, Jash Shah, Simon Rosen, Lindsay Devon Brin, Hoang H Nguyen, Anke Koelzer, Rachel Hansen, Tara Bogavelli, and Fanny Riols. 2026. Beyond Naturalness: Probing Automated TTS Evaluators on Linguistically Grounded Dimensions. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 42215–42243, Budapest, Hungary. Association for Computational Linguistics.