@inproceedings{bogavelli-etal-2026-eva,
title = "{EVA}-Bench: A New End-to-end Framework for Evaluating Voice Agents",
author = "Bogavelli, Tara and
Melancon, Gabrielle Gauthier and
Stankiewicz, Katrina and
Bamgbose, Oluwanifemi and
Riols, Fanny and
Nguyen, Hoang H and
Mehndiratta, Raghav and
Brin, Lindsay Devon and
Subramani, Hari and
Marinier, Joseph and
Nemala, Sridhar Krishna and
Madamala, Anil and
Sunkara, Srinivas",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2182/",
pages = "46671--46748",
ISBN = "979-8-89176-462-0",
abstract = "Voice agents are increasingly deployed across enterprise applications. However, no existing benchmark jointly addresses realistic conversation simulation and comprehensive voice-specific evaluation. We present EVA-Bench, an end-to-end evaluation framework that addresses both. On the simulation side, EVA-Bench orchestrates dynamic bot-to-bot audio conversations with automatic simulation validation that detects user simulator error and appropriately regenerates conversations before scoring. On the measurement side, EVA-Bench introduces two composite metrics: EVA-A (Accuracy) and EVA-X (Experience). EVA-Bench includes 213 scenarios across three enterprise domains, a controlled perturbation suite for accent and noise robustness, and multi-trial measurements that distinguish peak from reliable capability. Across 12 systems spanning three architectures, we find: (1) no system simultaneously exceeds 0.5 on both EVA-A pass@1 and EVA-X pass@1; (2) peak and reliable performance diverge substantially (median pass@k - pass{\textasciicircum}k gap of 0.44 on EVA-A; and (3) accent and noise perturbations expose substantial robustness gaps, with effects varying across architectures, systems, and metrics (mean {\ensuremath{\Delta}} up to 0.314). We release EVA-Bench under an open-source license."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="bogavelli-etal-2026-eva">
<titleInfo>
<title>EVA-Bench: A New End-to-end Framework for Evaluating Voice Agents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tara</namePart>
<namePart type="family">Bogavelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabrielle</namePart>
<namePart type="given">Gauthier</namePart>
<namePart type="family">Melancon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Katrina</namePart>
<namePart type="family">Stankiewicz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Oluwanifemi</namePart>
<namePart type="family">Bamgbose</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fanny</namePart>
<namePart type="family">Riols</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hoang</namePart>
<namePart type="given">H</namePart>
<namePart type="family">Nguyen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raghav</namePart>
<namePart type="family">Mehndiratta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lindsay</namePart>
<namePart type="given">Devon</namePart>
<namePart type="family">Brin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hari</namePart>
<namePart type="family">Subramani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Joseph</namePart>
<namePart type="family">Marinier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sridhar</namePart>
<namePart type="given">Krishna</namePart>
<namePart type="family">Nemala</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anil</namePart>
<namePart type="family">Madamala</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Srinivas</namePart>
<namePart type="family">Sunkara</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Voice agents are increasingly deployed across enterprise applications. However, no existing benchmark jointly addresses realistic conversation simulation and comprehensive voice-specific evaluation. We present EVA-Bench, an end-to-end evaluation framework that addresses both. On the simulation side, EVA-Bench orchestrates dynamic bot-to-bot audio conversations with automatic simulation validation that detects user simulator error and appropriately regenerates conversations before scoring. On the measurement side, EVA-Bench introduces two composite metrics: EVA-A (Accuracy) and EVA-X (Experience). EVA-Bench includes 213 scenarios across three enterprise domains, a controlled perturbation suite for accent and noise robustness, and multi-trial measurements that distinguish peak from reliable capability. Across 12 systems spanning three architectures, we find: (1) no system simultaneously exceeds 0.5 on both EVA-A pass@1 and EVA-X pass@1; (2) peak and reliable performance diverge substantially (median pass@k - pass⌃k gap of 0.44 on EVA-A; and (3) accent and noise perturbations expose substantial robustness gaps, with effects varying across architectures, systems, and metrics (mean \ensuremathΔ up to 0.314). We release EVA-Bench under an open-source license.</abstract>
<identifier type="citekey">bogavelli-etal-2026-eva</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2182/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>46671</start>
<end>46748</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T EVA-Bench: A New End-to-end Framework for Evaluating Voice Agents
%A Bogavelli, Tara
%A Melancon, Gabrielle Gauthier
%A Stankiewicz, Katrina
%A Bamgbose, Oluwanifemi
%A Riols, Fanny
%A Nguyen, Hoang H.
%A Mehndiratta, Raghav
%A Brin, Lindsay Devon
%A Subramani, Hari
%A Marinier, Joseph
%A Nemala, Sridhar Krishna
%A Madamala, Anil
%A Sunkara, Srinivas
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F bogavelli-etal-2026-eva
%X Voice agents are increasingly deployed across enterprise applications. However, no existing benchmark jointly addresses realistic conversation simulation and comprehensive voice-specific evaluation. We present EVA-Bench, an end-to-end evaluation framework that addresses both. On the simulation side, EVA-Bench orchestrates dynamic bot-to-bot audio conversations with automatic simulation validation that detects user simulator error and appropriately regenerates conversations before scoring. On the measurement side, EVA-Bench introduces two composite metrics: EVA-A (Accuracy) and EVA-X (Experience). EVA-Bench includes 213 scenarios across three enterprise domains, a controlled perturbation suite for accent and noise robustness, and multi-trial measurements that distinguish peak from reliable capability. Across 12 systems spanning three architectures, we find: (1) no system simultaneously exceeds 0.5 on both EVA-A pass@1 and EVA-X pass@1; (2) peak and reliable performance diverge substantially (median pass@k - pass⌃k gap of 0.44 on EVA-A; and (3) accent and noise perturbations expose substantial robustness gaps, with effects varying across architectures, systems, and metrics (mean \ensuremathΔ up to 0.314). We release EVA-Bench under an open-source license.
%U https://aclanthology.org/2026.findings-emnlp.2182/
%P 46671-46748
Markdown (Informal)
[EVA-Bench: A New End-to-end Framework for Evaluating Voice Agents](https://aclanthology.org/2026.findings-emnlp.2182/) (Bogavelli et al., Findings 2026)
ACL
- Tara Bogavelli, Gabrielle Gauthier Melancon, Katrina Stankiewicz, Oluwanifemi Bamgbose, Fanny Riols, Hoang H Nguyen, Raghav Mehndiratta, Lindsay Devon Brin, Hari Subramani, Joseph Marinier, Sridhar Krishna Nemala, Anil Madamala, and Srinivas Sunkara. 2026. EVA-Bench: A New End-to-end Framework for Evaluating Voice Agents. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 46671–46748, Budapest, Hungary. Association for Computational Linguistics.