@inproceedings{salinas-etal-2026-echolens,
title = "{E}cho{L}ens: A Human-Speech Dataset for Auditing Demographic Sensitivity in Audio-Language Models",
author = "Salinas, Alejandro and
Koenecke, Allison and
Nyarko, Julian",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2027/",
doi = "10.18653/2026.findings-emnlp.2027",
pages = "43127--43178",
ISBN = "979-8-89176-462-0",
abstract = "Voice interfaces are increasingly moving away from transcription pipelines towards end-to-end systems that directly respond to audio inputs. This development in turn requires a shift in evaluation methodology away from transcription accuracy and towards more substantive markers such as response validity. We introduce EchoLens, a demographically stratified dataset of 20,008 recordings from 515 adult participants residing in the U.S., with self-reported demographic information across race, gender, age, accent and primary language, among others. Each participant speaks out loud and verbatim a randomized subset of 69 advice-seeking and estimation prompts spanning 11 domains grounded in the American Time Use Survey. Most prompts elicit quantitative responses from the model, allowing for direct analysis of output distributions across demographic subgroups without requiring reliance on LLM-as-a-judge. EchoLens also contains a documented audit protocol, which{---}in an illustrative application{---}we use to evaluate six current audio-language models. Under this assessment, we find that racial disparities are more frequent and larger in magnitude than gender disparities, with particular concentration in two models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="salinas-etal-2026-echolens">
<titleInfo>
<title>EchoLens: A Human-Speech Dataset for Auditing Demographic Sensitivity in Audio-Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alejandro</namePart>
<namePart type="family">Salinas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Allison</namePart>
<namePart type="family">Koenecke</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julian</namePart>
<namePart type="family">Nyarko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Voice interfaces are increasingly moving away from transcription pipelines towards end-to-end systems that directly respond to audio inputs. This development in turn requires a shift in evaluation methodology away from transcription accuracy and towards more substantive markers such as response validity. We introduce EchoLens, a demographically stratified dataset of 20,008 recordings from 515 adult participants residing in the U.S., with self-reported demographic information across race, gender, age, accent and primary language, among others. Each participant speaks out loud and verbatim a randomized subset of 69 advice-seeking and estimation prompts spanning 11 domains grounded in the American Time Use Survey. Most prompts elicit quantitative responses from the model, allowing for direct analysis of output distributions across demographic subgroups without requiring reliance on LLM-as-a-judge. EchoLens also contains a documented audit protocol, which—in an illustrative application—we use to evaluate six current audio-language models. Under this assessment, we find that racial disparities are more frequent and larger in magnitude than gender disparities, with particular concentration in two models.</abstract>
<identifier type="citekey">salinas-etal-2026-echolens</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.2027</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2027/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>43127</start>
<end>43178</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T EchoLens: A Human-Speech Dataset for Auditing Demographic Sensitivity in Audio-Language Models
%A Salinas, Alejandro
%A Koenecke, Allison
%A Nyarko, Julian
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F salinas-etal-2026-echolens
%X Voice interfaces are increasingly moving away from transcription pipelines towards end-to-end systems that directly respond to audio inputs. This development in turn requires a shift in evaluation methodology away from transcription accuracy and towards more substantive markers such as response validity. We introduce EchoLens, a demographically stratified dataset of 20,008 recordings from 515 adult participants residing in the U.S., with self-reported demographic information across race, gender, age, accent and primary language, among others. Each participant speaks out loud and verbatim a randomized subset of 69 advice-seeking and estimation prompts spanning 11 domains grounded in the American Time Use Survey. Most prompts elicit quantitative responses from the model, allowing for direct analysis of output distributions across demographic subgroups without requiring reliance on LLM-as-a-judge. EchoLens also contains a documented audit protocol, which—in an illustrative application—we use to evaluate six current audio-language models. Under this assessment, we find that racial disparities are more frequent and larger in magnitude than gender disparities, with particular concentration in two models.
%R 10.18653/2026.findings-emnlp.2027
%U https://aclanthology.org/2026.findings-emnlp.2027/
%U https://doi.org/10.18653/2026.findings-emnlp.2027
%P 43127-43178
Markdown (Informal)
[EchoLens: A Human-Speech Dataset for Auditing Demographic Sensitivity in Audio-Language Models](https://aclanthology.org/2026.findings-emnlp.2027/) (Salinas et al., Findings 2026)
ACL