@inproceedings{herron-etal-2026-responsible,
title = "Responsible Benchmarking of Fairness for Automatic Speech Recognition",
author = "Herron, Felix E. and
Richard, Ange and
Portet, Fran{\c{c}}ois and
Allauzen, Alexandre and
Rossato, Solange",
editor = "Hosseini-Kivanani, Nina and
Brutti, Alessio and
Matassoni, Marco and
Dowerah, Sandipana and
Liga, Davide and
Schommer, Christoph",
booktitle = "Proceedings of Speech Language Models in Low-Resource Settings: Performance, Evaluation, and Bias Analysis ({SPEAKABLE}) @ {LREC} 2026",
month = may,
year = "2026",
address = "Palma, Mallorca (Spain)",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.speakable-1.8/",
doi = "10.63317/3jpc2uj4pp3g",
pages = "66--78",
abstract = "Many studies have shown automatic speech processing (ASR) systems have unequal performance across speaker groups (SG{'}s). However, the manner in which such studies arrive at this conclusion is inconsistent. To pave the way for more reliable results in future studies, we lay out best practices for benchmarking ASR fairness based on literature from machine learning fairness, social sciences, and speech science. We then perform a case study on the Fair-speech benchmark, applying aforementioned best practices, and discuss how failing to do so can result in erroneous conclusions. On the whole, we advocate for as fine-grained an analysis as possible, taking into account as many variables as are available, in order to eschew dataset-level bias."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="herron-etal-2026-responsible">
<titleInfo>
<title>Responsible Benchmarking of Fairness for Automatic Speech Recognition</title>
</titleInfo>
<name type="personal">
<namePart type="given">Felix</namePart>
<namePart type="given">E</namePart>
<namePart type="family">Herron</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ange</namePart>
<namePart type="family">Richard</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">François</namePart>
<namePart type="family">Portet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexandre</namePart>
<namePart type="family">Allauzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Solange</namePart>
<namePart type="family">Rossato</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of Speech Language Models in Low-Resource Settings: Performance, Evaluation, and Bias Analysis (SPEAKABLE) @ LREC 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nina</namePart>
<namePart type="family">Hosseini-Kivanani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alessio</namePart>
<namePart type="family">Brutti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marco</namePart>
<namePart type="family">Matassoni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sandipana</namePart>
<namePart type="family">Dowerah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Davide</namePart>
<namePart type="family">Liga</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christoph</namePart>
<namePart type="family">Schommer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca (Spain)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Many studies have shown automatic speech processing (ASR) systems have unequal performance across speaker groups (SG’s). However, the manner in which such studies arrive at this conclusion is inconsistent. To pave the way for more reliable results in future studies, we lay out best practices for benchmarking ASR fairness based on literature from machine learning fairness, social sciences, and speech science. We then perform a case study on the Fair-speech benchmark, applying aforementioned best practices, and discuss how failing to do so can result in erroneous conclusions. On the whole, we advocate for as fine-grained an analysis as possible, taking into account as many variables as are available, in order to eschew dataset-level bias.</abstract>
<identifier type="citekey">herron-etal-2026-responsible</identifier>
<identifier type="doi">10.63317/3jpc2uj4pp3g</identifier>
<location>
<url>https://aclanthology.org/2026.speakable-1.8/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>66</start>
<end>78</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Responsible Benchmarking of Fairness for Automatic Speech Recognition
%A Herron, Felix E.
%A Richard, Ange
%A Portet, François
%A Allauzen, Alexandre
%A Rossato, Solange
%Y Hosseini-Kivanani, Nina
%Y Brutti, Alessio
%Y Matassoni, Marco
%Y Dowerah, Sandipana
%Y Liga, Davide
%Y Schommer, Christoph
%S Proceedings of Speech Language Models in Low-Resource Settings: Performance, Evaluation, and Bias Analysis (SPEAKABLE) @ LREC 2026
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca (Spain)
%F herron-etal-2026-responsible
%X Many studies have shown automatic speech processing (ASR) systems have unequal performance across speaker groups (SG’s). However, the manner in which such studies arrive at this conclusion is inconsistent. To pave the way for more reliable results in future studies, we lay out best practices for benchmarking ASR fairness based on literature from machine learning fairness, social sciences, and speech science. We then perform a case study on the Fair-speech benchmark, applying aforementioned best practices, and discuss how failing to do so can result in erroneous conclusions. On the whole, we advocate for as fine-grained an analysis as possible, taking into account as many variables as are available, in order to eschew dataset-level bias.
%R 10.63317/3jpc2uj4pp3g
%U https://aclanthology.org/2026.speakable-1.8/
%U https://doi.org/10.63317/3jpc2uj4pp3g
%P 66-78
Markdown (Informal)
[Responsible Benchmarking of Fairness for Automatic Speech Recognition](https://aclanthology.org/2026.speakable-1.8/) (Herron et al., SPEAKABLE 2026)
ACL
- Felix E. Herron, Ange Richard, François Portet, Alexandre Allauzen, and Solange Rossato. 2026. Responsible Benchmarking of Fairness for Automatic Speech Recognition. In Proceedings of Speech Language Models in Low-Resource Settings: Performance, Evaluation, and Bias Analysis (SPEAKABLE) @ LREC 2026, pages 66–78, Palma, Mallorca (Spain). ELRA Language Resources Association (ELRA).