@inproceedings{sim-lee-2026-evaluating,
title = "Evaluating the Evaluators: A Systematic Analysis of Reference Bias in Audio Captioning Metrics",
author = "Sim, Jaeheon and
Lee, Kyogu",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1942/",
pages = "44191--44207",
ISBN = "979-8-89176-459-0",
abstract = "Current evaluation metrics for automated audio captioning (AAC) rely on reference-based comparison, implicitly assuming that a limited reference set adequately represents valid descriptions. However, the inherent cross-modal gap between audio and language gives rise to a many-to-many correspondence: a single audio event can be accurately described by numerous semantically equivalent yet lexically diverse captions. We first demonstrate the resulting instability through ground-truth intra-evaluation on Clotho and AudioCaps, evaluating each human caption against the remaining four references using a leave-one-out protocol. To provide a controlled demonstration, we use a CVAE-based AAC model that generates diverse candidates replicating the ground-truth captions, showing that n-gram metrics reward surface-level replication. We then conduct non-word replacement, synonym replacement, and adjacent-token swapping experiments on both datasets. Overlap metrics penalize synonym replacements almost as strongly as non-word corruption, whereas embedding and CLAP-based metrics respond weakly to local word-order changes. Additional evaluation of CAF-Score and FLEUR (AF3) shows partial progress, but neither configuration clearly separates synonym replacement from adjacent swapping. No evaluated metric jointly provides acoustic grounding, tolerance to valid lexical variation, and local word-order sensitivity. We therefore contribute a reusable AAC metric diagnostic protocol for exposing these trade-offs and guiding future metric development."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sim-lee-2026-evaluating">
<titleInfo>
<title>Evaluating the Evaluators: A Systematic Analysis of Reference Bias in Audio Captioning Metrics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jaeheon</namePart>
<namePart type="family">Sim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kyogu</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Current evaluation metrics for automated audio captioning (AAC) rely on reference-based comparison, implicitly assuming that a limited reference set adequately represents valid descriptions. However, the inherent cross-modal gap between audio and language gives rise to a many-to-many correspondence: a single audio event can be accurately described by numerous semantically equivalent yet lexically diverse captions. We first demonstrate the resulting instability through ground-truth intra-evaluation on Clotho and AudioCaps, evaluating each human caption against the remaining four references using a leave-one-out protocol. To provide a controlled demonstration, we use a CVAE-based AAC model that generates diverse candidates replicating the ground-truth captions, showing that n-gram metrics reward surface-level replication. We then conduct non-word replacement, synonym replacement, and adjacent-token swapping experiments on both datasets. Overlap metrics penalize synonym replacements almost as strongly as non-word corruption, whereas embedding and CLAP-based metrics respond weakly to local word-order changes. Additional evaluation of CAF-Score and FLEUR (AF3) shows partial progress, but neither configuration clearly separates synonym replacement from adjacent swapping. No evaluated metric jointly provides acoustic grounding, tolerance to valid lexical variation, and local word-order sensitivity. We therefore contribute a reusable AAC metric diagnostic protocol for exposing these trade-offs and guiding future metric development.</abstract>
<identifier type="citekey">sim-lee-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1942/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44191</start>
<end>44207</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating the Evaluators: A Systematic Analysis of Reference Bias in Audio Captioning Metrics
%A Sim, Jaeheon
%A Lee, Kyogu
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F sim-lee-2026-evaluating
%X Current evaluation metrics for automated audio captioning (AAC) rely on reference-based comparison, implicitly assuming that a limited reference set adequately represents valid descriptions. However, the inherent cross-modal gap between audio and language gives rise to a many-to-many correspondence: a single audio event can be accurately described by numerous semantically equivalent yet lexically diverse captions. We first demonstrate the resulting instability through ground-truth intra-evaluation on Clotho and AudioCaps, evaluating each human caption against the remaining four references using a leave-one-out protocol. To provide a controlled demonstration, we use a CVAE-based AAC model that generates diverse candidates replicating the ground-truth captions, showing that n-gram metrics reward surface-level replication. We then conduct non-word replacement, synonym replacement, and adjacent-token swapping experiments on both datasets. Overlap metrics penalize synonym replacements almost as strongly as non-word corruption, whereas embedding and CLAP-based metrics respond weakly to local word-order changes. Additional evaluation of CAF-Score and FLEUR (AF3) shows partial progress, but neither configuration clearly separates synonym replacement from adjacent swapping. No evaluated metric jointly provides acoustic grounding, tolerance to valid lexical variation, and local word-order sensitivity. We therefore contribute a reusable AAC metric diagnostic protocol for exposing these trade-offs and guiding future metric development.
%U https://aclanthology.org/2026.emnlp-main.1942/
%P 44191-44207
Markdown (Informal)
[Evaluating the Evaluators: A Systematic Analysis of Reference Bias in Audio Captioning Metrics](https://aclanthology.org/2026.emnlp-main.1942/) (Sim & Lee, EMNLP 2026)
ACL