@inproceedings{wu-horner-2026-heard,
title = "Heard or Read? Separating Text Shortcuts from Audio Perception in Music Multiple Choice",
author = "Wu, Xinyang and
Horner, Andrew",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.950/",
pages = "21907--21922",
ISBN = "979-8-89176-459-0",
abstract = "Multiple-choice music benchmarks can reward two different behaviors: listening to the audio and exploiting cues in the question/options. We introduce ListenOnly-Music, a 947-item perception-required slice of MuChoMusic and MMAU-Pro, by filtering out items that two strong text-only reasoners can solve from the question and answer options alone. We audit five public audio-LLMs, from early music-specialist systems to recent 7B-8B chat models, under audio, text-only, noise, and mismatched-audio conditions. On ListenOnly-Music, earlier 2023 systems are statistically equivalent to no listening under a pre-specified TOST margin, whereas newer models show positive audio contribution. We then evaluate two inference-time interventions. Audio-contrastive decoding improves perception-required accuracy but reduces accuracy on text-solvable items, making it a diagnostic probe rather than a blanket decoding policy. Audio-first chain-of-thought helps caption-style models but hurts MCQ-tuned models. Overall, music-MCQ accuracy should be reported separately for text-solvable and perception-required items: recent audio-LLMs do listen more than earlier systems, but benchmark shortcuts still obscure how much progress is genuinely audio-grounded."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wu-horner-2026-heard">
<titleInfo>
<title>Heard or Read? Separating Text Shortcuts from Audio Perception in Music Multiple Choice</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xinyang</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrew</namePart>
<namePart type="family">Horner</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Multiple-choice music benchmarks can reward two different behaviors: listening to the audio and exploiting cues in the question/options. We introduce ListenOnly-Music, a 947-item perception-required slice of MuChoMusic and MMAU-Pro, by filtering out items that two strong text-only reasoners can solve from the question and answer options alone. We audit five public audio-LLMs, from early music-specialist systems to recent 7B-8B chat models, under audio, text-only, noise, and mismatched-audio conditions. On ListenOnly-Music, earlier 2023 systems are statistically equivalent to no listening under a pre-specified TOST margin, whereas newer models show positive audio contribution. We then evaluate two inference-time interventions. Audio-contrastive decoding improves perception-required accuracy but reduces accuracy on text-solvable items, making it a diagnostic probe rather than a blanket decoding policy. Audio-first chain-of-thought helps caption-style models but hurts MCQ-tuned models. Overall, music-MCQ accuracy should be reported separately for text-solvable and perception-required items: recent audio-LLMs do listen more than earlier systems, but benchmark shortcuts still obscure how much progress is genuinely audio-grounded.</abstract>
<identifier type="citekey">wu-horner-2026-heard</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.950/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>21907</start>
<end>21922</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Heard or Read? Separating Text Shortcuts from Audio Perception in Music Multiple Choice
%A Wu, Xinyang
%A Horner, Andrew
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wu-horner-2026-heard
%X Multiple-choice music benchmarks can reward two different behaviors: listening to the audio and exploiting cues in the question/options. We introduce ListenOnly-Music, a 947-item perception-required slice of MuChoMusic and MMAU-Pro, by filtering out items that two strong text-only reasoners can solve from the question and answer options alone. We audit five public audio-LLMs, from early music-specialist systems to recent 7B-8B chat models, under audio, text-only, noise, and mismatched-audio conditions. On ListenOnly-Music, earlier 2023 systems are statistically equivalent to no listening under a pre-specified TOST margin, whereas newer models show positive audio contribution. We then evaluate two inference-time interventions. Audio-contrastive decoding improves perception-required accuracy but reduces accuracy on text-solvable items, making it a diagnostic probe rather than a blanket decoding policy. Audio-first chain-of-thought helps caption-style models but hurts MCQ-tuned models. Overall, music-MCQ accuracy should be reported separately for text-solvable and perception-required items: recent audio-LLMs do listen more than earlier systems, but benchmark shortcuts still obscure how much progress is genuinely audio-grounded.
%U https://aclanthology.org/2026.emnlp-main.950/
%P 21907-21922
Markdown (Informal)
[Heard or Read? Separating Text Shortcuts from Audio Perception in Music Multiple Choice](https://aclanthology.org/2026.emnlp-main.950/) (Wu & Horner, EMNLP 2026)
ACL