@inproceedings{das-roy-2026-speak-changes,
title = "When How You Speak Changes What Models Refuse: Investigating Prosody-Conditioned Safety Divergence in Audio-Language Models",
author = "Das, Akash and
Roy, Ishan",
editor = "T.Y.S.S., Santosh and
Ghosh, Sreyan and
Duraiswami, Ramani and
Watanabe, Shinji and
Arora, Siddhant and
Kumar, Sonal and
Adi, Yossi and
Manocha, Dinesh and
Garcia Perera, Leibny Paola",
booktitle = "The Proceedings of Speech and Audio Language Models Workshop (2nd Edition)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.salma-1.3/",
pages = "18--37",
ISBN = "979-8-89176-496-5",
abstract = "Audio-Language Models (ALMs) natively fuse lexical content with continuous paralinguistic acoustics such as intonation, tone, and speaker identity. While considerable progress has been made in aligning the semantic outputs of text-based models against harmful instructions, current ALM safety alignment evaluation largely treats speech as a noisy text channel, leaving modality-specific vulnerabilities unexamined. In this work, we introduce and investigate Prosody-Conditioned Safety Divergence: a vulnerability where an ALM{'}s safety behavior shifts systematically based entirely on paralinguistic variation, even when lexical semantic payload is held completely constant. Rather than assuming the existence of coarse, high-variance acoustic ``authority'' subspaces, we conduct a controlled causal analysis to track how specific prosodic manipulations{---}including variations in pitch contour and intensity{---}interact with safety-associated latent directions across the encoder, projector, and decoder layers. Our preliminary empirical evaluations characterize the localization of this divergence, establishing a framework for targeted inference-time interventions that restore alignment metrics without compromising baseline utility on benign speech tasks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="das-roy-2026-speak-changes">
<titleInfo>
<title>When How You Speak Changes What Models Refuse: Investigating Prosody-Conditioned Safety Divergence in Audio-Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Akash</namePart>
<namePart type="family">Das</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ishan</namePart>
<namePart type="family">Roy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of Speech and Audio Language Models Workshop (2nd Edition)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sreyan</namePart>
<namePart type="family">Ghosh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ramani</namePart>
<namePart type="family">Duraiswami</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shinji</namePart>
<namePart type="family">Watanabe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siddhant</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sonal</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yossi</namePart>
<namePart type="family">Adi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dinesh</namePart>
<namePart type="family">Manocha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leibny</namePart>
<namePart type="given">Paola</namePart>
<namePart type="family">Garcia Perera</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-496-5</identifier>
</relatedItem>
<abstract>Audio-Language Models (ALMs) natively fuse lexical content with continuous paralinguistic acoustics such as intonation, tone, and speaker identity. While considerable progress has been made in aligning the semantic outputs of text-based models against harmful instructions, current ALM safety alignment evaluation largely treats speech as a noisy text channel, leaving modality-specific vulnerabilities unexamined. In this work, we introduce and investigate Prosody-Conditioned Safety Divergence: a vulnerability where an ALM’s safety behavior shifts systematically based entirely on paralinguistic variation, even when lexical semantic payload is held completely constant. Rather than assuming the existence of coarse, high-variance acoustic “authority” subspaces, we conduct a controlled causal analysis to track how specific prosodic manipulations—including variations in pitch contour and intensity—interact with safety-associated latent directions across the encoder, projector, and decoder layers. Our preliminary empirical evaluations characterize the localization of this divergence, establishing a framework for targeted inference-time interventions that restore alignment metrics without compromising baseline utility on benign speech tasks.</abstract>
<identifier type="citekey">das-roy-2026-speak-changes</identifier>
<location>
<url>https://aclanthology.org/2026.salma-1.3/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>18</start>
<end>37</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When How You Speak Changes What Models Refuse: Investigating Prosody-Conditioned Safety Divergence in Audio-Language Models
%A Das, Akash
%A Roy, Ishan
%Y T.Y.S.S., Santosh
%Y Ghosh, Sreyan
%Y Duraiswami, Ramani
%Y Watanabe, Shinji
%Y Arora, Siddhant
%Y Kumar, Sonal
%Y Adi, Yossi
%Y Manocha, Dinesh
%Y Garcia Perera, Leibny Paola
%S The Proceedings of Speech and Audio Language Models Workshop (2nd Edition)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-496-5
%F das-roy-2026-speak-changes
%X Audio-Language Models (ALMs) natively fuse lexical content with continuous paralinguistic acoustics such as intonation, tone, and speaker identity. While considerable progress has been made in aligning the semantic outputs of text-based models against harmful instructions, current ALM safety alignment evaluation largely treats speech as a noisy text channel, leaving modality-specific vulnerabilities unexamined. In this work, we introduce and investigate Prosody-Conditioned Safety Divergence: a vulnerability where an ALM’s safety behavior shifts systematically based entirely on paralinguistic variation, even when lexical semantic payload is held completely constant. Rather than assuming the existence of coarse, high-variance acoustic “authority” subspaces, we conduct a controlled causal analysis to track how specific prosodic manipulations—including variations in pitch contour and intensity—interact with safety-associated latent directions across the encoder, projector, and decoder layers. Our preliminary empirical evaluations characterize the localization of this divergence, establishing a framework for targeted inference-time interventions that restore alignment metrics without compromising baseline utility on benign speech tasks.
%U https://aclanthology.org/2026.salma-1.3/
%P 18-37
Markdown (Informal)
[When How You Speak Changes What Models Refuse: Investigating Prosody-Conditioned Safety Divergence in Audio-Language Models](https://aclanthology.org/2026.salma-1.3/) (Das & Roy, SALMA 2026)
ACL