@inproceedings{yeh-etal-2026-meld,
title = "{MELD}: Mel-Spectrogram-Based Speech Language Modeling with Discrete Latent Variables",
author = "Yeh, Sung-Lin and
Zhou, Wei and
Keren, Gil and
Le, Duc and
Meng, Zhong and
Tang, Hao and
Mahadeokar, Jay and
Kalinli, Ozlem and
Mourachko, Alexandre",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1584/",
pages = "36038--36050",
ISBN = "979-8-89176-459-0",
abstract = "Recent speech language models rely on encoders that are optimized separately from autoregressive models. Since these encoders are unaware of the downstream objectives, the extracted representations may not be optimal for downstream tasks. To address this limitation, we introduce a discrete latent variable model on mel spectrograms that jointly optimizes the encoder and the speech language model. Joint optimization not only brings improvements over codec-based and other mel-spectrogram-based baselines on zero-shot Text-to-Speech (TTS) and Speech-to-Text (STT) tasks, but also effectively alleviates common issues in autoregressive mel spectrogram modeling, such as prolonged silence generation and word omissions. Samples are available at the [demo page](https://anonymous29300909.github.io/anonymous2930-demo/)."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yeh-etal-2026-meld">
<titleInfo>
<title>MELD: Mel-Spectrogram-Based Speech Language Modeling with Discrete Latent Variables</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sung-Lin</namePart>
<namePart type="family">Yeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gil</namePart>
<namePart type="family">Keren</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Duc</namePart>
<namePart type="family">Le</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhong</namePart>
<namePart type="family">Meng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jay</namePart>
<namePart type="family">Mahadeokar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ozlem</namePart>
<namePart type="family">Kalinli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexandre</namePart>
<namePart type="family">Mourachko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Recent speech language models rely on encoders that are optimized separately from autoregressive models. Since these encoders are unaware of the downstream objectives, the extracted representations may not be optimal for downstream tasks. To address this limitation, we introduce a discrete latent variable model on mel spectrograms that jointly optimizes the encoder and the speech language model. Joint optimization not only brings improvements over codec-based and other mel-spectrogram-based baselines on zero-shot Text-to-Speech (TTS) and Speech-to-Text (STT) tasks, but also effectively alleviates common issues in autoregressive mel spectrogram modeling, such as prolonged silence generation and word omissions. Samples are available at the [demo page](https://anonymous29300909.github.io/anonymous2930-demo/).</abstract>
<identifier type="citekey">yeh-etal-2026-meld</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1584/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>36038</start>
<end>36050</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MELD: Mel-Spectrogram-Based Speech Language Modeling with Discrete Latent Variables
%A Yeh, Sung-Lin
%A Zhou, Wei
%A Keren, Gil
%A Le, Duc
%A Meng, Zhong
%A Tang, Hao
%A Mahadeokar, Jay
%A Kalinli, Ozlem
%A Mourachko, Alexandre
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yeh-etal-2026-meld
%X Recent speech language models rely on encoders that are optimized separately from autoregressive models. Since these encoders are unaware of the downstream objectives, the extracted representations may not be optimal for downstream tasks. To address this limitation, we introduce a discrete latent variable model on mel spectrograms that jointly optimizes the encoder and the speech language model. Joint optimization not only brings improvements over codec-based and other mel-spectrogram-based baselines on zero-shot Text-to-Speech (TTS) and Speech-to-Text (STT) tasks, but also effectively alleviates common issues in autoregressive mel spectrogram modeling, such as prolonged silence generation and word omissions. Samples are available at the [demo page](https://anonymous29300909.github.io/anonymous2930-demo/).
%U https://aclanthology.org/2026.emnlp-main.1584/
%P 36038-36050
Markdown (Informal)
[MELD: Mel-Spectrogram-Based Speech Language Modeling with Discrete Latent Variables](https://aclanthology.org/2026.emnlp-main.1584/) (Yeh et al., EMNLP 2026)
ACL
- Sung-Lin Yeh, Wei Zhou, Gil Keren, Duc Le, Zhong Meng, Hao Tang, Jay Mahadeokar, Ozlem Kalinli, and Alexandre Mourachko. 2026. MELD: Mel-Spectrogram-Based Speech Language Modeling with Discrete Latent Variables. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 36038–36050, Budapest, Hungary. Association for Computational Linguistics.