@inproceedings{renhang-etal-2026-jam,
title = "{JAM}: A Tiny Flow-based Song Generator with Fine-grained Controllability and Aesthetic Alignment",
author = "Renhang, Liu and
Hung, Chia-Yu and
Majumder, Navonil and
Herremans, Dorien and
Poria, Soujanya",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.48/",
pages = "966--981",
ISBN = "979-8-89176-462-0",
abstract = "Diffusion and flow-matching models have revolutionized automatic text-to-audio generation, increasingly capable of generating high-quality audio outputs for speech and acoustic events. However, creative audio generation involving music and songs still has room for improvement. Recent open lyrics-to-song models like DiffRhythm, ACE-Step, and LeVo have set an acceptable standard for recreational use but lack the fine-grained word-level controllability desired by musicians. To our knowledge, our flow-matching-based JAM is the first to provide word-level timing and duration control in song generation, enabling fine-grained vocal control. To better align generated songs with human preferences, we implement aesthetic alignment through Direct Preference Optimization, iteratively refining the model using a synthetic dataset without manual annotations. Additionally, we aim to standardize evaluation of lyrics-to-song models through our public evaluation dataset JAME. We demonstrate that JAM outperforms open models on music-specific attributes."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="renhang-etal-2026-jam">
<titleInfo>
<title>JAM: A Tiny Flow-based Song Generator with Fine-grained Controllability and Aesthetic Alignment</title>
</titleInfo>
<name type="personal">
<namePart type="given">Liu</namePart>
<namePart type="family">Renhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chia-Yu</namePart>
<namePart type="family">Hung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Navonil</namePart>
<namePart type="family">Majumder</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dorien</namePart>
<namePart type="family">Herremans</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Soujanya</namePart>
<namePart type="family">Poria</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Diffusion and flow-matching models have revolutionized automatic text-to-audio generation, increasingly capable of generating high-quality audio outputs for speech and acoustic events. However, creative audio generation involving music and songs still has room for improvement. Recent open lyrics-to-song models like DiffRhythm, ACE-Step, and LeVo have set an acceptable standard for recreational use but lack the fine-grained word-level controllability desired by musicians. To our knowledge, our flow-matching-based JAM is the first to provide word-level timing and duration control in song generation, enabling fine-grained vocal control. To better align generated songs with human preferences, we implement aesthetic alignment through Direct Preference Optimization, iteratively refining the model using a synthetic dataset without manual annotations. Additionally, we aim to standardize evaluation of lyrics-to-song models through our public evaluation dataset JAME. We demonstrate that JAM outperforms open models on music-specific attributes.</abstract>
<identifier type="citekey">renhang-etal-2026-jam</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.48/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>966</start>
<end>981</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T JAM: A Tiny Flow-based Song Generator with Fine-grained Controllability and Aesthetic Alignment
%A Renhang, Liu
%A Hung, Chia-Yu
%A Majumder, Navonil
%A Herremans, Dorien
%A Poria, Soujanya
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F renhang-etal-2026-jam
%X Diffusion and flow-matching models have revolutionized automatic text-to-audio generation, increasingly capable of generating high-quality audio outputs for speech and acoustic events. However, creative audio generation involving music and songs still has room for improvement. Recent open lyrics-to-song models like DiffRhythm, ACE-Step, and LeVo have set an acceptable standard for recreational use but lack the fine-grained word-level controllability desired by musicians. To our knowledge, our flow-matching-based JAM is the first to provide word-level timing and duration control in song generation, enabling fine-grained vocal control. To better align generated songs with human preferences, we implement aesthetic alignment through Direct Preference Optimization, iteratively refining the model using a synthetic dataset without manual annotations. Additionally, we aim to standardize evaluation of lyrics-to-song models through our public evaluation dataset JAME. We demonstrate that JAM outperforms open models on music-specific attributes.
%U https://aclanthology.org/2026.findings-emnlp.48/
%P 966-981
Markdown (Informal)
[JAM: A Tiny Flow-based Song Generator with Fine-grained Controllability and Aesthetic Alignment](https://aclanthology.org/2026.findings-emnlp.48/) (Renhang et al., Findings 2026)
ACL