@inproceedings{li-etal-2026-unison,
title = "{UNISON}: A Unified Sound Generation and Editing Framework via Deep {LLM} Fusion",
author = "Li, Zhaoqing and
Xu, Haoning and
Su, Jingran and
Liu, Yaofang and
Rao, Zhefan and
Wang, Huimeng and
Deng, Jiajun and
Wang, Tianzi and
Jin, Zengrui and
Liu, Rui and
Che, Haoxuan and
Liu, Xunying",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2015/",
pages = "45902--45922",
ISBN = "979-8-89176-459-0",
abstract = "We present UNISON, a latent flow-matching framework that unifies speech generation, sound generation, and audio editing within a single model. A single model handles text-to-audio, text-to-speech, zero-shot speaker cloning, mixed speech-and-sound generation, scene-level audio editing, speech-in-scene editing, and timed audio composition, all of which share a single set of weights. Our architecture features two core designs: (1) Layer-wise deep LLM fusion, which injects hidden states from uniformly sampled layers of a frozen MLLM into corresponding MM-DiT blocks via learned projections, providing depth-varying semantic conditioning. This configuration achieves better T2A FD and CLAP than a persistent penultimate-layer conditioning baseline; and (2) a unified multi-task architecture where task identity is encoded solely by a channel-wise mask and source audio is provided through VAE-encoded channel concatenation. Training is stabilized by an online GPU-side multi-task data synthesis pipeline with task-homogeneous batching and a two-stage curriculum. With 621M{--}732M trainable generative parameters, UNISON achieves competitive results across the evaluated domains. Its trainable generative backbone uses 1.8{--}4.9x fewer parameters than the closest unified systems considered, while deployment additionally requires a frozen 7B text encoder. Audio samples and source code are publicly available."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-unison">
<titleInfo>
<title>UNISON: A Unified Sound Generation and Editing Framework via Deep LLM Fusion</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhaoqing</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haoning</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingran</namePart>
<namePart type="family">Su</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yaofang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhefan</namePart>
<namePart type="family">Rao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Huimeng</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiajun</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianzi</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zengrui</namePart>
<namePart type="family">Jin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rui</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haoxuan</namePart>
<namePart type="family">Che</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xunying</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>We present UNISON, a latent flow-matching framework that unifies speech generation, sound generation, and audio editing within a single model. A single model handles text-to-audio, text-to-speech, zero-shot speaker cloning, mixed speech-and-sound generation, scene-level audio editing, speech-in-scene editing, and timed audio composition, all of which share a single set of weights. Our architecture features two core designs: (1) Layer-wise deep LLM fusion, which injects hidden states from uniformly sampled layers of a frozen MLLM into corresponding MM-DiT blocks via learned projections, providing depth-varying semantic conditioning. This configuration achieves better T2A FD and CLAP than a persistent penultimate-layer conditioning baseline; and (2) a unified multi-task architecture where task identity is encoded solely by a channel-wise mask and source audio is provided through VAE-encoded channel concatenation. Training is stabilized by an online GPU-side multi-task data synthesis pipeline with task-homogeneous batching and a two-stage curriculum. With 621M–732M trainable generative parameters, UNISON achieves competitive results across the evaluated domains. Its trainable generative backbone uses 1.8–4.9x fewer parameters than the closest unified systems considered, while deployment additionally requires a frozen 7B text encoder. Audio samples and source code are publicly available.</abstract>
<identifier type="citekey">li-etal-2026-unison</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2015/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>45902</start>
<end>45922</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T UNISON: A Unified Sound Generation and Editing Framework via Deep LLM Fusion
%A Li, Zhaoqing
%A Xu, Haoning
%A Su, Jingran
%A Liu, Yaofang
%A Rao, Zhefan
%A Wang, Huimeng
%A Deng, Jiajun
%A Wang, Tianzi
%A Jin, Zengrui
%A Liu, Rui
%A Che, Haoxuan
%A Liu, Xunying
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F li-etal-2026-unison
%X We present UNISON, a latent flow-matching framework that unifies speech generation, sound generation, and audio editing within a single model. A single model handles text-to-audio, text-to-speech, zero-shot speaker cloning, mixed speech-and-sound generation, scene-level audio editing, speech-in-scene editing, and timed audio composition, all of which share a single set of weights. Our architecture features two core designs: (1) Layer-wise deep LLM fusion, which injects hidden states from uniformly sampled layers of a frozen MLLM into corresponding MM-DiT blocks via learned projections, providing depth-varying semantic conditioning. This configuration achieves better T2A FD and CLAP than a persistent penultimate-layer conditioning baseline; and (2) a unified multi-task architecture where task identity is encoded solely by a channel-wise mask and source audio is provided through VAE-encoded channel concatenation. Training is stabilized by an online GPU-side multi-task data synthesis pipeline with task-homogeneous batching and a two-stage curriculum. With 621M–732M trainable generative parameters, UNISON achieves competitive results across the evaluated domains. Its trainable generative backbone uses 1.8–4.9x fewer parameters than the closest unified systems considered, while deployment additionally requires a frozen 7B text encoder. Audio samples and source code are publicly available.
%U https://aclanthology.org/2026.emnlp-main.2015/
%P 45902-45922
Markdown (Informal)
[UNISON: A Unified Sound Generation and Editing Framework via Deep LLM Fusion](https://aclanthology.org/2026.emnlp-main.2015/) (Li et al., EMNLP 2026)
ACL
- Zhaoqing Li, Haoning Xu, Jingran Su, Yaofang Liu, Zhefan Rao, Huimeng Wang, Jiajun Deng, Tianzi Wang, Zengrui Jin, Rui Liu, Haoxuan Che, and Xunying Liu. 2026. UNISON: A Unified Sound Generation and Editing Framework via Deep LLM Fusion. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 45902–45922, Budapest, Hungary. Association for Computational Linguistics.