@inproceedings{koh-etal-2026-factorization,
title = "When i.i.d. Factorization Breaks Language: The Structural Failure of Discrete Diffusion Language Models",
author = "Koh, Hyukhun and
Joo, Seongho and
Pyun, Haesung and
Jung, Kyomin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.193/",
pages = "4417--4436",
ISBN = "979-8-89176-459-0",
abstract = "Discrete diffusion language models (DDLMs) offer potential advantages for language generation, such as parallel decoding and iterative refinement, but they often underperform autoregressive (AR) models and exhibit higher training instability. Recent approaches incorporate AR training signals to stabilize training, yet this introduces strong sequential inductive biases that limit the benefits of non-autoregressive generation. In this work, we analyze the structural causes of degeneration in non-AR DDLMs. We show that standard diffusion formulations rely on an i.i.d. corruption process and factorized sampling structure that implicitly assume token exchangeability, which conflicts with the non-exchangeable dependency structure of natural language. This mismatch leads to degeneration under semantic sparsity, where contextual signals are insufficient. To guarantee stable language generation, we propose the \textit{Dependency-Coupling Principle}, a structural constraint that preserves necessary contextual dependencies during diffusion generation and provides insight into the performance gap between AR and diffusion language models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="koh-etal-2026-factorization">
<titleInfo>
<title>When i.i.d. Factorization Breaks Language: The Structural Failure of Discrete Diffusion Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hyukhun</namePart>
<namePart type="family">Koh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Seongho</namePart>
<namePart type="family">Joo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haesung</namePart>
<namePart type="family">Pyun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kyomin</namePart>
<namePart type="family">Jung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Discrete diffusion language models (DDLMs) offer potential advantages for language generation, such as parallel decoding and iterative refinement, but they often underperform autoregressive (AR) models and exhibit higher training instability. Recent approaches incorporate AR training signals to stabilize training, yet this introduces strong sequential inductive biases that limit the benefits of non-autoregressive generation. In this work, we analyze the structural causes of degeneration in non-AR DDLMs. We show that standard diffusion formulations rely on an i.i.d. corruption process and factorized sampling structure that implicitly assume token exchangeability, which conflicts with the non-exchangeable dependency structure of natural language. This mismatch leads to degeneration under semantic sparsity, where contextual signals are insufficient. To guarantee stable language generation, we propose the Dependency-Coupling Principle, a structural constraint that preserves necessary contextual dependencies during diffusion generation and provides insight into the performance gap between AR and diffusion language models.</abstract>
<identifier type="citekey">koh-etal-2026-factorization</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.193/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>4417</start>
<end>4436</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When i.i.d. Factorization Breaks Language: The Structural Failure of Discrete Diffusion Language Models
%A Koh, Hyukhun
%A Joo, Seongho
%A Pyun, Haesung
%A Jung, Kyomin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F koh-etal-2026-factorization
%X Discrete diffusion language models (DDLMs) offer potential advantages for language generation, such as parallel decoding and iterative refinement, but they often underperform autoregressive (AR) models and exhibit higher training instability. Recent approaches incorporate AR training signals to stabilize training, yet this introduces strong sequential inductive biases that limit the benefits of non-autoregressive generation. In this work, we analyze the structural causes of degeneration in non-AR DDLMs. We show that standard diffusion formulations rely on an i.i.d. corruption process and factorized sampling structure that implicitly assume token exchangeability, which conflicts with the non-exchangeable dependency structure of natural language. This mismatch leads to degeneration under semantic sparsity, where contextual signals are insufficient. To guarantee stable language generation, we propose the Dependency-Coupling Principle, a structural constraint that preserves necessary contextual dependencies during diffusion generation and provides insight into the performance gap between AR and diffusion language models.
%U https://aclanthology.org/2026.emnlp-main.193/
%P 4417-4436
Markdown (Informal)
[When i.i.d. Factorization Breaks Language: The Structural Failure of Discrete Diffusion Language Models](https://aclanthology.org/2026.emnlp-main.193/) (Koh et al., EMNLP 2026)
ACL