@inproceedings{pan-etal-2026-fast,
title = "Fast Block Diffusion Training via Multi-Mask Context Sharing",
author = "Pan, Tengyu and
Zhao, Weilin and
Sun, Yutao and
Li, Yanghao and
Wang, Jianyong and
Han, Xu",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2034/",
pages = "46353--46370",
ISBN = "979-8-89176-459-0",
abstract = "Block Diffusion Language Models generate multiple blocks of text sequentially, while tokens within each block are denoised in parallel, enabling efficient parallel decoding. However, their training suffers from low computational utilization: each sample is duplicated into a clean context and a masked copy, yet only the masked portion contributes to the training loss, resulting in merely 50{\%} compute utilization. In this paper, we propose Multi-Mask Training (MMT), a simple yet effective training paradigm that significantly improves computational efficiency through context sharing. Our key idea is to generate $K$ different mask patterns for the same clean text and process them together in a single forward pass via a sparse attention mechanism, where each masked segment attends to the shared clean context while remaining isolated from other segments. In pretraining, our proposed approach yields better performance under matched compute, despite being exposed to fewer unique training instances. Under a matched-step protocol, MMT maintains comparable downstream quality while improving training efficiency in AR-to-Diffusion adaptation. This reveals a compelling trade-off between data uniqueness and training efficiency in block diffusion models, demonstrating that the rich training signals from diverse mask samplings can effectively compensate for reduced exposure to unique data."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="pan-etal-2026-fast">
<titleInfo>
<title>Fast Block Diffusion Training via Multi-Mask Context Sharing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tengyu</namePart>
<namePart type="family">Pan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weilin</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yutao</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanghao</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianyong</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xu</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Block Diffusion Language Models generate multiple blocks of text sequentially, while tokens within each block are denoised in parallel, enabling efficient parallel decoding. However, their training suffers from low computational utilization: each sample is duplicated into a clean context and a masked copy, yet only the masked portion contributes to the training loss, resulting in merely 50% compute utilization. In this paper, we propose Multi-Mask Training (MMT), a simple yet effective training paradigm that significantly improves computational efficiency through context sharing. Our key idea is to generate K different mask patterns for the same clean text and process them together in a single forward pass via a sparse attention mechanism, where each masked segment attends to the shared clean context while remaining isolated from other segments. In pretraining, our proposed approach yields better performance under matched compute, despite being exposed to fewer unique training instances. Under a matched-step protocol, MMT maintains comparable downstream quality while improving training efficiency in AR-to-Diffusion adaptation. This reveals a compelling trade-off between data uniqueness and training efficiency in block diffusion models, demonstrating that the rich training signals from diverse mask samplings can effectively compensate for reduced exposure to unique data.</abstract>
<identifier type="citekey">pan-etal-2026-fast</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2034/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>46353</start>
<end>46370</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Fast Block Diffusion Training via Multi-Mask Context Sharing
%A Pan, Tengyu
%A Zhao, Weilin
%A Sun, Yutao
%A Li, Yanghao
%A Wang, Jianyong
%A Han, Xu
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F pan-etal-2026-fast
%X Block Diffusion Language Models generate multiple blocks of text sequentially, while tokens within each block are denoised in parallel, enabling efficient parallel decoding. However, their training suffers from low computational utilization: each sample is duplicated into a clean context and a masked copy, yet only the masked portion contributes to the training loss, resulting in merely 50% compute utilization. In this paper, we propose Multi-Mask Training (MMT), a simple yet effective training paradigm that significantly improves computational efficiency through context sharing. Our key idea is to generate K different mask patterns for the same clean text and process them together in a single forward pass via a sparse attention mechanism, where each masked segment attends to the shared clean context while remaining isolated from other segments. In pretraining, our proposed approach yields better performance under matched compute, despite being exposed to fewer unique training instances. Under a matched-step protocol, MMT maintains comparable downstream quality while improving training efficiency in AR-to-Diffusion adaptation. This reveals a compelling trade-off between data uniqueness and training efficiency in block diffusion models, demonstrating that the rich training signals from diverse mask samplings can effectively compensate for reduced exposure to unique data.
%U https://aclanthology.org/2026.emnlp-main.2034/
%P 46353-46370
Markdown (Informal)
[Fast Block Diffusion Training via Multi-Mask Context Sharing](https://aclanthology.org/2026.emnlp-main.2034/) (Pan et al., EMNLP 2026)
ACL
- Tengyu Pan, Weilin Zhao, Yutao Sun, Yanghao Li, Jianyong Wang, and Xu Han. 2026. Fast Block Diffusion Training via Multi-Mask Context Sharing. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 46353–46370, Budapest, Hungary. Association for Computational Linguistics.