@inproceedings{peng-etal-2026-beyond,
title = "Beyond Prefix Answering: Parallel Infilling Reading for Long Contexts with Diffusion Language Models",
author = "Peng, Han and
Dong, Zican and
Liu, Peiyu and
Zou, Jingyuan and
Li, Pengfei and
Zhao, Xin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.282/",
pages = "6463--6476",
ISBN = "979-8-89176-459-0",
abstract = "Diffusion Language Models (DLMs) have emerged as a promising alternative to autoregressive language models, showing strong non-autoregressive generation abilities. Although the masked denoising objective trains DLMs to recover tokens at arbitrary positions, current long-context usage still mostly follows a prefix-based paradigm: the full context is given as a prefix, and the answer is generated only at the end, leaving the model{'}s infilling ability underutilized. To investigate whether DLMs can read long contexts by infilling, we introduce Parallel Infilling Reading (PIR), a training-free method that inserts extraction slots into long contexts to generate intermediate reading states. Our empirical study shows that naive PIR improves over prefix-based DLM inference, but suffers from slot repetition, where multiple slots generate redundant evidence. To mitigate this issue, we propose Slot-Isolated PIR, which blocks direct slot-to-slot attention, and PIR-SFT, which uses fine-grained evidence supervision to teach DLMs effective evidence extraction while reading. Experiments on long-context question answering benchmarks show that our methods improve answer accuracy and generation efficiency, with increasingly larger gains as the context length grows."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="peng-etal-2026-beyond">
<titleInfo>
<title>Beyond Prefix Answering: Parallel Infilling Reading for Long Contexts with Diffusion Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Han</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zican</namePart>
<namePart type="family">Dong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peiyu</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingyuan</namePart>
<namePart type="family">Zou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pengfei</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Diffusion Language Models (DLMs) have emerged as a promising alternative to autoregressive language models, showing strong non-autoregressive generation abilities. Although the masked denoising objective trains DLMs to recover tokens at arbitrary positions, current long-context usage still mostly follows a prefix-based paradigm: the full context is given as a prefix, and the answer is generated only at the end, leaving the model’s infilling ability underutilized. To investigate whether DLMs can read long contexts by infilling, we introduce Parallel Infilling Reading (PIR), a training-free method that inserts extraction slots into long contexts to generate intermediate reading states. Our empirical study shows that naive PIR improves over prefix-based DLM inference, but suffers from slot repetition, where multiple slots generate redundant evidence. To mitigate this issue, we propose Slot-Isolated PIR, which blocks direct slot-to-slot attention, and PIR-SFT, which uses fine-grained evidence supervision to teach DLMs effective evidence extraction while reading. Experiments on long-context question answering benchmarks show that our methods improve answer accuracy and generation efficiency, with increasingly larger gains as the context length grows.</abstract>
<identifier type="citekey">peng-etal-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.282/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>6463</start>
<end>6476</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Prefix Answering: Parallel Infilling Reading for Long Contexts with Diffusion Language Models
%A Peng, Han
%A Dong, Zican
%A Liu, Peiyu
%A Zou, Jingyuan
%A Li, Pengfei
%A Zhao, Xin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F peng-etal-2026-beyond
%X Diffusion Language Models (DLMs) have emerged as a promising alternative to autoregressive language models, showing strong non-autoregressive generation abilities. Although the masked denoising objective trains DLMs to recover tokens at arbitrary positions, current long-context usage still mostly follows a prefix-based paradigm: the full context is given as a prefix, and the answer is generated only at the end, leaving the model’s infilling ability underutilized. To investigate whether DLMs can read long contexts by infilling, we introduce Parallel Infilling Reading (PIR), a training-free method that inserts extraction slots into long contexts to generate intermediate reading states. Our empirical study shows that naive PIR improves over prefix-based DLM inference, but suffers from slot repetition, where multiple slots generate redundant evidence. To mitigate this issue, we propose Slot-Isolated PIR, which blocks direct slot-to-slot attention, and PIR-SFT, which uses fine-grained evidence supervision to teach DLMs effective evidence extraction while reading. Experiments on long-context question answering benchmarks show that our methods improve answer accuracy and generation efficiency, with increasingly larger gains as the context length grows.
%U https://aclanthology.org/2026.emnlp-main.282/
%P 6463-6476
Markdown (Informal)
[Beyond Prefix Answering: Parallel Infilling Reading for Long Contexts with Diffusion Language Models](https://aclanthology.org/2026.emnlp-main.282/) (Peng et al., EMNLP 2026)
ACL