@inproceedings{rheinboldt-etal-2026-treeflash,
title = "{T}ree{F}lash: Parallel {AR}-Approximation for Faster Speculative Decoding",
author = "Rheinboldt, Peer and
Berdoz, Fr{\'e}d{\'e}ric and
Wattenhofer, Roger",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2125/",
doi = "10.18653/2026.findings-emnlp.2125",
pages = "45297--45310",
ISBN = "979-8-89176-462-0",
abstract = "One-shot block drafters for speculative decoding generate the full draft in a single forward pass, achieving strong throughput by eliminating sequential token generation. However, they predict each draft token conditioned only on the prefix context, with no dependence on previously drafted tokens. This non-autoregressive conditioning causes the drafter{'}s distribution to diverge from the verifier{'}s true autoregressive distribution as draft depth grows. This problem becomes more severe in tree-based drafting, where distinct branches are forced to share the same marginal distribution for subsequent tokens. We propose \textbf{TreeFlash}, which addresses this by incorporating an MLP layer conditioned on the drafter{'}s hidden state and the previous token to approximate an autoregressive distribution. TreeFlash retains the $\mathcal{O}(1)$ decoding time complexity of one-shot drafters by employing a two-stage approximation mechanism. TreeFlash achieves state-of-the-art performance across a variety of tasks and models, improving over marginal tree drafting by 12{\%} higher block efficiency and 9{\%} higher speedup at a drafting budget of 64."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rheinboldt-etal-2026-treeflash">
<titleInfo>
<title>TreeFlash: Parallel AR-Approximation for Faster Speculative Decoding</title>
</titleInfo>
<name type="personal">
<namePart type="given">Peer</namePart>
<namePart type="family">Rheinboldt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Frédéric</namePart>
<namePart type="family">Berdoz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Roger</namePart>
<namePart type="family">Wattenhofer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>One-shot block drafters for speculative decoding generate the full draft in a single forward pass, achieving strong throughput by eliminating sequential token generation. However, they predict each draft token conditioned only on the prefix context, with no dependence on previously drafted tokens. This non-autoregressive conditioning causes the drafter’s distribution to diverge from the verifier’s true autoregressive distribution as draft depth grows. This problem becomes more severe in tree-based drafting, where distinct branches are forced to share the same marginal distribution for subsequent tokens. We propose TreeFlash, which addresses this by incorporating an MLP layer conditioned on the drafter’s hidden state and the previous token to approximate an autoregressive distribution. TreeFlash retains the \mathcalO(1) decoding time complexity of one-shot drafters by employing a two-stage approximation mechanism. TreeFlash achieves state-of-the-art performance across a variety of tasks and models, improving over marginal tree drafting by 12% higher block efficiency and 9% higher speedup at a drafting budget of 64.</abstract>
<identifier type="citekey">rheinboldt-etal-2026-treeflash</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.2125</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2125/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>45297</start>
<end>45310</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T TreeFlash: Parallel AR-Approximation for Faster Speculative Decoding
%A Rheinboldt, Peer
%A Berdoz, Frédéric
%A Wattenhofer, Roger
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F rheinboldt-etal-2026-treeflash
%X One-shot block drafters for speculative decoding generate the full draft in a single forward pass, achieving strong throughput by eliminating sequential token generation. However, they predict each draft token conditioned only on the prefix context, with no dependence on previously drafted tokens. This non-autoregressive conditioning causes the drafter’s distribution to diverge from the verifier’s true autoregressive distribution as draft depth grows. This problem becomes more severe in tree-based drafting, where distinct branches are forced to share the same marginal distribution for subsequent tokens. We propose TreeFlash, which addresses this by incorporating an MLP layer conditioned on the drafter’s hidden state and the previous token to approximate an autoregressive distribution. TreeFlash retains the \mathcalO(1) decoding time complexity of one-shot drafters by employing a two-stage approximation mechanism. TreeFlash achieves state-of-the-art performance across a variety of tasks and models, improving over marginal tree drafting by 12% higher block efficiency and 9% higher speedup at a drafting budget of 64.
%R 10.18653/2026.findings-emnlp.2125
%U https://aclanthology.org/2026.findings-emnlp.2125/
%U https://doi.org/10.18653/2026.findings-emnlp.2125
%P 45297-45310
Markdown (Informal)
[TreeFlash: Parallel AR-Approximation for Faster Speculative Decoding](https://aclanthology.org/2026.findings-emnlp.2125/) (Rheinboldt et al., Findings 2026)
ACL