@inproceedings{zhao-etal-2026-semantic,
title = "Semantic Retrieval Speculative Decoding with Intermediate Representations",
author = "Zhao, Fangming and
Yue, Xiaofei and
Ye, Fulun and
Peng, Yu and
Zhao, Ziming and
Jin, Lewei and
Wu, Wei and
Chen, Tianxiang and
Li, Tingting and
Yin, Jianwei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.40/",
pages = "810--824",
ISBN = "979-8-89176-462-0",
abstract = "Autoregressive decoding remains a key bottleneck for LLM inference, due to its token-by-token nature. Speculative decoding is proposed to reduce latency by drafting multiple tokens and verifying them with fewer target-model passes. However, existing methods either incur auxiliary-drafter overhead or rely on brittle lexical span reuse. In structured tasks (e.g., code generation and text editing), we observe significant locality of reference and semantic redundancy for potential semantic context reuse. In light of this, we present Semantic Retrieval Speculative Decoding (SRSD), a training-free scheme that treats intermediate-layer hidden states as an in-context semantic index. At each decoding step, SRSD retrieves semantically similar past positions. Multiple candidates are merged into a compact tree and verified jointly in a single forward pass. SRSD requires no auxiliary models or external datastores. Across a variety of popular LLMs, SRSD achieves up to 5.42{\texttimes} throughput speedups over state-of-the-art training-free methods and parametric drafters. The code is available at https://github.com/Secbrain/SRSD."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhao-etal-2026-semantic">
<titleInfo>
<title>Semantic Retrieval Speculative Decoding with Intermediate Representations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Fangming</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaofei</namePart>
<namePart type="family">Yue</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fulun</namePart>
<namePart type="family">Ye</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yu</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziming</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lewei</namePart>
<namePart type="family">Jin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianxiang</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tingting</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianwei</namePart>
<namePart type="family">Yin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Autoregressive decoding remains a key bottleneck for LLM inference, due to its token-by-token nature. Speculative decoding is proposed to reduce latency by drafting multiple tokens and verifying them with fewer target-model passes. However, existing methods either incur auxiliary-drafter overhead or rely on brittle lexical span reuse. In structured tasks (e.g., code generation and text editing), we observe significant locality of reference and semantic redundancy for potential semantic context reuse. In light of this, we present Semantic Retrieval Speculative Decoding (SRSD), a training-free scheme that treats intermediate-layer hidden states as an in-context semantic index. At each decoding step, SRSD retrieves semantically similar past positions. Multiple candidates are merged into a compact tree and verified jointly in a single forward pass. SRSD requires no auxiliary models or external datastores. Across a variety of popular LLMs, SRSD achieves up to 5.42× throughput speedups over state-of-the-art training-free methods and parametric drafters. The code is available at https://github.com/Secbrain/SRSD.</abstract>
<identifier type="citekey">zhao-etal-2026-semantic</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.40/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>810</start>
<end>824</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Semantic Retrieval Speculative Decoding with Intermediate Representations
%A Zhao, Fangming
%A Yue, Xiaofei
%A Ye, Fulun
%A Peng, Yu
%A Zhao, Ziming
%A Jin, Lewei
%A Wu, Wei
%A Chen, Tianxiang
%A Li, Tingting
%A Yin, Jianwei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhao-etal-2026-semantic
%X Autoregressive decoding remains a key bottleneck for LLM inference, due to its token-by-token nature. Speculative decoding is proposed to reduce latency by drafting multiple tokens and verifying them with fewer target-model passes. However, existing methods either incur auxiliary-drafter overhead or rely on brittle lexical span reuse. In structured tasks (e.g., code generation and text editing), we observe significant locality of reference and semantic redundancy for potential semantic context reuse. In light of this, we present Semantic Retrieval Speculative Decoding (SRSD), a training-free scheme that treats intermediate-layer hidden states as an in-context semantic index. At each decoding step, SRSD retrieves semantically similar past positions. Multiple candidates are merged into a compact tree and verified jointly in a single forward pass. SRSD requires no auxiliary models or external datastores. Across a variety of popular LLMs, SRSD achieves up to 5.42× throughput speedups over state-of-the-art training-free methods and parametric drafters. The code is available at https://github.com/Secbrain/SRSD.
%U https://aclanthology.org/2026.findings-emnlp.40/
%P 810-824
Markdown (Informal)
[Semantic Retrieval Speculative Decoding with Intermediate Representations](https://aclanthology.org/2026.findings-emnlp.40/) (Zhao et al., Findings 2026)
ACL
- Fangming Zhao, Xiaofei Yue, Fulun Ye, Yu Peng, Ziming Zhao, Lewei Jin, Wei Wu, Tianxiang Chen, Tingting Li, and Jianwei Yin. 2026. Semantic Retrieval Speculative Decoding with Intermediate Representations. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 810–824, Budapest, Hungary. Association for Computational Linguistics.