@inproceedings{xie-etal-2026-flexee,
title = "{F}lex{EE}: Self-Speculative and {KV}-Compatible Early Exiting for Offloading-Aware {LLM} Inference",
author = "Xie, Qihu and
Li, Ziwei and
Kang, Yi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1426/",
doi = "10.18653/2026.findings-emnlp.1426",
pages = "30396--30411",
ISBN = "979-8-89176-462-0",
abstract = "Large language model (LLM) inference is often constrained by both computation and memory, especially in offloading-based deployments where model weights are transferred across memory hierarchies during autoregressive decoding. In this setting, reducing the number of executed layers can lower token-generation latency while also avoiding costly weight movement. Motivated by this observation, we present **FlexEE**, an early exiting framework for resource-constrained and offloading-based LLM inference. FlexEE makes early exiting practical for LLM decoding through layer-wise exit supervision for reliable intermediate-layer prediction, self-speculative decoding over a Top-K local vocabulary for low-cost exit decisions, and dynamic hidden state management for KV-cache-correct and memory-aware execution. Across generative and downstream tasks, FlexEE enables efficient early exit with minimal accuracy degradation, delivering up to 1.27$\times$/3.16$\times$ and 1.25$\times$/2.83$\times$ end-to-end speedups on Llama2-7B and Llama3-8B under 0{\%}/50{\%} weight offloading, respectively."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xie-etal-2026-flexee">
<titleInfo>
<title>FlexEE: Self-Speculative and KV-Compatible Early Exiting for Offloading-Aware LLM Inference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Qihu</namePart>
<namePart type="family">Xie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziwei</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language model (LLM) inference is often constrained by both computation and memory, especially in offloading-based deployments where model weights are transferred across memory hierarchies during autoregressive decoding. In this setting, reducing the number of executed layers can lower token-generation latency while also avoiding costly weight movement. Motivated by this observation, we present **FlexEE**, an early exiting framework for resource-constrained and offloading-based LLM inference. FlexEE makes early exiting practical for LLM decoding through layer-wise exit supervision for reliable intermediate-layer prediction, self-speculative decoding over a Top-K local vocabulary for low-cost exit decisions, and dynamic hidden state management for KV-cache-correct and memory-aware execution. Across generative and downstream tasks, FlexEE enables efficient early exit with minimal accuracy degradation, delivering up to 1.27\times/3.16\times and 1.25\times/2.83\times end-to-end speedups on Llama2-7B and Llama3-8B under 0%/50% weight offloading, respectively.</abstract>
<identifier type="citekey">xie-etal-2026-flexee</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1426</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1426/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>30396</start>
<end>30411</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T FlexEE: Self-Speculative and KV-Compatible Early Exiting for Offloading-Aware LLM Inference
%A Xie, Qihu
%A Li, Ziwei
%A Kang, Yi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F xie-etal-2026-flexee
%X Large language model (LLM) inference is often constrained by both computation and memory, especially in offloading-based deployments where model weights are transferred across memory hierarchies during autoregressive decoding. In this setting, reducing the number of executed layers can lower token-generation latency while also avoiding costly weight movement. Motivated by this observation, we present **FlexEE**, an early exiting framework for resource-constrained and offloading-based LLM inference. FlexEE makes early exiting practical for LLM decoding through layer-wise exit supervision for reliable intermediate-layer prediction, self-speculative decoding over a Top-K local vocabulary for low-cost exit decisions, and dynamic hidden state management for KV-cache-correct and memory-aware execution. Across generative and downstream tasks, FlexEE enables efficient early exit with minimal accuracy degradation, delivering up to 1.27\times/3.16\times and 1.25\times/2.83\times end-to-end speedups on Llama2-7B and Llama3-8B under 0%/50% weight offloading, respectively.
%R 10.18653/2026.findings-emnlp.1426
%U https://aclanthology.org/2026.findings-emnlp.1426/
%U https://doi.org/10.18653/2026.findings-emnlp.1426
%P 30396-30411
Markdown (Informal)
[FlexEE: Self-Speculative and KV-Compatible Early Exiting for Offloading-Aware LLM Inference](https://aclanthology.org/2026.findings-emnlp.1426/) (Xie et al., Findings 2026)
ACL