@inproceedings{zhang-etal-2026-long,
title = "Long-Context Demonstration Selection Using State Space Models",
author = "Zhang, Ziniu and
Zhang, Zhenshuo and
Xiong, Ruoxuan and
Cooperman, Gene and
Zhang, Hongyang R.",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1415/",
doi = "10.18653/2026.findings-emnlp.1415",
pages = "30161--30178",
ISBN = "979-8-89176-462-0",
abstract = "We study the problem of demonstration selection, which involves selecting a subset of examples for pre-pending to a query to a language model. This problem is closely related to in-context learning and model inference. Since the inference cost of a transformer model scales quadratically with sequence length, the selection problem becomes especially challenging in a long-context scenario. In this paper, we tackle this problem by building on state space models (SSMs), which has linear inference time given the input embeddings. Our approach works in two stages. First, we learn a small set of SSMs through a distillation procedure {---} given a (trained) transformer model, we partition all the layers into consecutive (layer) groups {---} for each group, we learn one SSM to mimic the input-output behavior, thereby capturing structural similarity in adjacent layers. Second, we map the SSM outputs to a small set of tokens, and apply them for downstream demonstration selection. We run extensive experiments in both synthetic and real-world datasets to validate this approach. We show that the SSMs trained from distillation incur a relative approximation error of less than 0.7{\%}. As for downstream demonstration selection, we find that on several text classification and reasoning tasks, our approach reduces FLOPs by $14.2\times$ and improves accuracy by 6.48{\%} relative to baseline selection methods."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-long">
<titleInfo>
<title>Long-Context Demonstration Selection Using State Space Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ziniu</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhenshuo</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruoxuan</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gene</namePart>
<namePart type="family">Cooperman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongyang</namePart>
<namePart type="given">R</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>We study the problem of demonstration selection, which involves selecting a subset of examples for pre-pending to a query to a language model. This problem is closely related to in-context learning and model inference. Since the inference cost of a transformer model scales quadratically with sequence length, the selection problem becomes especially challenging in a long-context scenario. In this paper, we tackle this problem by building on state space models (SSMs), which has linear inference time given the input embeddings. Our approach works in two stages. First, we learn a small set of SSMs through a distillation procedure — given a (trained) transformer model, we partition all the layers into consecutive (layer) groups — for each group, we learn one SSM to mimic the input-output behavior, thereby capturing structural similarity in adjacent layers. Second, we map the SSM outputs to a small set of tokens, and apply them for downstream demonstration selection. We run extensive experiments in both synthetic and real-world datasets to validate this approach. We show that the SSMs trained from distillation incur a relative approximation error of less than 0.7%. As for downstream demonstration selection, we find that on several text classification and reasoning tasks, our approach reduces FLOPs by 14.2\times and improves accuracy by 6.48% relative to baseline selection methods.</abstract>
<identifier type="citekey">zhang-etal-2026-long</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1415</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1415/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>30161</start>
<end>30178</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Long-Context Demonstration Selection Using State Space Models
%A Zhang, Ziniu
%A Zhang, Zhenshuo
%A Xiong, Ruoxuan
%A Cooperman, Gene
%A Zhang, Hongyang R.
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhang-etal-2026-long
%X We study the problem of demonstration selection, which involves selecting a subset of examples for pre-pending to a query to a language model. This problem is closely related to in-context learning and model inference. Since the inference cost of a transformer model scales quadratically with sequence length, the selection problem becomes especially challenging in a long-context scenario. In this paper, we tackle this problem by building on state space models (SSMs), which has linear inference time given the input embeddings. Our approach works in two stages. First, we learn a small set of SSMs through a distillation procedure — given a (trained) transformer model, we partition all the layers into consecutive (layer) groups — for each group, we learn one SSM to mimic the input-output behavior, thereby capturing structural similarity in adjacent layers. Second, we map the SSM outputs to a small set of tokens, and apply them for downstream demonstration selection. We run extensive experiments in both synthetic and real-world datasets to validate this approach. We show that the SSMs trained from distillation incur a relative approximation error of less than 0.7%. As for downstream demonstration selection, we find that on several text classification and reasoning tasks, our approach reduces FLOPs by 14.2\times and improves accuracy by 6.48% relative to baseline selection methods.
%R 10.18653/2026.findings-emnlp.1415
%U https://aclanthology.org/2026.findings-emnlp.1415/
%U https://doi.org/10.18653/2026.findings-emnlp.1415
%P 30161-30178
Markdown (Informal)
[Long-Context Demonstration Selection Using State Space Models](https://aclanthology.org/2026.findings-emnlp.1415/) (Zhang et al., Findings 2026)
ACL
- Ziniu Zhang, Zhenshuo Zhang, Ruoxuan Xiong, Gene Cooperman, and Hongyang R. Zhang. 2026. Long-Context Demonstration Selection Using State Space Models. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 30161–30178, Budapest, Hungary. Association for Computational Linguistics.