@inproceedings{wu-zhang-2026-stable,
title = "How Stable Is In-Context Learning? Lipschitz Bounds for Transformer Output under Context Perturbation",
author = "Wu, Jinheng and
Zhang, Di",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1065/",
pages = "24469--24480",
ISBN = "979-8-89176-459-0",
abstract = "In-context learning (ICL) enables large language models to adapt from a few demonstrations, yet its sensitivity to context perturbations remains poorly understood. We study the context-conditioned map $ C \mapsto f_\theta(x; C) $ through Lipschitz analysis. Modeling the Transformer as a composition of attention and feed-forward layers, we derive a worst-case certificate $ |f(x; C + \delta) - f(x; C)| \leq L|\delta| $ and a data-dependent local estimator based on finite-difference Jacobian probes. Across six language models (117M{--}7B) on controlled arithmetic ICL tasks, we find that: (i) output changes saturate as perturbation magnitude increases; (ii) global spectral bounds are extremely conservative, while local estimates are far closer to empirical sensitivity; (iii) sensitivity concentrates in late attention layers; (iv) additional demonstrations often stabilize predictions through redundancy; and (v) perturbation direction matters: label flips have much larger normalized effect than reorderings. SST-2 validation supports the same local-sensitivity and perturbation-direction trends. Our results position Lipschitz analysis less as a tight global predictor and more as a diagnostic framework for locating and comparing ICL vulnerabilities."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wu-zhang-2026-stable">
<titleInfo>
<title>How Stable Is In-Context Learning? Lipschitz Bounds for Transformer Output under Context Perturbation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jinheng</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>In-context learning (ICL) enables large language models to adapt from a few demonstrations, yet its sensitivity to context perturbations remains poorly understood. We study the context-conditioned map C \mapsto f_θ(x; C) through Lipschitz analysis. Modeling the Transformer as a composition of attention and feed-forward layers, we derive a worst-case certificate |f(x; C + δ) - f(x; C)| łeq L|δ| and a data-dependent local estimator based on finite-difference Jacobian probes. Across six language models (117M–7B) on controlled arithmetic ICL tasks, we find that: (i) output changes saturate as perturbation magnitude increases; (ii) global spectral bounds are extremely conservative, while local estimates are far closer to empirical sensitivity; (iii) sensitivity concentrates in late attention layers; (iv) additional demonstrations often stabilize predictions through redundancy; and (v) perturbation direction matters: label flips have much larger normalized effect than reorderings. SST-2 validation supports the same local-sensitivity and perturbation-direction trends. Our results position Lipschitz analysis less as a tight global predictor and more as a diagnostic framework for locating and comparing ICL vulnerabilities.</abstract>
<identifier type="citekey">wu-zhang-2026-stable</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1065/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>24469</start>
<end>24480</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T How Stable Is In-Context Learning? Lipschitz Bounds for Transformer Output under Context Perturbation
%A Wu, Jinheng
%A Zhang, Di
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wu-zhang-2026-stable
%X In-context learning (ICL) enables large language models to adapt from a few demonstrations, yet its sensitivity to context perturbations remains poorly understood. We study the context-conditioned map C \mapsto f_θ(x; C) through Lipschitz analysis. Modeling the Transformer as a composition of attention and feed-forward layers, we derive a worst-case certificate |f(x; C + δ) - f(x; C)| łeq L|δ| and a data-dependent local estimator based on finite-difference Jacobian probes. Across six language models (117M–7B) on controlled arithmetic ICL tasks, we find that: (i) output changes saturate as perturbation magnitude increases; (ii) global spectral bounds are extremely conservative, while local estimates are far closer to empirical sensitivity; (iii) sensitivity concentrates in late attention layers; (iv) additional demonstrations often stabilize predictions through redundancy; and (v) perturbation direction matters: label flips have much larger normalized effect than reorderings. SST-2 validation supports the same local-sensitivity and perturbation-direction trends. Our results position Lipschitz analysis less as a tight global predictor and more as a diagnostic framework for locating and comparing ICL vulnerabilities.
%U https://aclanthology.org/2026.emnlp-main.1065/
%P 24469-24480
Markdown (Informal)
[How Stable Is In-Context Learning? Lipschitz Bounds for Transformer Output under Context Perturbation](https://aclanthology.org/2026.emnlp-main.1065/) (Wu & Zhang, EMNLP 2026)
ACL