@inproceedings{abbes-etal-2026-layer,
title = "What Does Layer-Importance Reveal About Transformers and State-Space Models?",
author = "Abbes, Istabrak and
Islah, Nizar and
Rish, Irina and
Chandar, Sarath",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1459/",
pages = "31048--31075",
ISBN = "979-8-89176-462-0",
abstract = "Transformers and state-space models (SSMs) are the two dominant families of sequence models, and a central open question is how far the analytical knowledge built for transformers transfers to SSMs. We address this through the lens of layer importance which underpins compression, selective fine-tuning, and interpretability across both families. We decompose layer importance into two distinct notions. \textit{Necessity} captures how much the pretrained model depends on a layer{'}s existing contribution, measured by the loss increase from bypassing it. \textit{Plasticity} captures where the model absorbs new information during fine-tuning, measured by the magnitude of task-specific weight updates. Our analysis reveals that transformers and SSMs behave fundamentally differently: in residual transformers, Necessity and Plasticity anti-align across depth, whereas in Mamba-style SSMs they point to overlapping regions. The sign of this alignment also predicts downstream adaptation behavior. In transformers, the updates are in the most plastic layers, which increases catastrophic forgetting, while this tier-dependent effect disappears in Mamba-style SSMs"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="abbes-etal-2026-layer">
<titleInfo>
<title>What Does Layer-Importance Reveal About Transformers and State-Space Models?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Istabrak</namePart>
<namePart type="family">Abbes</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nizar</namePart>
<namePart type="family">Islah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Irina</namePart>
<namePart type="family">Rish</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarath</namePart>
<namePart type="family">Chandar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Transformers and state-space models (SSMs) are the two dominant families of sequence models, and a central open question is how far the analytical knowledge built for transformers transfers to SSMs. We address this through the lens of layer importance which underpins compression, selective fine-tuning, and interpretability across both families. We decompose layer importance into two distinct notions. Necessity captures how much the pretrained model depends on a layer’s existing contribution, measured by the loss increase from bypassing it. Plasticity captures where the model absorbs new information during fine-tuning, measured by the magnitude of task-specific weight updates. Our analysis reveals that transformers and SSMs behave fundamentally differently: in residual transformers, Necessity and Plasticity anti-align across depth, whereas in Mamba-style SSMs they point to overlapping regions. The sign of this alignment also predicts downstream adaptation behavior. In transformers, the updates are in the most plastic layers, which increases catastrophic forgetting, while this tier-dependent effect disappears in Mamba-style SSMs</abstract>
<identifier type="citekey">abbes-etal-2026-layer</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1459/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>31048</start>
<end>31075</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Does Layer-Importance Reveal About Transformers and State-Space Models?
%A Abbes, Istabrak
%A Islah, Nizar
%A Rish, Irina
%A Chandar, Sarath
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F abbes-etal-2026-layer
%X Transformers and state-space models (SSMs) are the two dominant families of sequence models, and a central open question is how far the analytical knowledge built for transformers transfers to SSMs. We address this through the lens of layer importance which underpins compression, selective fine-tuning, and interpretability across both families. We decompose layer importance into two distinct notions. Necessity captures how much the pretrained model depends on a layer’s existing contribution, measured by the loss increase from bypassing it. Plasticity captures where the model absorbs new information during fine-tuning, measured by the magnitude of task-specific weight updates. Our analysis reveals that transformers and SSMs behave fundamentally differently: in residual transformers, Necessity and Plasticity anti-align across depth, whereas in Mamba-style SSMs they point to overlapping regions. The sign of this alignment also predicts downstream adaptation behavior. In transformers, the updates are in the most plastic layers, which increases catastrophic forgetting, while this tier-dependent effect disappears in Mamba-style SSMs
%U https://aclanthology.org/2026.findings-emnlp.1459/
%P 31048-31075
Markdown (Informal)
[What Does Layer-Importance Reveal About Transformers and State-Space Models?](https://aclanthology.org/2026.findings-emnlp.1459/) (Abbes et al., Findings 2026)
ACL