@inproceedings{rathi-bauer-2026-linear,
title = "Linear Layer Repair: Residual-Controlled Layer Profiling for Fault-Tolerant Transformer Inference",
author = "Rathi, Jiya Sheetal and
Bauer, Andr{\'e}",
editor = "Arad, Dana and
Belinkov, Yonatan and
Chen, Hanjie and
Feldhus, Nils and
Mueller, Aaron and
Mohebbi, Hosein and
Sarti, Gabriele and
Tutek, Martin",
booktitle = "Proceedings of the 9th {B}lackbox{NLP} Workshop: Analyzing and Interpreting Neural Networks for {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.blackboxnlp-1.31/",
pages = "514--522",
ISBN = "979-8-89176-476-7",
abstract = "Deciding which transformer layers can be cheaply replaced - when a node fails mid-computation, or when a block is pruned - requires knowing what each layer actually computes. A straightforward approach is to fit a linear map between consecutive hidden states and read off its $R^2$. In our analysis, these fits look excellent, exceeding $R^2$ of 0.97. However this number is largely an artifact of the residual stream: an untrained network scores just as well. The informative quantity is instead how linearly predictable a layer{'}s update $h_{L+1}-h_L$ is. Measured this way, and across six models from three architecture families, layers fall into a clear U-shaped pattern: those at the beginning and end of the stack perform large but almost perfectly linear transformations, while those in the middle perform small nonlinear ones. Given these insights, we present Linear Layer Repair, a policy that reads a layer{'}s profile and prescribes one of three responses when it goes missing: (i) skip it, (ii) swap in a fitted linear map, or (iii) declare it unrecoverable. Using WikiText-103, we test Linear Layer Repair by removing layers and measuring the damage. The results show that the linear map beats plain skipping at all 16 LLaMA-3.2-1B layers and 26 of 28 in Qwen2-1.5B; replacing a single middle layer costs under half a point of perplexity against a baseline of 15; even four consecutive layers cost only $\sim$2.5 points; and at an early layer, where skipping raises perplexity above $10^5$, the patch reduces it more than a thousand-fold. Task-level evaluations show corresponding recovery on prediction-sensitive behavior."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rathi-bauer-2026-linear">
<titleInfo>
<title>Linear Layer Repair: Residual-Controlled Layer Profiling for Fault-Tolerant Transformer Inference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jiya</namePart>
<namePart type="given">Sheetal</namePart>
<namePart type="family">Rathi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">André</namePart>
<namePart type="family">Bauer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="family">Arad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yonatan</namePart>
<namePart type="family">Belinkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nils</namePart>
<namePart type="family">Feldhus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hosein</namePart>
<namePart type="family">Mohebbi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriele</namePart>
<namePart type="family">Sarti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Tutek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-476-7</identifier>
</relatedItem>
<abstract>Deciding which transformer layers can be cheaply replaced - when a node fails mid-computation, or when a block is pruned - requires knowing what each layer actually computes. A straightforward approach is to fit a linear map between consecutive hidden states and read off its R². In our analysis, these fits look excellent, exceeding R² of 0.97. However this number is largely an artifact of the residual stream: an untrained network scores just as well. The informative quantity is instead how linearly predictable a layer’s update h_L+1-h_L is. Measured this way, and across six models from three architecture families, layers fall into a clear U-shaped pattern: those at the beginning and end of the stack perform large but almost perfectly linear transformations, while those in the middle perform small nonlinear ones. Given these insights, we present Linear Layer Repair, a policy that reads a layer’s profile and prescribes one of three responses when it goes missing: (i) skip it, (ii) swap in a fitted linear map, or (iii) declare it unrecoverable. Using WikiText-103, we test Linear Layer Repair by removing layers and measuring the damage. The results show that the linear map beats plain skipping at all 16 LLaMA-3.2-1B layers and 26 of 28 in Qwen2-1.5B; replacing a single middle layer costs under half a point of perplexity against a baseline of 15; even four consecutive layers cost only \sim2.5 points; and at an early layer, where skipping raises perplexity above 10⁵, the patch reduces it more than a thousand-fold. Task-level evaluations show corresponding recovery on prediction-sensitive behavior.</abstract>
<identifier type="citekey">rathi-bauer-2026-linear</identifier>
<location>
<url>https://aclanthology.org/2026.blackboxnlp-1.31/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>514</start>
<end>522</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Linear Layer Repair: Residual-Controlled Layer Profiling for Fault-Tolerant Transformer Inference
%A Rathi, Jiya Sheetal
%A Bauer, André
%Y Arad, Dana
%Y Belinkov, Yonatan
%Y Chen, Hanjie
%Y Feldhus, Nils
%Y Mueller, Aaron
%Y Mohebbi, Hosein
%Y Sarti, Gabriele
%Y Tutek, Martin
%S Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-476-7
%F rathi-bauer-2026-linear
%X Deciding which transformer layers can be cheaply replaced - when a node fails mid-computation, or when a block is pruned - requires knowing what each layer actually computes. A straightforward approach is to fit a linear map between consecutive hidden states and read off its R². In our analysis, these fits look excellent, exceeding R² of 0.97. However this number is largely an artifact of the residual stream: an untrained network scores just as well. The informative quantity is instead how linearly predictable a layer’s update h_L+1-h_L is. Measured this way, and across six models from three architecture families, layers fall into a clear U-shaped pattern: those at the beginning and end of the stack perform large but almost perfectly linear transformations, while those in the middle perform small nonlinear ones. Given these insights, we present Linear Layer Repair, a policy that reads a layer’s profile and prescribes one of three responses when it goes missing: (i) skip it, (ii) swap in a fitted linear map, or (iii) declare it unrecoverable. Using WikiText-103, we test Linear Layer Repair by removing layers and measuring the damage. The results show that the linear map beats plain skipping at all 16 LLaMA-3.2-1B layers and 26 of 28 in Qwen2-1.5B; replacing a single middle layer costs under half a point of perplexity against a baseline of 15; even four consecutive layers cost only \sim2.5 points; and at an early layer, where skipping raises perplexity above 10⁵, the patch reduces it more than a thousand-fold. Task-level evaluations show corresponding recovery on prediction-sensitive behavior.
%U https://aclanthology.org/2026.blackboxnlp-1.31/
%P 514-522
Markdown (Informal)
[Linear Layer Repair: Residual-Controlled Layer Profiling for Fault-Tolerant Transformer Inference](https://aclanthology.org/2026.blackboxnlp-1.31/) (Rathi & Bauer, BlackboxNLP 2026)
ACL