@inproceedings{zihan-2026-msra,
title = "{MSRA}: A Parameter-Efficient Spectral Correction for Local Attention {---} Where It Helps, and Where It Fails",
author = "Zihan, Tian",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.3/",
pages = "33--42",
ISBN = "979-8-89176-478-1",
abstract = "Restricting attention to a sliding window makes long documents affordable, but it discards everything beyond the window. This paper asks what can be recovered when the model is already trained, its weights are frozen, and the budget for new parameters is only a few tens of thousands. We propose Multi-scale Spectral Residual Attention (MSRA): a far-field branch grafted onto the last few layers of a frozen model, which condenses out-of-window history through smoothly distance-decayed weights and adds it back to the attention output{---}roughly 9K{--}45K trainable parameters, plus minimal 8-to-16-scalar probes for attribution. On PG-19 long-form fiction, the branch closes up to 31.7{\%} of the perplexity gap between local and full attention, with gains stable across two architectures, multiple seeds, and the official test partition. The attribution proves sharper than the headline: 16 learnable output-gain scalars alone close 26.4{\%}, returns accelerate with grafting depth, the carefully designed kernel structure and gating contribute only secondarily{---}and a budget-matched LoRA closes 87.4{\%}, indicating that the gap is mostly generic capacity rather than a missing far-field pathway. Retrieval experiments complete the map: the correction is a statistical smoother that helps just beyond the window, slightly hurts precise in-window prediction, recovers essentially nothing at genuine long distance, and does not transfer to task metrics{---}while slowing prefill two-to-three-fold and leaving decoding unaffected. These results bound what a lightweight spectral correction can and cannot recover, forming a reproducible decision reference for local long-context design."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zihan-2026-msra">
<titleInfo>
<title>MSRA: A Parameter-Efficient Spectral Correction for Local Attention — Where It Helps, and Where It Fails</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tian</namePart>
<namePart type="family">Zihan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>Restricting attention to a sliding window makes long documents affordable, but it discards everything beyond the window. This paper asks what can be recovered when the model is already trained, its weights are frozen, and the budget for new parameters is only a few tens of thousands. We propose Multi-scale Spectral Residual Attention (MSRA): a far-field branch grafted onto the last few layers of a frozen model, which condenses out-of-window history through smoothly distance-decayed weights and adds it back to the attention output—roughly 9K–45K trainable parameters, plus minimal 8-to-16-scalar probes for attribution. On PG-19 long-form fiction, the branch closes up to 31.7% of the perplexity gap between local and full attention, with gains stable across two architectures, multiple seeds, and the official test partition. The attribution proves sharper than the headline: 16 learnable output-gain scalars alone close 26.4%, returns accelerate with grafting depth, the carefully designed kernel structure and gating contribute only secondarily—and a budget-matched LoRA closes 87.4%, indicating that the gap is mostly generic capacity rather than a missing far-field pathway. Retrieval experiments complete the map: the correction is a statistical smoother that helps just beyond the window, slightly hurts precise in-window prediction, recovers essentially nothing at genuine long distance, and does not transfer to task metrics—while slowing prefill two-to-three-fold and leaving decoding unaffected. These results bound what a lightweight spectral correction can and cannot recover, forming a reproducible decision reference for local long-context design.</abstract>
<identifier type="citekey">zihan-2026-msra</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.3/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>33</start>
<end>42</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MSRA: A Parameter-Efficient Spectral Correction for Local Attention — Where It Helps, and Where It Fails
%A Zihan, Tian
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F zihan-2026-msra
%X Restricting attention to a sliding window makes long documents affordable, but it discards everything beyond the window. This paper asks what can be recovered when the model is already trained, its weights are frozen, and the budget for new parameters is only a few tens of thousands. We propose Multi-scale Spectral Residual Attention (MSRA): a far-field branch grafted onto the last few layers of a frozen model, which condenses out-of-window history through smoothly distance-decayed weights and adds it back to the attention output—roughly 9K–45K trainable parameters, plus minimal 8-to-16-scalar probes for attribution. On PG-19 long-form fiction, the branch closes up to 31.7% of the perplexity gap between local and full attention, with gains stable across two architectures, multiple seeds, and the official test partition. The attribution proves sharper than the headline: 16 learnable output-gain scalars alone close 26.4%, returns accelerate with grafting depth, the carefully designed kernel structure and gating contribute only secondarily—and a budget-matched LoRA closes 87.4%, indicating that the gap is mostly generic capacity rather than a missing far-field pathway. Retrieval experiments complete the map: the correction is a statistical smoother that helps just beyond the window, slightly hurts precise in-window prediction, recovers essentially nothing at genuine long distance, and does not transfer to task metrics—while slowing prefill two-to-three-fold and leaving decoding unaffected. These results bound what a lightweight spectral correction can and cannot recover, forming a reproducible decision reference for local long-context design.
%U https://aclanthology.org/2026.groundlm-1.3/
%P 33-42
Markdown (Informal)
[MSRA: A Parameter-Efficient Spectral Correction for Local Attention — Where It Helps, and Where It Fails](https://aclanthology.org/2026.groundlm-1.3/) (Zihan, GroundLM 2026)
ACL