@inproceedings{spiegel-etal-2026-attend,
title = "Attend or Perish: Benchmarking Attention on Algorithmic Reasoning",
author = "Spiegel, Michal and
{\v{S}}tef{\'a}nik, Michal and
Kadl{\v{c}}{\'i}k, Marek and
Ruscio, Valeria and
Faj{\v{c}}{\'i}k, Martin and
Kucha{\v{r}}, Josef",
editor = "Arad, Dana and
Belinkov, Yonatan and
Chen, Hanjie and
Feldhus, Nils and
Mueller, Aaron and
Mohebbi, Hosein and
Sarti, Gabriele and
Tutek, Martin",
booktitle = "Proceedings of the 9th {B}lackbox{NLP} Workshop: Analyzing and Interpreting Neural Networks for {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.blackboxnlp-1.41/",
pages = "707--725",
ISBN = "979-8-89176-476-7",
abstract = "While Transformer models can learn algorithmic tasks and generalize reliably to unseen, in-distribution data, they often fail catastrophically when required to extrapolate to sequence lengths beyond their training regime. Using AttentionSpan, a suite of synthetic diagnostic tasks designed to enable inspection of internal model computations, we analyze the model{'}s behavior on length extrapolation. Our findings indicate that this failure is not fully explained by an inability to induce general rules for the task. Instead, we identify inconsistent attention patterns as a major contributing factor: information retrieval strategies learned by individual attention heads, which fail to remain stable as sequence length increases. Reinforcing a reference attention pattern at inference time, using no out-of-distribution data, largely recovers accuracy on String Reversal and partially on Value Assignment, while leaving multi-digit arithmetic unresolved. We also study \textit{length adaptation}, how a model trained on one length adjusts to another by fine-tuning its Key and Query parameters, and find that a single dimension in every attention head is enough, across 4 algorithmic tasks and the RULER benchmark. Adaptation does not carry past the fine-tuned lengths, and our interventions are diagnostic rather than a proposed remedy. We make our benchmark and code publicly available."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="spiegel-etal-2026-attend">
<titleInfo>
<title>Attend or Perish: Benchmarking Attention on Algorithmic Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Michal</namePart>
<namePart type="family">Spiegel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Michal</namePart>
<namePart type="family">Štefánik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marek</namePart>
<namePart type="family">Kadlčík</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Valeria</namePart>
<namePart type="family">Ruscio</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Fajčík</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Josef</namePart>
<namePart type="family">Kuchař</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="family">Arad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yonatan</namePart>
<namePart type="family">Belinkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nils</namePart>
<namePart type="family">Feldhus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hosein</namePart>
<namePart type="family">Mohebbi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriele</namePart>
<namePart type="family">Sarti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Tutek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-476-7</identifier>
</relatedItem>
<abstract>While Transformer models can learn algorithmic tasks and generalize reliably to unseen, in-distribution data, they often fail catastrophically when required to extrapolate to sequence lengths beyond their training regime. Using AttentionSpan, a suite of synthetic diagnostic tasks designed to enable inspection of internal model computations, we analyze the model’s behavior on length extrapolation. Our findings indicate that this failure is not fully explained by an inability to induce general rules for the task. Instead, we identify inconsistent attention patterns as a major contributing factor: information retrieval strategies learned by individual attention heads, which fail to remain stable as sequence length increases. Reinforcing a reference attention pattern at inference time, using no out-of-distribution data, largely recovers accuracy on String Reversal and partially on Value Assignment, while leaving multi-digit arithmetic unresolved. We also study length adaptation, how a model trained on one length adjusts to another by fine-tuning its Key and Query parameters, and find that a single dimension in every attention head is enough, across 4 algorithmic tasks and the RULER benchmark. Adaptation does not carry past the fine-tuned lengths, and our interventions are diagnostic rather than a proposed remedy. We make our benchmark and code publicly available.</abstract>
<identifier type="citekey">spiegel-etal-2026-attend</identifier>
<location>
<url>https://aclanthology.org/2026.blackboxnlp-1.41/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>707</start>
<end>725</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Attend or Perish: Benchmarking Attention on Algorithmic Reasoning
%A Spiegel, Michal
%A Štefánik, Michal
%A Kadlčík, Marek
%A Ruscio, Valeria
%A Fajčík, Martin
%A Kuchař, Josef
%Y Arad, Dana
%Y Belinkov, Yonatan
%Y Chen, Hanjie
%Y Feldhus, Nils
%Y Mueller, Aaron
%Y Mohebbi, Hosein
%Y Sarti, Gabriele
%Y Tutek, Martin
%S Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-476-7
%F spiegel-etal-2026-attend
%X While Transformer models can learn algorithmic tasks and generalize reliably to unseen, in-distribution data, they often fail catastrophically when required to extrapolate to sequence lengths beyond their training regime. Using AttentionSpan, a suite of synthetic diagnostic tasks designed to enable inspection of internal model computations, we analyze the model’s behavior on length extrapolation. Our findings indicate that this failure is not fully explained by an inability to induce general rules for the task. Instead, we identify inconsistent attention patterns as a major contributing factor: information retrieval strategies learned by individual attention heads, which fail to remain stable as sequence length increases. Reinforcing a reference attention pattern at inference time, using no out-of-distribution data, largely recovers accuracy on String Reversal and partially on Value Assignment, while leaving multi-digit arithmetic unresolved. We also study length adaptation, how a model trained on one length adjusts to another by fine-tuning its Key and Query parameters, and find that a single dimension in every attention head is enough, across 4 algorithmic tasks and the RULER benchmark. Adaptation does not carry past the fine-tuned lengths, and our interventions are diagnostic rather than a proposed remedy. We make our benchmark and code publicly available.
%U https://aclanthology.org/2026.blackboxnlp-1.41/
%P 707-725
Markdown (Informal)
[Attend or Perish: Benchmarking Attention on Algorithmic Reasoning](https://aclanthology.org/2026.blackboxnlp-1.41/) (Spiegel et al., BlackboxNLP 2026)
ACL
- Michal Spiegel, Michal Štefánik, Marek Kadlčík, Valeria Ruscio, Martin Fajčík, and Josef Kuchař. 2026. Attend or Perish: Benchmarking Attention on Algorithmic Reasoning. In Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP, pages 707–725, Budapest, Hungary. Association for Computational Linguistics.