@inproceedings{kiya-etal-2026-rope,
title = "It{'}s Not {R}o{PE} that Creates Sinks: The Role of Self-Concentration and Value-Non-Mixing in Attention",
author = "Kiya, Raito and
Ohashi, Satoki and
Sato, Kosuke and
Kamoda, Go and
Takahashi, Ryosuke and
Yamamoto, Yuji and
Shiono, Daiki and
Sakaguchi, Keisuke and
Kobayashi, Goro",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1366/",
pages = "31160--31184",
ISBN = "979-8-89176-459-0",
abstract = "Large Language Models (LLMs) often exhibit ``Attention Sink'' (AS) and the accompanying ``Massive Activations'' (MAs) at the initial position of a sequence. These phenomena frequently co-occur, and MAs can pose challenges for low-bit quantization. In this study, we analyze the factors underlying AS and MAs that emerge at the initial position regardless of the token occupying it. Our experiments suggest that self-concentration of attention, resulting from the causal mask, and the subsequent Value-non-mixing in attention outputs contribute to AS and MAs. These findings provide new empirical evidence on the internal dynamics of LLMs, offering insights that may inform future quantization strategies and advance our understanding of the internal mechanisms of attention layers."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kiya-etal-2026-rope">
<titleInfo>
<title>It’s Not RoPE that Creates Sinks: The Role of Self-Concentration and Value-Non-Mixing in Attention</title>
</titleInfo>
<name type="personal">
<namePart type="given">Raito</namePart>
<namePart type="family">Kiya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Satoki</namePart>
<namePart type="family">Ohashi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kosuke</namePart>
<namePart type="family">Sato</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Go</namePart>
<namePart type="family">Kamoda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryosuke</namePart>
<namePart type="family">Takahashi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuji</namePart>
<namePart type="family">Yamamoto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daiki</namePart>
<namePart type="family">Shiono</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keisuke</namePart>
<namePart type="family">Sakaguchi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Goro</namePart>
<namePart type="family">Kobayashi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) often exhibit “Attention Sink” (AS) and the accompanying “Massive Activations” (MAs) at the initial position of a sequence. These phenomena frequently co-occur, and MAs can pose challenges for low-bit quantization. In this study, we analyze the factors underlying AS and MAs that emerge at the initial position regardless of the token occupying it. Our experiments suggest that self-concentration of attention, resulting from the causal mask, and the subsequent Value-non-mixing in attention outputs contribute to AS and MAs. These findings provide new empirical evidence on the internal dynamics of LLMs, offering insights that may inform future quantization strategies and advance our understanding of the internal mechanisms of attention layers.</abstract>
<identifier type="citekey">kiya-etal-2026-rope</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1366/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>31160</start>
<end>31184</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T It’s Not RoPE that Creates Sinks: The Role of Self-Concentration and Value-Non-Mixing in Attention
%A Kiya, Raito
%A Ohashi, Satoki
%A Sato, Kosuke
%A Kamoda, Go
%A Takahashi, Ryosuke
%A Yamamoto, Yuji
%A Shiono, Daiki
%A Sakaguchi, Keisuke
%A Kobayashi, Goro
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kiya-etal-2026-rope
%X Large Language Models (LLMs) often exhibit “Attention Sink” (AS) and the accompanying “Massive Activations” (MAs) at the initial position of a sequence. These phenomena frequently co-occur, and MAs can pose challenges for low-bit quantization. In this study, we analyze the factors underlying AS and MAs that emerge at the initial position regardless of the token occupying it. Our experiments suggest that self-concentration of attention, resulting from the causal mask, and the subsequent Value-non-mixing in attention outputs contribute to AS and MAs. These findings provide new empirical evidence on the internal dynamics of LLMs, offering insights that may inform future quantization strategies and advance our understanding of the internal mechanisms of attention layers.
%U https://aclanthology.org/2026.emnlp-main.1366/
%P 31160-31184
Markdown (Informal)
[It’s Not RoPE that Creates Sinks: The Role of Self-Concentration and Value-Non-Mixing in Attention](https://aclanthology.org/2026.emnlp-main.1366/) (Kiya et al., EMNLP 2026)
ACL
- Raito Kiya, Satoki Ohashi, Kosuke Sato, Go Kamoda, Ryosuke Takahashi, Yuji Yamamoto, Daiki Shiono, Keisuke Sakaguchi, and Goro Kobayashi. 2026. It’s Not RoPE that Creates Sinks: The Role of Self-Concentration and Value-Non-Mixing in Attention. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 31160–31184, Budapest, Hungary. Association for Computational Linguistics.