@inproceedings{hiraoka-inui-2025-repetition,
title = "Repetition Neurons: How Do Language Models Produce Repetitions?",
author = "Hiraoka, Tatsuya and
Inui, Kentaro",
editor = "Chiruzzo, Luis and
Ritter, Alan and
Wang, Lu",
booktitle = "Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 2: Short Papers)",
month = apr,
year = "2025",
address = "Albuquerque, New Mexico",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2025.naacl-short.41/",
doi = "10.18653/v1/2025.naacl-short.41",
pages = "483--495",
ISBN = "979-8-89176-190-2",
abstract = "This paper introduces repetition neurons, which can be regarded as ``skill neurons'' responsible for the repetition problem in text generation tasks. These neurons are progressively activated more strongly as repetition continues, indicating that they perceive repetition as a task to copy the previous context repeatedly, similar to in-context learning. We identify these repetition neurons by comparing activation values before and after the onset of repetition in texts generated by recent pre-trained language models. We analyze the repetition neurons in three English and one Japanese pre-trained language models and observe similar patterns across them."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hiraoka-inui-2025-repetition">
<titleInfo>
<title>Repetition Neurons: How Do Language Models Produce Repetitions?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tatsuya</namePart>
<namePart type="family">Hiraoka</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kentaro</namePart>
<namePart type="family">Inui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2025-04</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 2: Short Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Luis</namePart>
<namePart type="family">Chiruzzo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ritter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Albuquerque, New Mexico</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-190-2</identifier>
</relatedItem>
<abstract>This paper introduces repetition neurons, which can be regarded as “skill neurons” responsible for the repetition problem in text generation tasks. These neurons are progressively activated more strongly as repetition continues, indicating that they perceive repetition as a task to copy the previous context repeatedly, similar to in-context learning. We identify these repetition neurons by comparing activation values before and after the onset of repetition in texts generated by recent pre-trained language models. We analyze the repetition neurons in three English and one Japanese pre-trained language models and observe similar patterns across them.</abstract>
<identifier type="citekey">hiraoka-inui-2025-repetition</identifier>
<identifier type="doi">10.18653/v1/2025.naacl-short.41</identifier>
<location>
<url>https://aclanthology.org/2025.naacl-short.41/</url>
</location>
<part>
<date>2025-04</date>
<extent unit="page">
<start>483</start>
<end>495</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Repetition Neurons: How Do Language Models Produce Repetitions?
%A Hiraoka, Tatsuya
%A Inui, Kentaro
%Y Chiruzzo, Luis
%Y Ritter, Alan
%Y Wang, Lu
%S Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 2: Short Papers)
%D 2025
%8 April
%I Association for Computational Linguistics
%C Albuquerque, New Mexico
%@ 979-8-89176-190-2
%F hiraoka-inui-2025-repetition
%X This paper introduces repetition neurons, which can be regarded as “skill neurons” responsible for the repetition problem in text generation tasks. These neurons are progressively activated more strongly as repetition continues, indicating that they perceive repetition as a task to copy the previous context repeatedly, similar to in-context learning. We identify these repetition neurons by comparing activation values before and after the onset of repetition in texts generated by recent pre-trained language models. We analyze the repetition neurons in three English and one Japanese pre-trained language models and observe similar patterns across them.
%R 10.18653/v1/2025.naacl-short.41
%U https://aclanthology.org/2025.naacl-short.41/
%U https://doi.org/10.18653/v1/2025.naacl-short.41
%P 483-495
Markdown (Informal)
[Repetition Neurons: How Do Language Models Produce Repetitions?](https://aclanthology.org/2025.naacl-short.41/) (Hiraoka & Inui, NAACL 2025)
ACL
- Tatsuya Hiraoka and Kentaro Inui. 2025. Repetition Neurons: How Do Language Models Produce Repetitions?. In Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 2: Short Papers), pages 483–495, Albuquerque, New Mexico. Association for Computational Linguistics.