@inproceedings{chen-etal-2026-comparative,
title = "A Comparative Analysis of {LLM} Memorization at Statistical and Internal Levels: Cross-Model Commonalities and Model-Specific Signatures",
author = "Chen, Bowen and
Han, Namgi and
Miyao, Yusuke",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1863/",
pages = "42392--42420",
ISBN = "979-8-89176-459-0",
abstract = "Memorization is a fundamental component of intelligence for both humans and LLMs. However, while LLM performance scales rapidly, our understanding of memorization lags behind. Due to limited access to the pre-training data of LLMs, most previous studies focus on a single model series, leading to isolated observations across series and making it unclear which findings are general and which are model-specific. In this study, we collect multiple model series (Pythia, OpenLLaMA, StarCoder, OLMo1/2/3) and analyze their shared or unique memorization behavior at both the statistical and internal levels, connecting individual observations while showing new findings. At the statistical level, we reveal that the memorization rate scales log-linearly with model size, and memorized sequences can be further compressed. Further analysis demonstrated a shared frequency and domain distribution pattern for memorized sequences. However, different models also show individual features under the above observations. At the internal level, we find that LLMs can remove certain injected perturbations, while memorized sequences are more sensitive. By decoding the middle layers and ablating attention heads, we reveal the general decoding process and the important heads that are shared for memorization. However, the distribution of those important heads differs across families, showing a unique family-level feature. By bridging various experiments and revealing new findings, this study paves the way for a universal and fundamental understanding of memorization in LLMs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chen-etal-2026-comparative">
<titleInfo>
<title>A Comparative Analysis of LLM Memorization at Statistical and Internal Levels: Cross-Model Commonalities and Model-Specific Signatures</title>
</titleInfo>
<name type="personal">
<namePart type="given">Bowen</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Namgi</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yusuke</namePart>
<namePart type="family">Miyao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Memorization is a fundamental component of intelligence for both humans and LLMs. However, while LLM performance scales rapidly, our understanding of memorization lags behind. Due to limited access to the pre-training data of LLMs, most previous studies focus on a single model series, leading to isolated observations across series and making it unclear which findings are general and which are model-specific. In this study, we collect multiple model series (Pythia, OpenLLaMA, StarCoder, OLMo1/2/3) and analyze their shared or unique memorization behavior at both the statistical and internal levels, connecting individual observations while showing new findings. At the statistical level, we reveal that the memorization rate scales log-linearly with model size, and memorized sequences can be further compressed. Further analysis demonstrated a shared frequency and domain distribution pattern for memorized sequences. However, different models also show individual features under the above observations. At the internal level, we find that LLMs can remove certain injected perturbations, while memorized sequences are more sensitive. By decoding the middle layers and ablating attention heads, we reveal the general decoding process and the important heads that are shared for memorization. However, the distribution of those important heads differs across families, showing a unique family-level feature. By bridging various experiments and revealing new findings, this study paves the way for a universal and fundamental understanding of memorization in LLMs.</abstract>
<identifier type="citekey">chen-etal-2026-comparative</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1863/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>42392</start>
<end>42420</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Comparative Analysis of LLM Memorization at Statistical and Internal Levels: Cross-Model Commonalities and Model-Specific Signatures
%A Chen, Bowen
%A Han, Namgi
%A Miyao, Yusuke
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F chen-etal-2026-comparative
%X Memorization is a fundamental component of intelligence for both humans and LLMs. However, while LLM performance scales rapidly, our understanding of memorization lags behind. Due to limited access to the pre-training data of LLMs, most previous studies focus on a single model series, leading to isolated observations across series and making it unclear which findings are general and which are model-specific. In this study, we collect multiple model series (Pythia, OpenLLaMA, StarCoder, OLMo1/2/3) and analyze their shared or unique memorization behavior at both the statistical and internal levels, connecting individual observations while showing new findings. At the statistical level, we reveal that the memorization rate scales log-linearly with model size, and memorized sequences can be further compressed. Further analysis demonstrated a shared frequency and domain distribution pattern for memorized sequences. However, different models also show individual features under the above observations. At the internal level, we find that LLMs can remove certain injected perturbations, while memorized sequences are more sensitive. By decoding the middle layers and ablating attention heads, we reveal the general decoding process and the important heads that are shared for memorization. However, the distribution of those important heads differs across families, showing a unique family-level feature. By bridging various experiments and revealing new findings, this study paves the way for a universal and fundamental understanding of memorization in LLMs.
%U https://aclanthology.org/2026.emnlp-main.1863/
%P 42392-42420
Markdown (Informal)
[A Comparative Analysis of LLM Memorization at Statistical and Internal Levels: Cross-Model Commonalities and Model-Specific Signatures](https://aclanthology.org/2026.emnlp-main.1863/) (Chen et al., EMNLP 2026)
ACL