@inproceedings{ni-etal-2026-enabling,
title = "Enabling Confidence-Aware Control of Effective Memorization in Large Language Models",
author = "Ni, Chenjie and
Mersa, Omid and
Wang, Zhepeng and
Bao, Runxue and
Li, Qun and
Gao, Shangqian and
Zhang, Yanfu",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2252/",
pages = "51233--51246",
ISBN = "979-8-89176-459-0",
abstract = "Current evaluations of memorization in Large Language Models (LLMs) predominantly rely on overlap ratios, which fails to distinguish between sporadic token matches and reliable, sequential recall. What{'}s more, these methods become inapplicable in practical scenarios where ground truth suffixes is unavailable. To address these limitations, we propose a paradigm shift from quantifying ``rates'' to determining the Effective Memorization Length{---}the distinct boundary where a model transitions from faithful recall to open-ended generation. We introduce a Ensemble Memorization Classifier that leverages internal states from the LLM to estimate this boundary in a reference-free manner. This framework enables Confidence-Aware Generation Control, allowing to extract high-precision suffixes by dynamically truncating sequences precisely at the memorization boundary. Furthermore, we demonstrate that this boundary is optimizable; by integrating our classifier as a reward model within a reinforcement learning framework, we actively extend the average effective length of generated suffixes. Experiments on the GPT-Neo and Pythia families demonstrate that our approach not only accurately filters non-memorization tokens but also effectively ``unlocks'' deeper chains of memorized knowledge, transforming memorization from an unpredictable risk into a controllable utility."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ni-etal-2026-enabling">
<titleInfo>
<title>Enabling Confidence-Aware Control of Effective Memorization in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Chenjie</namePart>
<namePart type="family">Ni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omid</namePart>
<namePart type="family">Mersa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhepeng</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Runxue</namePart>
<namePart type="family">Bao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qun</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shangqian</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanfu</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Current evaluations of memorization in Large Language Models (LLMs) predominantly rely on overlap ratios, which fails to distinguish between sporadic token matches and reliable, sequential recall. What’s more, these methods become inapplicable in practical scenarios where ground truth suffixes is unavailable. To address these limitations, we propose a paradigm shift from quantifying “rates” to determining the Effective Memorization Length—the distinct boundary where a model transitions from faithful recall to open-ended generation. We introduce a Ensemble Memorization Classifier that leverages internal states from the LLM to estimate this boundary in a reference-free manner. This framework enables Confidence-Aware Generation Control, allowing to extract high-precision suffixes by dynamically truncating sequences precisely at the memorization boundary. Furthermore, we demonstrate that this boundary is optimizable; by integrating our classifier as a reward model within a reinforcement learning framework, we actively extend the average effective length of generated suffixes. Experiments on the GPT-Neo and Pythia families demonstrate that our approach not only accurately filters non-memorization tokens but also effectively “unlocks” deeper chains of memorized knowledge, transforming memorization from an unpredictable risk into a controllable utility.</abstract>
<identifier type="citekey">ni-etal-2026-enabling</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2252/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>51233</start>
<end>51246</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Enabling Confidence-Aware Control of Effective Memorization in Large Language Models
%A Ni, Chenjie
%A Mersa, Omid
%A Wang, Zhepeng
%A Bao, Runxue
%A Li, Qun
%A Gao, Shangqian
%A Zhang, Yanfu
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F ni-etal-2026-enabling
%X Current evaluations of memorization in Large Language Models (LLMs) predominantly rely on overlap ratios, which fails to distinguish between sporadic token matches and reliable, sequential recall. What’s more, these methods become inapplicable in practical scenarios where ground truth suffixes is unavailable. To address these limitations, we propose a paradigm shift from quantifying “rates” to determining the Effective Memorization Length—the distinct boundary where a model transitions from faithful recall to open-ended generation. We introduce a Ensemble Memorization Classifier that leverages internal states from the LLM to estimate this boundary in a reference-free manner. This framework enables Confidence-Aware Generation Control, allowing to extract high-precision suffixes by dynamically truncating sequences precisely at the memorization boundary. Furthermore, we demonstrate that this boundary is optimizable; by integrating our classifier as a reward model within a reinforcement learning framework, we actively extend the average effective length of generated suffixes. Experiments on the GPT-Neo and Pythia families demonstrate that our approach not only accurately filters non-memorization tokens but also effectively “unlocks” deeper chains of memorized knowledge, transforming memorization from an unpredictable risk into a controllable utility.
%U https://aclanthology.org/2026.emnlp-main.2252/
%P 51233-51246
Markdown (Informal)
[Enabling Confidence-Aware Control of Effective Memorization in Large Language Models](https://aclanthology.org/2026.emnlp-main.2252/) (Ni et al., EMNLP 2026)
ACL