@inproceedings{lee-kim-2026-memorization,
title = "Memorization or Extraction? Auditing {LLM}-Based Recovery of Scientific Assumptions",
author = "Lee, Suan and
Kim, JaeSeong",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.141/",
pages = "3081--3102",
ISBN = "979-8-89176-439-2",
abstract = "We audit whether benchmarks for LLM-based recovery of scientific assumptions measure corpus-grounded extraction or pretraining memorization. Using author-defined targets for 10 historical AI breakthroughs (frozen 4-case protocol plus 6 post-hoc stress cases) and a ``field-wide paradigmatic assumption'' prompt, GPT-4o{'}s top-10 extractions cover the target in 6/10 cases under a generous any-of-top-$K$ criterion{---}an upper bound that a 10-rater study corroborates for only ${\sim}0.19$ of sampled top-ranked pairs (human{--}rule $\kappa{=}0.70$, $\alpha{=}0.634$). Controlled stress tests structure the diagnosis: a no-corpus control (field name only) still scores 4/4 on the primary shifts (memorization dominates), while a wrong-corpus control collapses to 0/12 across all $4{\times}3$ cross-pairings (corpus can be causally read); a five-LLM canonical sweep (GPT-4o 4/4, Qwen 2/4, Claude 1/4, Llama-3.1 1/4, Mistral 0/4) shows the gap is model-specific rather than closed-vs-open, with format compliance a major but not fully isolable contributor; a Min-K{\%} Prob cross-check actually mis-ranks synthetic non-members as more contaminated than real pre-shift papers, motivating the behavioral framing; and synthetic/niche controls are mixed (causal in 1 of 3 synthetic scenarios, one shows corpus harm). We release an audit kit {---} five factorable controls (no-corpus, wrong-corpus, pair-level precision, ${\geq}2$-LLM, multi-seed; ${\sim}\$15$) {---} as a reusable benchmark-validity probe, with all artifacts released."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lee-kim-2026-memorization">
<titleInfo>
<title>Memorization or Extraction? Auditing LLM-Based Recovery of Scientific Assumptions</title>
</titleInfo>
<name type="personal">
<namePart type="given">Suan</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">JaeSeong</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>We audit whether benchmarks for LLM-based recovery of scientific assumptions measure corpus-grounded extraction or pretraining memorization. Using author-defined targets for 10 historical AI breakthroughs (frozen 4-case protocol plus 6 post-hoc stress cases) and a “field-wide paradigmatic assumption” prompt, GPT-4o’s top-10 extractions cover the target in 6/10 cases under a generous any-of-top-K criterion—an upper bound that a 10-rater study corroborates for only \sim0.19 of sampled top-ranked pairs (human–rule ąppa=0.70, α=0.634). Controlled stress tests structure the diagnosis: a no-corpus control (field name only) still scores 4/4 on the primary shifts (memorization dominates), while a wrong-corpus control collapses to 0/12 across all 4\times3 cross-pairings (corpus can be causally read); a five-LLM canonical sweep (GPT-4o 4/4, Qwen 2/4, Claude 1/4, Llama-3.1 1/4, Mistral 0/4) shows the gap is model-specific rather than closed-vs-open, with format compliance a major but not fully isolable contributor; a Min-K% Prob cross-check actually mis-ranks synthetic non-members as more contaminated than real pre-shift papers, motivating the behavioral framing; and synthetic/niche controls are mixed (causal in 1 of 3 synthetic scenarios, one shows corpus harm). We release an audit kit — five factorable controls (no-corpus, wrong-corpus, pair-level precision, \geq2-LLM, multi-seed; \sim$15) — as a reusable benchmark-validity probe, with all artifacts released.</abstract>
<identifier type="citekey">lee-kim-2026-memorization</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.141/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>3081</start>
<end>3102</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Memorization or Extraction? Auditing LLM-Based Recovery of Scientific Assumptions
%A Lee, Suan
%A Kim, JaeSeong
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F lee-kim-2026-memorization
%X We audit whether benchmarks for LLM-based recovery of scientific assumptions measure corpus-grounded extraction or pretraining memorization. Using author-defined targets for 10 historical AI breakthroughs (frozen 4-case protocol plus 6 post-hoc stress cases) and a “field-wide paradigmatic assumption” prompt, GPT-4o’s top-10 extractions cover the target in 6/10 cases under a generous any-of-top-K criterion—an upper bound that a 10-rater study corroborates for only \sim0.19 of sampled top-ranked pairs (human–rule ąppa=0.70, α=0.634). Controlled stress tests structure the diagnosis: a no-corpus control (field name only) still scores 4/4 on the primary shifts (memorization dominates), while a wrong-corpus control collapses to 0/12 across all 4\times3 cross-pairings (corpus can be causally read); a five-LLM canonical sweep (GPT-4o 4/4, Qwen 2/4, Claude 1/4, Llama-3.1 1/4, Mistral 0/4) shows the gap is model-specific rather than closed-vs-open, with format compliance a major but not fully isolable contributor; a Min-K% Prob cross-check actually mis-ranks synthetic non-members as more contaminated than real pre-shift papers, motivating the behavioral framing; and synthetic/niche controls are mixed (causal in 1 of 3 synthetic scenarios, one shows corpus harm). We release an audit kit — five factorable controls (no-corpus, wrong-corpus, pair-level precision, \geq2-LLM, multi-seed; \sim$15) — as a reusable benchmark-validity probe, with all artifacts released.
%U https://aclanthology.org/2026.aacl-long.141/
%P 3081-3102
Markdown (Informal)
[Memorization or Extraction? Auditing LLM-Based Recovery of Scientific Assumptions](https://aclanthology.org/2026.aacl-long.141/) (Lee & Kim, AACL-IJCNLP 2026)
ACL
- Suan Lee and JaeSeong Kim. 2026. Memorization or Extraction? Auditing LLM-Based Recovery of Scientific Assumptions. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 3081–3102, Hengqin, China. Association for Computational Linguistics.