@inproceedings{wang-etal-2026-fine,
title = "Are Fine-Tuned {LLM} Membership Signals Real? Separating Memorization from Likelihood Shortcuts",
author = "Wang, Zilong and
Xu, Yingjie and
Wu, Yutong and
Li, Ruiyang and
Ji, Qinglin and
Tang, Xueming and
Lu, Songfeng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1544/",
pages = "32850--32869",
ISBN = "979-8-89176-462-0",
abstract = "Membership inference attacks (MIAs) are increasingly used to audit whether protected examples were included in language model fine-tuning. Yet for fine-tuned LLMs, high attack AUC alone is not reliable evidence of data use: a score may separate members from non-members due to exposure, but also due to length, base model difficulty, typicality, or domain alignment. We propose DeMIA, a diagnostic membership inference framework that learns separate channels for membership scoring and difficulty modeling using likelihood features from the base and target models. DeMIA combines membership prediction with matched-pair ranking, base-difficulty anchoring, and adversarial length control, encouraging the final score to preserve membership-relevant evidence while reducing direct shortcut reliance. Across GPT-Neo-125M, Qwen2.5-1.5B, and Llama-3.1-8B on Wiki10k, AGNews10k, and XSum10k, DeMIA outperforms scalar likelihood, reference-calibrated, neighborhood, calibrated Min-K, and DF-style baselines while maintaining strong low-FPR recall. The signal remains strong on matched hard subsets, while blind controls using length, base loss, or other features from the base model remain near chance. These results show that fine-tuned LLM membership signals can persist under the tested nuisance controls and should be evaluated with reliability diagnostics rather than raw AUC alone."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-fine">
<titleInfo>
<title>Are Fine-Tuned LLM Membership Signals Real? Separating Memorization from Likelihood Shortcuts</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zilong</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yingjie</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yutong</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruiyang</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qinglin</namePart>
<namePart type="family">Ji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xueming</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Songfeng</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Membership inference attacks (MIAs) are increasingly used to audit whether protected examples were included in language model fine-tuning. Yet for fine-tuned LLMs, high attack AUC alone is not reliable evidence of data use: a score may separate members from non-members due to exposure, but also due to length, base model difficulty, typicality, or domain alignment. We propose DeMIA, a diagnostic membership inference framework that learns separate channels for membership scoring and difficulty modeling using likelihood features from the base and target models. DeMIA combines membership prediction with matched-pair ranking, base-difficulty anchoring, and adversarial length control, encouraging the final score to preserve membership-relevant evidence while reducing direct shortcut reliance. Across GPT-Neo-125M, Qwen2.5-1.5B, and Llama-3.1-8B on Wiki10k, AGNews10k, and XSum10k, DeMIA outperforms scalar likelihood, reference-calibrated, neighborhood, calibrated Min-K, and DF-style baselines while maintaining strong low-FPR recall. The signal remains strong on matched hard subsets, while blind controls using length, base loss, or other features from the base model remain near chance. These results show that fine-tuned LLM membership signals can persist under the tested nuisance controls and should be evaluated with reliability diagnostics rather than raw AUC alone.</abstract>
<identifier type="citekey">wang-etal-2026-fine</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1544/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>32850</start>
<end>32869</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Are Fine-Tuned LLM Membership Signals Real? Separating Memorization from Likelihood Shortcuts
%A Wang, Zilong
%A Xu, Yingjie
%A Wu, Yutong
%A Li, Ruiyang
%A Ji, Qinglin
%A Tang, Xueming
%A Lu, Songfeng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wang-etal-2026-fine
%X Membership inference attacks (MIAs) are increasingly used to audit whether protected examples were included in language model fine-tuning. Yet for fine-tuned LLMs, high attack AUC alone is not reliable evidence of data use: a score may separate members from non-members due to exposure, but also due to length, base model difficulty, typicality, or domain alignment. We propose DeMIA, a diagnostic membership inference framework that learns separate channels for membership scoring and difficulty modeling using likelihood features from the base and target models. DeMIA combines membership prediction with matched-pair ranking, base-difficulty anchoring, and adversarial length control, encouraging the final score to preserve membership-relevant evidence while reducing direct shortcut reliance. Across GPT-Neo-125M, Qwen2.5-1.5B, and Llama-3.1-8B on Wiki10k, AGNews10k, and XSum10k, DeMIA outperforms scalar likelihood, reference-calibrated, neighborhood, calibrated Min-K, and DF-style baselines while maintaining strong low-FPR recall. The signal remains strong on matched hard subsets, while blind controls using length, base loss, or other features from the base model remain near chance. These results show that fine-tuned LLM membership signals can persist under the tested nuisance controls and should be evaluated with reliability diagnostics rather than raw AUC alone.
%U https://aclanthology.org/2026.findings-emnlp.1544/
%P 32850-32869
Markdown (Informal)
[Are Fine-Tuned LLM Membership Signals Real? Separating Memorization from Likelihood Shortcuts](https://aclanthology.org/2026.findings-emnlp.1544/) (Wang et al., Findings 2026)
ACL