@inproceedings{ronaghi-etal-2026-clinically,
title = "Clinically Grounded Privacy Evaluation of Medical {LM}s",
author = "Ronaghi, Sasha and
Tonekaboni, Sana and
Stempfle, Lena and
Utti, Vivian and
Cahoon, Jordan Li and
Hendrix, Nathaniel and
Vala, Ayin and
Ghassemi, Marzyeh and
Alsentzer, Emily",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2388/",
pages = "54266--54295",
ISBN = "979-8-89176-459-0",
abstract = "Medical language models (LMs) can memorize and reproduce protected health information, but privacy evaluations often focus on recovery of training text rather than disclosure under realistic threat models. We introduce a clinically grounded framework that evaluates leakage along a graded axis of adversarial access, ranging from publicly inferable demographics to leaked note fragments. At each tier, we measure verbatim memorization of patient-specific text and semantic leakage of sensitive diagnoses. Applying the framework to an LM continually pretrained on 378k clinical notes, we find that routine encounter metadata (i.e., name, date of birth, visit date, provider name, and practice location) elicits high rates of verbatim memorization across a patient{'}s timeline and sensitive-diagnosis recovery (AUROC 0.91 for abortion, 0.82 for HIV). At the same time, exact-match memorization can overstate disclosure: 36{\%} of memorized tokens reflect templated documentation. Our work highlights the risks of training on longitudinal clinical data and provides a practical, reusable framework for contextual privacy evaluation of medical LMs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ronaghi-etal-2026-clinically">
<titleInfo>
<title>Clinically Grounded Privacy Evaluation of Medical LMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sasha</namePart>
<namePart type="family">Ronaghi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sana</namePart>
<namePart type="family">Tonekaboni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lena</namePart>
<namePart type="family">Stempfle</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vivian</namePart>
<namePart type="family">Utti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jordan</namePart>
<namePart type="given">Li</namePart>
<namePart type="family">Cahoon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nathaniel</namePart>
<namePart type="family">Hendrix</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayin</namePart>
<namePart type="family">Vala</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marzyeh</namePart>
<namePart type="family">Ghassemi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Emily</namePart>
<namePart type="family">Alsentzer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Medical language models (LMs) can memorize and reproduce protected health information, but privacy evaluations often focus on recovery of training text rather than disclosure under realistic threat models. We introduce a clinically grounded framework that evaluates leakage along a graded axis of adversarial access, ranging from publicly inferable demographics to leaked note fragments. At each tier, we measure verbatim memorization of patient-specific text and semantic leakage of sensitive diagnoses. Applying the framework to an LM continually pretrained on 378k clinical notes, we find that routine encounter metadata (i.e., name, date of birth, visit date, provider name, and practice location) elicits high rates of verbatim memorization across a patient’s timeline and sensitive-diagnosis recovery (AUROC 0.91 for abortion, 0.82 for HIV). At the same time, exact-match memorization can overstate disclosure: 36% of memorized tokens reflect templated documentation. Our work highlights the risks of training on longitudinal clinical data and provides a practical, reusable framework for contextual privacy evaluation of medical LMs.</abstract>
<identifier type="citekey">ronaghi-etal-2026-clinically</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2388/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>54266</start>
<end>54295</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Clinically Grounded Privacy Evaluation of Medical LMs
%A Ronaghi, Sasha
%A Tonekaboni, Sana
%A Stempfle, Lena
%A Utti, Vivian
%A Cahoon, Jordan Li
%A Hendrix, Nathaniel
%A Vala, Ayin
%A Ghassemi, Marzyeh
%A Alsentzer, Emily
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F ronaghi-etal-2026-clinically
%X Medical language models (LMs) can memorize and reproduce protected health information, but privacy evaluations often focus on recovery of training text rather than disclosure under realistic threat models. We introduce a clinically grounded framework that evaluates leakage along a graded axis of adversarial access, ranging from publicly inferable demographics to leaked note fragments. At each tier, we measure verbatim memorization of patient-specific text and semantic leakage of sensitive diagnoses. Applying the framework to an LM continually pretrained on 378k clinical notes, we find that routine encounter metadata (i.e., name, date of birth, visit date, provider name, and practice location) elicits high rates of verbatim memorization across a patient’s timeline and sensitive-diagnosis recovery (AUROC 0.91 for abortion, 0.82 for HIV). At the same time, exact-match memorization can overstate disclosure: 36% of memorized tokens reflect templated documentation. Our work highlights the risks of training on longitudinal clinical data and provides a practical, reusable framework for contextual privacy evaluation of medical LMs.
%U https://aclanthology.org/2026.emnlp-main.2388/
%P 54266-54295
Markdown (Informal)
[Clinically Grounded Privacy Evaluation of Medical LMs](https://aclanthology.org/2026.emnlp-main.2388/) (Ronaghi et al., EMNLP 2026)
ACL
- Sasha Ronaghi, Sana Tonekaboni, Lena Stempfle, Vivian Utti, Jordan Li Cahoon, Nathaniel Hendrix, Ayin Vala, Marzyeh Ghassemi, and Emily Alsentzer. 2026. Clinically Grounded Privacy Evaluation of Medical LMs. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 54266–54295, Budapest, Hungary. Association for Computational Linguistics.