@inproceedings{hastuti-etal-2026-evaluating,
title = "Evaluating {LLM} Watermarking for Clinical {EHR}: Detectability, Style Shifts, and Helpfulness",
author = "Hastuti, Rochana Prih and
Rajagede, Rian Adam and
Zheng, Mengxin and
Lou, Qian",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2579/",
pages = "58616--58639",
ISBN = "979-8-89176-459-0",
abstract = "LLM Watermarking for clinical text requires evaluating detection capability alongside task performance. While existing benchmarks focus on accuracy, we show this misses critical quality degradation in Electronic Health Record (EHR) transformation tasks, where technical notes are simplified for patient understanding. We evaluate five watermarking methods across two EHR transformation tasks spanning five clinical note types, developing a multi-aspect framework that measures accuracy, style, and readability. We adapt linguistic distance of Part-of-Speech tags (LD$_{\text{PoS}}$) to quantify style shifts and enable mechanistic analysis.\par Results reveal a fundamental watermarking-induced helpfulness degradation and its detectability tension: EHR transformation achieves only 7.7{\%}{--}24.2{\%} detection rates under default parameters (vs. $>$90{\%} in narrative tasks). Tuning for practical detection degrades helpfulness by 24.6{\%}{--}32.5{\%}. Mechanistic analysis via LD$_{\text{PoS}}$ shows watermarking systematically shifts style toward generic terms and connected structures, degrading helpfulness even when accuracy is preserved."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hastuti-etal-2026-evaluating">
<titleInfo>
<title>Evaluating LLM Watermarking for Clinical EHR: Detectability, Style Shifts, and Helpfulness</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rochana</namePart>
<namePart type="given">Prih</namePart>
<namePart type="family">Hastuti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rian</namePart>
<namePart type="given">Adam</namePart>
<namePart type="family">Rajagede</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mengxin</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qian</namePart>
<namePart type="family">Lou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>LLM Watermarking for clinical text requires evaluating detection capability alongside task performance. While existing benchmarks focus on accuracy, we show this misses critical quality degradation in Electronic Health Record (EHR) transformation tasks, where technical notes are simplified for patient understanding. We evaluate five watermarking methods across two EHR transformation tasks spanning five clinical note types, developing a multi-aspect framework that measures accuracy, style, and readability. We adapt linguistic distance of Part-of-Speech tags (LD_\textPoS) to quantify style shifts and enable mechanistic analysis.\par Results reveal a fundamental watermarking-induced helpfulness degradation and its detectability tension: EHR transformation achieves only 7.7%–24.2% detection rates under default parameters (vs. >90% in narrative tasks). Tuning for practical detection degrades helpfulness by 24.6%–32.5%. Mechanistic analysis via LD_\textPoS shows watermarking systematically shifts style toward generic terms and connected structures, degrading helpfulness even when accuracy is preserved.</abstract>
<identifier type="citekey">hastuti-etal-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2579/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>58616</start>
<end>58639</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating LLM Watermarking for Clinical EHR: Detectability, Style Shifts, and Helpfulness
%A Hastuti, Rochana Prih
%A Rajagede, Rian Adam
%A Zheng, Mengxin
%A Lou, Qian
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F hastuti-etal-2026-evaluating
%X LLM Watermarking for clinical text requires evaluating detection capability alongside task performance. While existing benchmarks focus on accuracy, we show this misses critical quality degradation in Electronic Health Record (EHR) transformation tasks, where technical notes are simplified for patient understanding. We evaluate five watermarking methods across two EHR transformation tasks spanning five clinical note types, developing a multi-aspect framework that measures accuracy, style, and readability. We adapt linguistic distance of Part-of-Speech tags (LD_\textPoS) to quantify style shifts and enable mechanistic analysis.\par Results reveal a fundamental watermarking-induced helpfulness degradation and its detectability tension: EHR transformation achieves only 7.7%–24.2% detection rates under default parameters (vs. >90% in narrative tasks). Tuning for practical detection degrades helpfulness by 24.6%–32.5%. Mechanistic analysis via LD_\textPoS shows watermarking systematically shifts style toward generic terms and connected structures, degrading helpfulness even when accuracy is preserved.
%U https://aclanthology.org/2026.emnlp-main.2579/
%P 58616-58639
Markdown (Informal)
[Evaluating LLM Watermarking for Clinical EHR: Detectability, Style Shifts, and Helpfulness](https://aclanthology.org/2026.emnlp-main.2579/) (Hastuti et al., EMNLP 2026)
ACL